Skip to content

Commit

Permalink
fix: lagoon.persistent.name override checks (#383)
Browse files Browse the repository at this point in the history 
* fix: lagoon.persistent.name override checks

* test: fix autogenroutes after modifying service label checks

* fix: use persistent name as volume name

* test: add a complex persistent test
  • Loading branch information
@shreddedbacon
shreddedbacon authored Oct 22, 2024
1 parent b899eb6 commit 8af640d
Show file tree
Hide file tree
Showing 51 changed files with 2,811 additions and 104 deletions.
32 changes: 30 additions & 2 deletions cmd/template_autogen_ingress_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,10 @@ func TestAutogeneratedIngressGeneration(t *testing.T) {
want: "internal/testdata/node/autogen-templates/ingress-10",
},
{
name: "test13 autogenerated route development service type override",
name: "test-development-service-type-override-a",
description: `this test should fail because the replacement type has no persistent label in the associated docker-compose file
realistically, this is not something that should be done on an already deployed service due to changing
service types not being properly supported`,
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
Expand All @@ -230,7 +233,32 @@ func TestAutogeneratedIngressGeneration(t *testing.T) {
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/node/autogen-templates/ingress-11",
emptyDir: true,
want: "",
wantErr: true,
},
{
name: "test-development-service-type-override-b",
description: `this test should pass because the replacement service type is just a basic type
realistically, this is not something that should be done on an already deployed service due to changing
service types not being properly supported`,
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
EnvironmentType: "development",
LagoonYAML: "internal/testdata/node/lagoon.yml",
ProjectVariables: []lagoon.EnvironmentVariable{
{
Name: "LAGOON_SERVICE_TYPES",
Value: "node:basic",
Scope: "build",
},
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/node/autogen-templates/test-development-service-type-override",
},
{
name: "test14 autogenerated route development no service type override",
Expand Down
62 changes: 62 additions & 0 deletions cmd/template_lagoonservices_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,68 @@ func TestTemplateLagoonServices(t *testing.T) {
templatePath: "testoutput",
want: "internal/testdata/basic/service-templates/test-basic-spot-affinity",
},
{
name: "test-basic-persistent-name",
description: "tests a basic deployment with a persistent name",
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/basic/lagoon.persistent-name.yml",
ImageReferences: map[string]string{
"basic": "harbor.example/example-project/main/basic@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/basic/service-templates/test-basic-persistent-name",
},
{
name: "test-basic-persistent-names",
description: "tests a basic deployment with two services with persistent name",
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/basic/lagoon.persistent-name-2.yml",
ImageReferences: map[string]string{
"basic": "harbor.example/example-project/main/basic@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"basic2": "harbor.example/example-project/main/basic2@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/basic/service-templates/test-basic-persistent-names",
},
{
name: "test-complex-persistent-names",
description: "tests a complex deployment with two services with persistent name",
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/complex/lagoon.persistent-name.yml",
ImageReferences: map[string]string{
"cli": "harbor.example/example-project/main/cli@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"gotenberg": "harbor.example/example-project/main/gotenberg@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"nginx": "harbor.example/example-project/main/nginx@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"opensearch": "harbor.example/example-project/main/opensearch@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"php": "harbor.example/example-project/main/php@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"rabbitmq": "harbor.example/example-project/main/rabbitmq@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"redis": "harbor.example/example-project/main/redis@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"redis-persist": "harbor.example/example-project/main/redis-persist@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"redis-session": "harbor.example/example-project/main/redis-session@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"queue-worker-entity-index": "harbor.example/example-project/main/queue-worker-entity-index@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"queue-worker-priority-high": "harbor.example/example-project/main/queue-worker-priority-high@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"queue-worker-priority-instant": "harbor.example/example-project/main/queue-worker-priority-instant@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"queue-worker-priority-low": "harbor.example/example-project/main/queue-worker-priority-low@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"queue-worker-priority-medium": "harbor.example/example-project/main/queue-worker-priority-medium@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/complex/service-templates/test-complex-persistent-names",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions internal/generator/buildvalues.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ type ServiceValues struct {
IsDBaaS bool `json:"isDBaaS"`
IsSingle bool `json:"isSingle"`
AdditionalVolumes []ServiceVolume `json:"additonalVolumes,omitempty"`
CreateDefaultVolume bool `json:"createDefaultVolume"`
}

type ImageBuild struct {
Expand Down
6 changes: 6 additions & 0 deletions internal/generator/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,12 @@ func NewGenerator(
return nil, err
}

// strip out duplicate default volumes
err = flagDefaultVolumeCreation(&buildValues)
if err != nil {
return nil, err
}

if imageCacheBuildArgsJSON != "" {
err = json.Unmarshal([]byte(imageCacheBuildArgsJSON), &buildValues.ImageCacheBuildArguments)
if err != nil {
Expand Down
110 changes: 55 additions & 55 deletions internal/generator/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,61 +247,6 @@ func composeToServiceValues(
lagoonOverrideName = composeService
}

// check if the service has any persistent labels, this is the path that the volume will be mounted to
servicePersistentPath := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent")
if servicePersistentPath == "" {
// if there is no persistent path, check if the service type has a default path
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentPath = val.Volumes.PersistentVolumePath
// check if the service type provides or consumes a default persistent volume
if (val.ProvidesPersistentVolume || val.ConsumesPersistentVolume) && servicePersistentPath == "" {
return nil, fmt.Errorf("label lagoon.persistent not defined for service %s, no valid mount path was found", composeService)
}
}
}
servicePersistentName := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.name")
if servicePersistentName == "" && servicePersistentPath != "" {
// if there is a persistent path defined, then set the persistent name to be the compose service if no persistent name is provided
// persistent name is used by joined services like nginx/php or cli or worker pods to mount another service volume
servicePersistentName = lagoonOverrideName
}
servicePersistentSize := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.size")
if servicePersistentSize == "" {
// if there is no persistent size, check if the service type has a default size allocated
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentSize = val.Volumes.PersistentVolumeSize
// check if the service type provides persistent volume, and that a size was detected
if val.ProvidesPersistentVolume && servicePersistentSize == "" {
return nil, fmt.Errorf("label lagoon.persistent.size not defined for service %s, no valid size was found", composeService)
}
}
}
if servicePersistentSize != "" {
// check the provided size is a valid resource size for kubernetes
_, err := ValidateResourceSize(servicePersistentSize)
if err != nil {
return nil, fmt.Errorf("provided persistent volume size for %s is not valid: %v", servicePersistentName, err)
}
}

// if any `lagoon.base.image` labels are set, we note them for docker pulling
// this allows us to refresh the docker-host's cache in cases where an image
// may have an update without a change in tag (i.e. "latest" tagged images)
baseimage := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.base.image")
if baseimage != "" {
baseImageWithTag, errs := determineRefreshImage(composeService, baseimage, buildValues.EnvironmentVariables)
if len(errs) > 0 {
for idx, err := range errs {
if idx+1 == len(errs) {
return nil, err
} else {
fmt.Println(err)
}
}
}
buildValues.ForcePullImages = append(buildValues.ForcePullImages, baseImageWithTag)
}

// if there are overrides defined in the lagoon API `LAGOON_SERVICE_TYPES`
// handle those here
if buildValues.ServiceTypeOverrides != nil {
Expand Down Expand Up @@ -388,6 +333,61 @@ func composeToServiceValues(
}
}

// check if the service has any persistent labels, this is the path that the volume will be mounted to
servicePersistentPath := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent")
if servicePersistentPath == "" {
// if there is no persistent path, check if the service type has a default path
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentPath = val.Volumes.PersistentVolumePath
// check if the service type provides or consumes a default persistent volume
if (val.ProvidesPersistentVolume || val.ConsumesPersistentVolume) && servicePersistentPath == "" {
return nil, fmt.Errorf("label lagoon.persistent not defined for service %s, no valid mount path was found", composeService)
}
}
}
servicePersistentName := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.name")
if servicePersistentName == "" && servicePersistentPath != "" {
// if there is a persistent path defined, then set the persistent name to be the compose service if no persistent name is provided
// persistent name is used by joined services like nginx/php or cli or worker pods to mount another service volume
servicePersistentName = lagoonOverrideName
}
servicePersistentSize := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.size")
if servicePersistentSize == "" {
// if there is no persistent size, check if the service type has a default size allocated
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentSize = val.Volumes.PersistentVolumeSize
// check if the service type provides persistent volume, and that a size was detected
if val.ProvidesPersistentVolume && servicePersistentSize == "" {
return nil, fmt.Errorf("label lagoon.persistent.size not defined for service %s, no valid size was found", composeService)
}
}
}
if servicePersistentSize != "" {
// check the provided size is a valid resource size for kubernetes
_, err := ValidateResourceSize(servicePersistentSize)
if err != nil {
return nil, fmt.Errorf("provided persistent volume size for %s is not valid: %v", servicePersistentName, err)
}
}

// if any `lagoon.base.image` labels are set, we note them for docker pulling
// this allows us to refresh the docker-host's cache in cases where an image
// may have an update without a change in tag (i.e. "latest" tagged images)
baseimage := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.base.image")
if baseimage != "" {
baseImageWithTag, errs := determineRefreshImage(composeService, baseimage, buildValues.EnvironmentVariables)
if len(errs) > 0 {
for idx, err := range errs {
if idx+1 == len(errs) {
return nil, err
} else {
fmt.Println(err)
}
}
}
buildValues.ForcePullImages = append(buildValues.ForcePullImages, baseImageWithTag)
}

// calculate if this service needs any additional volumes attached from the calculated build volumes
// additional volumes can only be attached to certain
serviceVolumes, err := calculateServiceVolumes(buildValues, lagoonType, servicePersistentName, composeServiceValues.Labels)
Expand Down
64 changes: 53 additions & 11 deletions internal/generator/services_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"testing"
"time"

"github.com/andreyvit/diff"
composetypes "github.com/compose-spec/compose-go/types"
"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
"github.com/uselagoon/build-deploy-tool/internal/helpers"
Expand All @@ -19,10 +20,11 @@ func Test_composeToServiceValues(t *testing.T) {
composeServiceValues composetypes.ServiceConfig
}
tests := []struct {
name string
args args
want *ServiceValues
wantErr bool
name string
description string
args args
want *ServiceValues
wantErr bool
}{
{
name: "test1",
Expand Down Expand Up @@ -201,7 +203,9 @@ func Test_composeToServiceValues(t *testing.T) {
wantErr: true,
},
{
name: "test4 - variable servicetypes type override",
name: "test-variable-servicetypes-type-override",
description: `this test should fail to change the nginx service from nginx-php to nginx-php-persistent
this is because there is no lagoon.persistent labels on the dockercompose service to suit this change`,
args: args{
buildValues: &BuildValues{
Namespace: "example-project-main",
Expand Down Expand Up @@ -231,10 +235,46 @@ func Test_composeToServiceValues(t *testing.T) {
},
},
},
want: nil,
wantErr: true,
},
{
name: "test-variable-servicetypes-type-override-b",
description: `this test should pasee and change the nginx service from nginx to basic
this is because there is nothing really special between these two services besides the port`,
args: args{
buildValues: &BuildValues{
Namespace: "example-project-main",
Project: "example-project",
ImageRegistry: "harbor.example",
Environment: "main",
Branch: "main",
BuildType: "branch",
ServiceTypeOverrides: &lagoon.EnvironmentVariable{
Name: "LAGOON_SERVICE_TYPES",
Value: "nginx:basic,mariadb:mariadb-dbaas",
},
LagoonYAML: lagoon.YAML{
Environments: lagoon.Environments{
"main": lagoon.Environment{},
},
},
},
composeService: "nginx",
composeServiceValues: composetypes.ServiceConfig{
Labels: composetypes.Labels{
"lagoon.type": "nginx",
},
Build: &composetypes.BuildConfig{
Context: ".",
Dockerfile: "../testdata/basic/docker/basic.dockerfile",
},
},
},
want: &ServiceValues{
Name: "nginx",
OverrideName: "nginx",
Type: "nginx-php-persistent",
Type: "basic",
AutogeneratedRoutesEnabled: true,
AutogeneratedRoutesTLSAcme: true,
InPodCronjobs: []lagoon.Cronjob{},
Expand All @@ -245,7 +285,6 @@ func Test_composeToServiceValues(t *testing.T) {
DockerFile: "../testdata/basic/docker/basic.dockerfile",
BuildImage: "harbor.example/example-project/main/nginx:latest",
},
BackupsEnabled: true,
},
},
{
Expand Down Expand Up @@ -558,6 +597,9 @@ func Test_composeToServiceValues(t *testing.T) {
Type: "mariadb-single",
AutogeneratedRoutesEnabled: false,
AutogeneratedRoutesTLSAcme: false,
PersistentVolumePath: "/var/lib/mysql",
PersistentVolumeName: "mariadb",
PersistentVolumeSize: "5Gi",
DBaaSEnvironment: "development2",
InPodCronjobs: []lagoon.Cronjob{},
NativeCronjobs: []lagoon.Cronjob{},
Expand Down Expand Up @@ -1252,10 +1294,10 @@ func Test_composeToServiceValues(t *testing.T) {
t.Errorf("composeToServiceValues() error = %v, wantErr %v", err, tt.wantErr)
return
}
lValues, _ := json.Marshal(got)
wValues, _ := json.Marshal(tt.want)
if !reflect.DeepEqual(string(lValues), string(wValues)) {
t.Errorf("composeToServiceValues() = %v, want %v", string(lValues), string(wValues))
f1, _ := json.MarshalIndent(got, "", " ")
r1, _ := json.MarshalIndent(tt.want, "", " ")
if !reflect.DeepEqual(f1, r1) {
t.Errorf("composeToServiceValues() = \n%v", diff.LineDiff(string(r1), string(f1)))
}
})
}
Expand Down
Loading

0 comments on commit 8af640d

Please sign in to comment.