Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(E2E): Make sure the E2E tests can fully test that the application is reachable on K8s (using an Ingress on GH) #672

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

chore(E2E): Make sure the E2E tests can fully test that the application is reachable on K8s (using an Ingress on GH) #672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/nightly.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,19 @@ jobs:
- name: Start Minikube
if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
uses: medyagh/setup-minikube@d8c0eb871f6f455542491d86a574477bd3894533 # v0.0.18
with:
addons: ingress

- name: Build Ingress Domain
if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
run: |
echo "K8S_INGRESS_DOMAIN=$(minikube ip).sslip.io" >> $GITHUB_ENV

- name: Run E2E tests
if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
env:
BACKSTAGE_OPERATOR_TESTS_PLATFORM: minikube
BACKSTAGE_OPERATOR_TESTS_K8S_CREATE_INGRESS: 'true'
BACKSTAGE_OPERATOR_TESTS_K8S_INGRESS_DOMAIN: ${{ env.K8S_INGRESS_DOMAIN }}
IMG: ${{ env.OPERATOR_IMAGE }}
run: make test-e2e
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,7 @@ ENVTEST_VERSION ?= release-0.17
GOLANGCI_LINT_VERSION ?= v1.59.1
GOIMPORTS_VERSION ?= v0.16.1
GOSEC_VERSION ?= v2.20.0
GINKGO_VERSION ?= v2.22.1
GINKGO_VERSION ?= v2.22.2

## Gosec options - default format is sarif so we can integrate with Github code scanning
GOSEC_FMT ?= sarif # for other options, see https://github.com/securego/gosec#output-formats
Expand Down
6 changes: 4 additions & 2 deletions tests/e2e/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,17 @@ The behavior is configurable using the following environment variables:
| Name | Type | Description | Default value | Example |
|------------------------------------------------------------------------------------------------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------|---------------------------------------------------------|
| `BACKSTAGE_OPERATOR_TEST_MODE` | string | The test mode:<br>- if not set, it will call `make deploy`<br>- `olm`: it will call `make deploy-olm`<br>- `rhdh-latest` or `rhdh-next`: it will install the operator using the [`install-rhdh-catalog-source.sh`](../../.rhdh/scripts/install-rhdh-catalog-source.sh) script<br>- `rhdh-airgap`: it will install the operator using the [`prepare-restricted-environment.sh`](../../.rhdh/scripts/prepare-restricted-environment.sh) script. | | `rhdh-latest` |
| `IMG` (or any variables from the Makefile that are used by `make deploy` or `make deploy-olm`) | string | The image to use. Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is not set or set to `olm` | `VERSION` defined in [`Makefile`](../../Makefile) | `quay.io/rhdh/rhdh-rhel9-operator:0.0.1-latest` |
| `IMG` (or any variables from the Makefile that are used by `make deploy` or `make deploy-olm`) | string | The image to use. Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is not set or set to `olm` | `VERSION` defined in [`Makefile`](../../Makefile) | `quay.io/rhdh/rhdh-rhel9-operator:0.0.1-latest` |
| `BACKSTAGE_OPERATOR_TESTS_BUILD_IMAGES` | bool | If set to `true`, it will build the operator image with `make image-build`.<br>Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is not set or set to `olm`. | | `false` |
| `BACKSTAGE_OPERATOR_TESTS_PUSH_IMAGES` | bool | If set to `true`, it will push the operator image with `make image-push`.<br>Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is not set or set to `olm`. | | `false` |
| `BACKSTAGE_OPERATOR_TESTS_PLATFORM` | string | The platform type, to directly load the operator image if supported instead of pushing it.<br>Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is not set or set to `olm`.br>Supported values: [`kind`](#building-and-testing-local-changes-on-kind), [`k3d`](#building-and-testing-local-changes-on-k3d), [`minikube`](#building-and-testing-local-changes-on-minikube) | | `kind` |
| `BACKSTAGE_OPERATOR_TESTS_KIND_CLUSTER` | string | Name of the local KinD cluster to use. Relevant only if `BACKSTAGE_OPERATOR_TESTS_PLATFORM` is `kind`. | `kind` | `kind-local-k8s-cluster` |
| `BACKSTAGE_OPERATOR_TESTS_K3D_CLUSTER` | string | Name of the local k3d cluster to use. Relevant only if `BACKSTAGE_OPERATOR_TESTS_PLATFORM` is `k3d`. | `k3s-default` | `k3d-local-k8s-cluster` |
| `BACKSTAGE_OPERATOR_TESTS_AIRGAP_INDEX_IMAGE` | string | Index image to use in the airgap scenario.<br>Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is `rhdh-airgap`. | `quay.io/rhdh/iib:latest-v4.14-x86_64` | `registry.redhat.io/redhat/redhat-operator-index:v4.14` |
| `BACKSTAGE_OPERATOR_TESTS_AIRGAP_OPERATOR_VERSION` | string | Operator version to use in the airgap scenario.<br>Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is `rhdh-airgap`. | `v1.1.0` | `v1.1.0` |
| `BACKSTAGE_OPERATOR_TESTS_AIRGAP_MIRROR_REGISTRY` | string | Existing mirror registry to use in the airgap scenario.<br>Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is `rhdh-airgap`<br>. | | `my-registry.example.com` |
| `BACKSTAGE_OPERATOR_TESTS_AIRGAP_MIRROR_REGISTRY` | string | Existing mirror registry to use in the airgap scenario.<br>Relevant if `BACKSTAGE_OPERATOR_TEST_MODE` is `rhdh-airgap`. | | `my-registry.example.com` |
| `BACKSTAGE_OPERATOR_TESTS_K8S_CREATE_INGRESS` | bool | Whether to test access using an Ingress resource on K8s | | `true` |
| `BACKSTAGE_OPERATOR_TESTS_K8S_INGRESS_DOMAIN` | string | Ingress domain. Relevant only if `BACKSTAGE_OPERATOR_TESTS_K8S_CREATE_INGRESS` is `true`. | | `$(minikube ip).nip.io` |

### Examples

Expand Down
48 changes: 48 additions & 0 deletions tests/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package e2e
import (
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
Expand Down Expand Up @@ -301,6 +302,53 @@ subjects:
ensureRouteIsReachable(ns, tt.crName, tt.additionalApiEndpointTests)
})
}
} else {
// This is how we currently instruct users to deploy the application on vanilla K8s clusters,
// where an Ingress resource is not created OOTB by the Operator.
// TODO(rm3l): this is until https://issues.redhat.com/browse/RHIDP-2176 is supported.
// For now, we want to make sure the tests cover the same area as on OpenShift, i.e.,
// making sure that the application is reachable end-to-end from a user standpoint.
if os.Getenv("BACKSTAGE_OPERATOR_TESTS_K8S_CREATE_INGRESS") == "true" {
ingressDomain := os.Getenv("BACKSTAGE_OPERATOR_TESTS_K8S_INGRESS_DOMAIN")
if ingressDomain == "" {
Fail("Ingress Domain should be configured via the BACKSTAGE_OPERATOR_TESTS_K8S_INGRESS_DOMAIN env var")
}
ingressHost := fmt.Sprintf("%s.%s", tt.crName, ingressDomain)
By("manually creating a K8s Ingress", func() {
cmd := exec.Command(helper.GetPlatformTool(), "-n", ns, "create", "-f", "-")
stdin, err := cmd.StdinPipe()
ExpectWithOffset(1, err).NotTo(HaveOccurred())
go func() {
defer stdin.Close()
_, _ = io.WriteString(stdin, fmt.Sprintf(`
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: %[1]s
spec:
rules:
- host: %[2]s
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: backstage-%[1]s
port:
name: http-backend
`, tt.crName, ingressHost))
}()
_, err = helper.Run(cmd)
Expect(err).ShouldNot(HaveOccurred())
})

By("ensuring the application is fully reachable", func() {
Eventually(helper.VerifyBackstageAppAccess, 8*time.Minute, time.Second).
WithArguments(fmt.Sprintf("http://%s", ingressHost), tt.additionalApiEndpointTests).
Should(Succeed())
})
}
}

var isRouteEnabledNow bool
Expand Down
6 changes: 5 additions & 1 deletion tests/helper/helper_backstage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,11 @@ func VerifyBackstageRoute(g Gomega, ns string, crName string, tests []ApiEndpoin
fmt.Fprintln(GinkgoWriter, host)
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(host).ShouldNot(BeEmpty())
baseUrl := fmt.Sprintf("https://%s", host)

VerifyBackstageAppAccess(g, fmt.Sprintf("https://%s", host), tests)
}

func VerifyBackstageAppAccess(g Gomega, baseUrl string, tests []ApiEndpointTest) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // #nosec G402 -- test code only, not used in production
Expand Down Expand Up @@ -213,6 +216,7 @@ func VerifyBackstageRoute(g Gomega, ns string, crName string, tests []ApiEndpoin
g.Expect(bodyStr).Should(tt.BodyMatcher, "context: "+tt.Endpoint)
}
}

allTests := append(defaultApiEndpointTests, tests...)
for _, tt := range allTests {
performTest(tt)
Expand Down
Loading