From 2d120210a2dce1de0eaac740f63f1f18ccc2556a Mon Sep 17 00:00:00 2001 From: Boris Ilyushonak Date: Wed, 18 Dec 2024 12:48:46 +0100 Subject: [PATCH 1/4] chore: add support 8.16.2 to go-kibana-rest and acc tests --- .github/workflows/test.yml | 4 ++-- Makefile | 2 +- libs/go-kibana-rest/CONTRIBUTING.md | 15 +++++++++++++++ libs/go-kibana-rest/docker-compose.yml | 7 +++---- .../kbapi/api.kibana_spaces_test.go | 2 +- .../kbapi/api.kibana_synthetics_test.go | 19 ------------------- 6 files changed, 22 insertions(+), 27 deletions(-) create mode 100644 libs/go-kibana-rest/CONTRIBUTING.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd3b336d3..6d5793abd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,7 +153,7 @@ jobs: - id: setup-fleet name: Setup Fleet - if: matrix.version == '8.10.3' || matrix.version == '8.11.4' || matrix.version == '8.12.2' || matrix.version == '8.13.4' || matrix.version == '8.14.3' || matrix.version == '8.15.3' + if: matrix.version == '8.10.3' || matrix.version == '8.11.4' || matrix.version == '8.12.2' || matrix.version == '8.13.4' || matrix.version == '8.14.3' || matrix.version == '8.15.3' || matrix.version == '8.16.2' run: |- make setup-kibana-fleet env: @@ -162,7 +162,7 @@ jobs: - id: force-install-synthetics name: Force install synthetics - if: matrix.version == '8.14.3' || matrix.version == '8.15.3' + if: matrix.version == '8.14.3' || matrix.version == '8.15.3' || matrix.version == '8.16.2' run: |- for i in {1..5}; do curl -s -H "Authorization: ApiKey ${{ steps.get-api-key.outputs.apikey }}" --header "Content-Type: application/json" --header "kbn-xsrf: true" --request POST --data '{ "force": true }' http://localhost:5601/api/fleet/epm/packages/synthetics/1.2.2 && break || sleep 15; done diff --git a/Makefile b/Makefile index 0dbb5beb7..604bba1ca 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ SWAGGER_VERSION ?= 8.7 GOVERSION ?= $(shell grep -e '^go' go.mod | cut -f 2 -d ' ') -STACK_VERSION ?= 8.15.3 +STACK_VERSION ?= 8.16.2 ELASTICSEARCH_NAME ?= terraform-elasticstack-es ELASTICSEARCH_ENDPOINTS ?= http://$(ELASTICSEARCH_NAME):9200 diff --git a/libs/go-kibana-rest/CONTRIBUTING.md b/libs/go-kibana-rest/CONTRIBUTING.md new file mode 100644 index 000000000..bb08ac7a9 --- /dev/null +++ b/libs/go-kibana-rest/CONTRIBUTING.md @@ -0,0 +1,15 @@ +# Unit tests + +To run the unit tests, you can use the following command: +```bash + +docker compose up -d +make test +``` + +One could enable debug output for tests by setting environment variable `DEBUG=true`, that helps to see http requests and responses. + +Here is an example of environment variables to run a single test, e.g. from IDE or CLI: +```bash +DEBUG=true TEST="-run TestKBAPITestSuite/TestKibanaSpaces ./..." make test +``` \ No newline at end of file diff --git a/libs/go-kibana-rest/docker-compose.yml b/libs/go-kibana-rest/docker-compose.yml index 5cf3a9740..efcd51590 100644 --- a/libs/go-kibana-rest/docker-compose.yml +++ b/libs/go-kibana-rest/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.16.1@sha256:500b04e9282aeffd083fb8f917ee2c309c99f76c095c79681bd77ab37d6886e6 + image: docker.elastic.co/elasticsearch/elasticsearch:8.16.2 environment: cluster.name: test discovery.type: single-node @@ -11,7 +11,7 @@ services: ports: - "9200:9200/tcp" set-kibana-password: - image: docker.elastic.co/kibana/kibana:8.16.1@sha256:a22f9f9bb16fc7b01cfd03523a2d5a0d9873daefec6092368a8e2bb0b3380757 + image: docker.elastic.co/kibana/kibana:8.16.2 restart: on-failure links: - elasticsearch @@ -22,14 +22,13 @@ services: elasticsearch: condition: service_started kibana: - image: docker.elastic.co/kibana/kibana:8.16.1@sha256:a22f9f9bb16fc7b01cfd03523a2d5a0d9873daefec6092368a8e2bb0b3380757 + image: docker.elastic.co/kibana/kibana:8.16.2 environment: SERVER_NAME: kibana ELASTICSEARCH_HOSTS: http://es:9200 ELASTICSEARCH_USERNAME: kibana_system ELASTICSEARCH_PASSWORD: changeme XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: min-32-byte-long-strong-encryption-key -# LOGGING_ROOT_LEVEL: debug links: - elasticsearch:es ports: diff --git a/libs/go-kibana-rest/kbapi/api.kibana_spaces_test.go b/libs/go-kibana-rest/kbapi/api.kibana_spaces_test.go index 02596a9f0..6fcde0b59 100644 --- a/libs/go-kibana-rest/kbapi/api.kibana_spaces_test.go +++ b/libs/go-kibana-rest/kbapi/api.kibana_spaces_test.go @@ -42,7 +42,7 @@ func (s *KBAPITestSuite) TestKibanaSpaces() { Objects: []KibanaSpaceObjectParameter{ { Type: "config", - ID: "8.15.0", + ID: "8.16.2", }, }, } diff --git a/libs/go-kibana-rest/kbapi/api.kibana_synthetics_test.go b/libs/go-kibana-rest/kbapi/api.kibana_synthetics_test.go index fd527cf17..4d665df1b 100644 --- a/libs/go-kibana-rest/kbapi/api.kibana_synthetics_test.go +++ b/libs/go-kibana-rest/kbapi/api.kibana_synthetics_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "testing" - "time" "github.com/go-resty/resty/v2" "github.com/google/uuid" @@ -369,7 +368,6 @@ func (s *KBAPITestSuite) TestKibanaSyntheticsMonitorAPI() { monitor, err := syntheticsAPI.Monitor.Add(ctx, config, fields, space) assert.NoError(s.T(), err) assert.NotNil(s.T(), monitor) - updateDueToKibanaAPIDiff(monitor) get, err := syntheticsAPI.Monitor.Get(ctx, monitor.Id, space) assert.NoError(s.T(), err) @@ -382,11 +380,9 @@ func (s *KBAPITestSuite) TestKibanaSyntheticsMonitorAPI() { update, err := syntheticsAPI.Monitor.Update(ctx, monitor.Id, tc.update.config, tc.update.fields, space) assert.NoError(s.T(), err) assert.NotNil(s.T(), update) - updateDueToKibanaAPIDiff(update) get, err = syntheticsAPI.Monitor.Get(ctx, monitor.ConfigId, space) assert.NoError(s.T(), err) - get.CreatedAt = time.Time{} // update response doesn't have created_at field assert.Equal(s.T(), update, get) deleted, err := syntheticsAPI.Monitor.Delete(ctx, space, monitor.ConfigId) @@ -410,21 +406,6 @@ func (s *KBAPITestSuite) TestKibanaSyntheticsMonitorAPI() { } } -// see https://github.com/elastic/kibana/issues/189906 -func updateDueToKibanaAPIDiff(m *SyntheticsMonitor) { - m.Params = nil - m.Username = "" - m.Password = "" - m.ProxyHeaders = nil - m.CheckResponseBodyPositive = nil - m.CheckRequestBody = nil - m.CheckRequestHeaders = nil - m.CheckSend = "" - m.CheckReceive = "" - m.InlineScript = "" - m.SyntheticsArgs = nil -} - func (s *KBAPITestSuite) TestKibanaSyntheticsPrivateLocationAPI() { ctx := context.Background() From ae4b47bfce95f6e34e43489011c50884fe10e429 Mon Sep 17 00:00:00 2001 From: Boris Ilyushonak Date: Wed, 18 Dec 2024 13:30:50 +0100 Subject: [PATCH 2/4] try to fix GetDataStreamLifecycle --- CONTRIBUTING.md | 10 ++++++++++ internal/clients/elasticsearch/index.go | 7 +++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..2108d8cc3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,10 @@ +## Acceptance tests + +```bash +make docker-testacc +``` + +Run a single test with terraform debug enabled: +```bash +make docker-testacc TF_LOG=DEBUG TESTARGS='-run ^TestAccResourceDataStreamLifecycle$$' +``` \ No newline at end of file diff --git a/internal/clients/elasticsearch/index.go b/internal/clients/elasticsearch/index.go index 6af28ba8e..c2de64d5c 100644 --- a/internal/clients/elasticsearch/index.go +++ b/internal/clients/elasticsearch/index.go @@ -548,11 +548,14 @@ func GetDataStreamLifecycle(ctx context.Context, apiClient *clients.ApiClient, d return nil, utils.FrameworkDiagsFromSDK(diags) } - dStreams := make(map[string][]models.DataStreamLifecycle) + dStreams := struct { + DataStreams []models.DataStreamLifecycle `json:"data_streams,omitempty"` + }{} + if err := json.NewDecoder(res.Body).Decode(&dStreams); err != nil { return nil, utils.FrameworkDiagFromError(err) } - ds := dStreams["data_streams"] + ds := dStreams.DataStreams return &ds, nil } From 128b011119af014807346b9f10d4c7e726bb0af9 Mon Sep 17 00:00:00 2001 From: Boris Ilyushonak Date: Wed, 18 Dec 2024 16:54:33 +0100 Subject: [PATCH 3/4] restrict kibana versions for TestAccResourceIntegrationPolicySecretsFromSDK --- internal/fleet/integration_policy/resource_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/fleet/integration_policy/resource_test.go b/internal/fleet/integration_policy/resource_test.go index de15e0174..f2c4aeaa2 100644 --- a/internal/fleet/integration_policy/resource_test.go +++ b/internal/fleet/integration_policy/resource_test.go @@ -75,6 +75,9 @@ func TestAccResourceIntegrationPolicy(t *testing.T) { func TestAccResourceIntegrationPolicySecretsFromSDK(t *testing.T) { policyName := sdkacctest.RandStringFromCharSet(22, sdkacctest.CharSetAlphaNum) + sdkConstrains, err := version.NewConstraint(">=8.10.0,<8.16.0") + require.NoError(t, err) + resource.Test(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(t) }, CheckDestroy: checkResourceIntegrationPolicyDestroy, @@ -86,7 +89,7 @@ func TestAccResourceIntegrationPolicySecretsFromSDK(t *testing.T) { VersionConstraint: "0.11.7", }, }, - SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), + SkipFunc: versionutils.CheckIfVersionMeetsConstraints(sdkConstrains), Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"), ExpectNonEmptyPlan: true, // secret churn Check: resource.ComposeTestCheckFunc( @@ -103,7 +106,7 @@ func TestAccResourceIntegrationPolicySecretsFromSDK(t *testing.T) { }, { ProtoV6ProviderFactories: acctest.Providers, - SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), + SkipFunc: versionutils.CheckIfVersionMeetsConstraints(sdkConstrains), Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName), From 8a95e094d9be01224d2a303c91a32d34fae066ac Mon Sep 17 00:00:00 2001 From: Boris Ilyushonak Date: Wed, 18 Dec 2024 17:23:03 +0100 Subject: [PATCH 4/4] fix `TestAccDataSourceKibanaSecurityRole` test --- internal/kibana/role.go | 7 ++----- libs/go-kibana-rest/kbapi/api.kibana_role_management.go | 7 ++++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/kibana/role.go b/internal/kibana/role.go index 98f028098..c19aed808 100644 --- a/internal/kibana/role.go +++ b/internal/kibana/role.go @@ -265,14 +265,11 @@ func resourceRoleUpsert(ctx context.Context, d *schema.ResourceData, meta interf if err != nil { return diag.FromErr(err) } - queryParams := "" - if d.IsNewResource() { - queryParams = "?createOnly=true" - } kibanaRole := kbapi.KibanaRole{ - Name: fmt.Sprintf("%s%s", d.Get("name").(string), queryParams), + Name: d.Get("name").(string), Kibana: []kbapi.KibanaRoleKibana{}, Elasticsearch: &kbapi.KibanaRoleElasticsearch{}, + CreateOnly: d.IsNewResource(), } if v, ok := d.GetOk("kibana"); ok { diff --git a/libs/go-kibana-rest/kbapi/api.kibana_role_management.go b/libs/go-kibana-rest/kbapi/api.kibana_role_management.go index 2ab419515..2451cb637 100644 --- a/libs/go-kibana-rest/kbapi/api.kibana_role_management.go +++ b/libs/go-kibana-rest/kbapi/api.kibana_role_management.go @@ -19,6 +19,7 @@ type KibanaRole struct { TransientMedata *KibanaRoleTransientMetadata `json:"transient_metadata,omitempty"` Elasticsearch *KibanaRoleElasticsearch `json:"elasticsearch,omitempty"` Kibana []KibanaRoleKibana `json:"kibana,omitempty"` + CreateOnly bool `json:"-"` } // KibanaRoleTransientMetadata is the API TransientMedata object @@ -153,7 +154,11 @@ func newKibanaRoleManagementCreateOrUpdateFunc(c *resty.Client) KibanaRoleManage if err != nil { return nil, err } - resp, err := c.R().SetBody(jsonData).Put(path) + r := c.R() + if kibanaRole.CreateOnly { + r = r.SetQueryParam("createOnly", "true") + } + resp, err := r.SetBody(jsonData).Put(path) if err != nil { return nil, err }