From d344f11186def998970c512d6e9a93192678f369 Mon Sep 17 00:00:00 2001 From: kishie Date: Thu, 25 Jan 2024 12:24:14 -0500 Subject: [PATCH 1/2] adds arm64 building to ostree plugin adds more crud operations around ostree remotes adds ability to specify an ephemeral remote when syncing renames files in orasostree and beskerctl to make more sense relative to their contents --- build/mage/build.go | 1 - cmd/beskarctl/ostree/{repo.go => push.go} | 0 cmd/beskarctl/ostree/{file.go => root.go} | 0 internal/plugins/ostree/api.go | 16 +++ .../plugins/ostree/pkg/libostree/pull_test.go | 9 ++ .../ostree/pkg/ostreerepository/api.go | 77 +++++++++++++- pkg/orasostree/{ostree.go => file.go} | 0 pkg/orasostree/{push.go => repo.go} | 0 pkg/plugins/ostree/api/v1/api.go | 15 +++ pkg/plugins/ostree/api/v1/endpoint.go | 76 +++++++++++++ pkg/plugins/ostree/api/v1/http.go | 60 +++++++++++ pkg/plugins/ostree/api/v1/http_client.go | 100 ++++++++++++++++++ pkg/plugins/ostree/api/v1/oas2.go | 37 +++++++ 13 files changed, 389 insertions(+), 2 deletions(-) rename cmd/beskarctl/ostree/{repo.go => push.go} (100%) rename cmd/beskarctl/ostree/{file.go => root.go} (100%) rename pkg/orasostree/{ostree.go => file.go} (100%) rename pkg/orasostree/{push.go => repo.go} (100%) diff --git a/build/mage/build.go b/build/mage/build.go index b32bf1d..c2e4a2d 100644 --- a/build/mage/build.go +++ b/build/mage/build.go @@ -166,7 +166,6 @@ var binaries = map[string]binaryConfig{ "CGO_ENABLED": "1", }, excludedPlatforms: map[dagger.Platform]struct{}{ - "linux/arm64": {}, "linux/s390x": {}, "linux/ppc64le": {}, "linux/arm/v6": {}, diff --git a/cmd/beskarctl/ostree/repo.go b/cmd/beskarctl/ostree/push.go similarity index 100% rename from cmd/beskarctl/ostree/repo.go rename to cmd/beskarctl/ostree/push.go diff --git a/cmd/beskarctl/ostree/file.go b/cmd/beskarctl/ostree/root.go similarity index 100% rename from cmd/beskarctl/ostree/file.go rename to cmd/beskarctl/ostree/root.go diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go index 9bbaa2f..32b2c9e 100644 --- a/internal/plugins/ostree/api.go +++ b/internal/plugins/ostree/api.go @@ -43,6 +43,22 @@ func (p *Plugin) AddRemote(ctx context.Context, repository string, properties *a return p.repositoryManager.Get(ctx, repository).AddRemote(ctx, properties) } +func (p *Plugin) UpdateRemote(ctx context.Context, repository string, remoteName string, properties *apiv1.OSTreeRemoteProperties) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).UpdateRemote(ctx, remoteName, properties) +} + +func (p *Plugin) DeleteRemote(ctx context.Context, repository string, remoteName string) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).DeleteRemote(ctx, remoteName) +} + func (p *Plugin) SyncRepository(ctx context.Context, repository string, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { if err := checkRepository(repository); err != nil { return err diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go index f62965e..de8045c 100644 --- a/internal/plugins/ostree/pkg/libostree/pull_test.go +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -111,6 +111,15 @@ func TestRepo_Pull(t *testing.T) { assert.Error(t, err) }) + t.Run("should create then delete remote", func(t *testing.T) { + remoteToDelete := "delete-me" + err := repo.AddRemote(remoteToDelete, remoteURL, NoGPGVerify()) + assert.NoError(t, err) + + err = repo.DeleteRemote(remoteToDelete) + assert.NoError(t, err) + }) + t.Run("should list remotes", func(t *testing.T) { remotes := repo.ListRemotes() assert.Equal(t, remotes, []string{remoteName}) diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go index d25522d..76f182d 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/api.go +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -161,6 +161,60 @@ func (h *Handler) AddRemote(ctx context.Context, remote *apiv1.OSTreeRemotePrope }, SkipPull()) } +func (h *Handler) UpdateRemote(ctx context.Context, remoteName string, remote *apiv1.OSTreeRemoteProperties) (err error) { + // Transition to provisioning state + if err := h.setState(StateProvisioning); err != nil { + return err + } + defer h.clearState() + + if !h.checkRepoExists(ctx) { + return ctl.Errf("repository does not exist") + } + + return h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { + // Delete user provided remote + if err := repo.DeleteRemote(remoteName); err != nil { + // No need to make error pretty, it is already pretty + return false, err + } + + // Add user provided remote + var opts []libostree.Option + if remote.NoGPGVerify { + opts = append(opts, libostree.NoGPGVerify()) + } + if err := repo.AddRemote(remote.Name, remote.RemoteURL, opts...); err != nil { + // No need to make error pretty, it is already pretty + return false, err + } + + return true, nil + }, SkipPull()) +} + +func (h *Handler) DeleteRemote(ctx context.Context, remoteName string) (err error) { + // Transition to provisioning state + if err := h.setState(StateProvisioning); err != nil { + return err + } + defer h.clearState() + + if !h.checkRepoExists(ctx) { + return ctl.Errf("repository does not exist") + } + + return h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { + // Delete user provided remote + if err := repo.DeleteRemote(remoteName); err != nil { + // No need to make error pretty, it is already pretty + return false, err + } + + return true, nil + }, SkipPull()) +} + func (h *Handler) SyncRepository(_ context.Context, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { // Transition to syncing state if err := h.setState(StateSyncing); err != nil { @@ -197,8 +251,29 @@ func (h *Handler) SyncRepository(_ context.Context, properties *apiv1.OSTreeRepo opts = append(opts, libostree.Refs(properties.Refs...)) } + remoteName := properties.Remote + if properties.EphemeralRemote != nil { + remoteName = properties.EphemeralRemote.Name + + var opts []libostree.Option + if properties.EphemeralRemote.NoGPGVerify { + opts = append(opts, libostree.NoGPGVerify()) + } + + if err := repo.AddRemote(properties.EphemeralRemote.Name, properties.EphemeralRemote.RemoteURL, opts...); err != nil { + // No need to make error pretty, it is already pretty + return false, err + } + + defer func() { + if err == nil { + err = repo.DeleteRemote(properties.EphemeralRemote.Name) + } + }() + } + // pull remote content into local repo - if err := repo.Pull(ctx, properties.Remote, opts...); err != nil { + if err := repo.Pull(ctx, remoteName, opts...); err != nil { return false, ctl.Errf("pulling ostree repository: %s", err) } diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/file.go similarity index 100% rename from pkg/orasostree/ostree.go rename to pkg/orasostree/file.go diff --git a/pkg/orasostree/push.go b/pkg/orasostree/repo.go similarity index 100% rename from pkg/orasostree/push.go rename to pkg/orasostree/repo.go diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go index 46ab6c8..f8ad096 100644 --- a/pkg/plugins/ostree/api/v1/api.go +++ b/pkg/plugins/ostree/api/v1/api.go @@ -42,8 +42,13 @@ type OSTreeRemoteProperties struct { type OSTreeRepositorySyncRequest struct { // Remote - The name of the remote to sync. + // Remote and EphemeralRemote are mutually exclusive. EphemeralRemote takes precedence. Remote string `json:"remote"` + // EphemeralRemote - A remote to add to the repository in Beskar for the duration of the sync. + // Remote and EphemeralRemote are mutually exclusive. EphemeralRemote takes precedence. + EphemeralRemote *OSTreeRemoteProperties `json:"ephemeral_remote"` + // Refs - The branches/refs to mirror. Leave empty to mirror all branches/refs. Refs []string `json:"refs"` @@ -88,6 +93,16 @@ type OSTree interface { //kun:success statusCode=200 AddRemote(ctx context.Context, repository string, properties *OSTreeRemoteProperties) (err error) + // Updates a remote in the OSTree repository. If it doesn't exist it will be created. + //kun:op PUT /repository/remote + //kun:success statusCode=200 + UpdateRemote(ctx context.Context, repository string, remoteName string, properties *OSTreeRemoteProperties) (err error) + + // Delete an existing remote to the OSTree repository. + //kun:op DELETE /repository/remote + //kun:success statusCode=200 + DeleteRemote(ctx context.Context, repository string, remoteName string) (err error) + // Sync an ostree repository with one of the configured remotes. //kun:op POST /repository/sync //kun:success statusCode=202 diff --git a/pkg/plugins/ostree/api/v1/endpoint.go b/pkg/plugins/ostree/api/v1/endpoint.go index cabf265..c8dfb89 100644 --- a/pkg/plugins/ostree/api/v1/endpoint.go +++ b/pkg/plugins/ostree/api/v1/endpoint.go @@ -85,6 +85,43 @@ func MakeEndpointOfCreateRepository(s OSTree) endpoint.Endpoint { } } +type DeleteRemoteRequest struct { + Repository string `json:"repository"` + RemoteName string `json:"remote_name"` +} + +// ValidateDeleteRemoteRequest creates a validator for DeleteRemoteRequest. +func ValidateDeleteRemoteRequest(newSchema func(*DeleteRemoteRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*DeleteRemoteRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type DeleteRemoteResponse struct { + Err error `json:"-"` +} + +func (r *DeleteRemoteResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *DeleteRemoteResponse) Failed() error { return r.Err } + +// MakeEndpointOfDeleteRemote creates the endpoint for s.DeleteRemote. +func MakeEndpointOfDeleteRemote(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*DeleteRemoteRequest) + err := s.DeleteRemote( + ctx, + req.Repository, + req.RemoteName, + ) + return &DeleteRemoteResponse{ + Err: err, + }, nil + } +} + type DeleteRepositoryRequest struct { Repository string `json:"repository"` } @@ -193,3 +230,42 @@ func MakeEndpointOfSyncRepository(s OSTree) endpoint.Endpoint { }, nil } } + +type UpdateRemoteRequest struct { + Repository string `json:"repository"` + RemoteName string `json:"remote_name"` + Properties *OSTreeRemoteProperties `json:"properties"` +} + +// ValidateUpdateRemoteRequest creates a validator for UpdateRemoteRequest. +func ValidateUpdateRemoteRequest(newSchema func(*UpdateRemoteRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*UpdateRemoteRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type UpdateRemoteResponse struct { + Err error `json:"-"` +} + +func (r *UpdateRemoteResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *UpdateRemoteResponse) Failed() error { return r.Err } + +// MakeEndpointOfUpdateRemote creates the endpoint for s.UpdateRemote. +func MakeEndpointOfUpdateRemote(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*UpdateRemoteRequest) + err := s.UpdateRemote( + ctx, + req.Repository, + req.RemoteName, + req.Properties, + ) + return &UpdateRemoteResponse{ + Err: err, + }, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http.go b/pkg/plugins/ostree/api/v1/http.go index fb8eace..afbc422 100644 --- a/pkg/plugins/ostree/api/v1/http.go +++ b/pkg/plugins/ostree/api/v1/http.go @@ -52,6 +52,20 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio ), ) + codec = codecs.EncodeDecoder("DeleteRemote") + validator = options.RequestValidator("DeleteRemote") + r.Method( + "DELETE", "/repository/remote", + kithttp.NewServer( + MakeEndpointOfDeleteRemote(svc), + decodeDeleteRemoteRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + codec = codecs.EncodeDecoder("DeleteRepository") validator = options.RequestValidator("DeleteRepository") r.Method( @@ -94,6 +108,20 @@ func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Optio ), ) + codec = codecs.EncodeDecoder("UpdateRemote") + validator = options.RequestValidator("UpdateRemote") + r.Method( + "PUT", "/repository/remote", + kithttp.NewServer( + MakeEndpointOfUpdateRemote(svc), + decodeUpdateRemoteRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + return r } @@ -129,6 +157,22 @@ func decodeCreateRepositoryRequest(codec httpcodec.Codec, validator httpoption.V } } +func decodeDeleteRemoteRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req DeleteRemoteRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + func decodeDeleteRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { return func(_ context.Context, r *http.Request) (interface{}, error) { var _req DeleteRepositoryRequest @@ -176,3 +220,19 @@ func decodeSyncRepositoryRequest(codec httpcodec.Codec, validator httpoption.Val return &_req, nil } } + +func decodeUpdateRemoteRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req UpdateRemoteRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http_client.go b/pkg/plugins/ostree/api/v1/http_client.go index 989ec36..a50aa1e 100644 --- a/pkg/plugins/ostree/api/v1/http_client.go +++ b/pkg/plugins/ostree/api/v1/http_client.go @@ -132,6 +132,55 @@ func (c *HTTPClient) CreateRepository(ctx context.Context, repository string, pr return nil } +func (c *HTTPClient) DeleteRemote(ctx context.Context, repository string, remoteName string) (err error) { + codec := c.codecs.EncodeDecoder("DeleteRemote") + + path := "/repository/remote" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + RemoteName string `json:"remote_name"` + }{ + Repository: repository, + RemoteName: remoteName, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "DELETE", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} + func (c *HTTPClient) DeleteRepository(ctx context.Context, repository string) (err error) { codec := c.codecs.EncodeDecoder("DeleteRepository") @@ -279,3 +328,54 @@ func (c *HTTPClient) SyncRepository(ctx context.Context, repository string, prop return nil } + +func (c *HTTPClient) UpdateRemote(ctx context.Context, repository string, remoteName string, properties *OSTreeRemoteProperties) (err error) { + codec := c.codecs.EncodeDecoder("UpdateRemote") + + path := "/repository/remote" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + RemoteName string `json:"remote_name"` + Properties *OSTreeRemoteProperties `json:"properties"` + }{ + Repository: repository, + RemoteName: remoteName, + Properties: properties, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "PUT", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go index ba7f80b..800dfa2 100644 --- a/pkg/plugins/ostree/api/v1/oas2.go +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -41,6 +41,28 @@ paths: schema: $ref: "#/definitions/AddRemoteRequestBody" %s + delete: + description: "Delete an existing remote to the OSTree repository." + operationId: "DeleteRemote" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/DeleteRemoteRequestBody" + %s + put: + description: "Updates a remote in the OSTree repository. If it doesn't exist it will be created." + operationId: "UpdateRemote" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/UpdateRemoteRequestBody" + %s /repository: post: description: "Create an OSTree repository." @@ -93,6 +115,8 @@ paths: func getResponses(schema oas2.Schema) []oas2.OASResponses { return []oas2.OASResponses{ oas2.GetOASResponses(schema, "AddRemote", 200, &AddRemoteResponse{}), + oas2.GetOASResponses(schema, "DeleteRemote", 200, &DeleteRemoteResponse{}), + oas2.GetOASResponses(schema, "UpdateRemote", 200, &UpdateRemoteResponse{}), oas2.GetOASResponses(schema, "CreateRepository", 200, &CreateRepositoryResponse{}), oas2.GetOASResponses(schema, "DeleteRepository", 202, &DeleteRepositoryResponse{}), oas2.GetOASResponses(schema, "GetRepositorySyncStatus", 200, &GetRepositorySyncStatusResponse{}), @@ -115,6 +139,12 @@ func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { }{})) oas2.AddResponseDefinitions(defs, schema, "CreateRepository", 200, (&CreateRepositoryResponse{}).Body()) + oas2.AddDefinition(defs, "DeleteRemoteRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + RemoteName string `json:"remote_name"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "DeleteRemote", 200, (&DeleteRemoteResponse{}).Body()) + oas2.AddDefinition(defs, "DeleteRepositoryRequestBody", reflect.ValueOf(&struct { Repository string `json:"repository"` }{})) @@ -131,6 +161,13 @@ func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { }{})) oas2.AddResponseDefinitions(defs, schema, "SyncRepository", 202, (&SyncRepositoryResponse{}).Body()) + oas2.AddDefinition(defs, "UpdateRemoteRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + RemoteName string `json:"remote_name"` + Properties *OSTreeRemoteProperties `json:"properties"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "UpdateRemote", 200, (&UpdateRemoteResponse{}).Body()) + return defs } From be51599a940a5c0e7da22dc7214900dff721b322 Mon Sep 17 00:00:00 2001 From: kishie Date: Thu, 25 Jan 2024 12:37:29 -0500 Subject: [PATCH 2/2] implements better error handling for ephemeral remotes --- internal/plugins/ostree/pkg/ostreerepository/api.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go index 76f182d..2fa1be3 100644 --- a/internal/plugins/ostree/pkg/ostreerepository/api.go +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -241,7 +241,7 @@ func (h *Handler) SyncRepository(_ context.Context, properties *apiv1.OSTreeRepo ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - err = h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { + err = h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (commit bool, transactionFnErr error) { // Pull the latest changes from the remote. opts := []libostree.Option{ libostree.Depth(properties.Depth), @@ -266,8 +266,12 @@ func (h *Handler) SyncRepository(_ context.Context, properties *apiv1.OSTreeRepo } defer func() { - if err == nil { - err = repo.DeleteRemote(properties.EphemeralRemote.Name) + if transactionFnErr == nil { + if err := repo.DeleteRemote(properties.EphemeralRemote.Name); err != nil { + h.logger.Error("deleting ephemeral remote", "error", err.Error()) + commit = false + transactionFnErr = err + } } }() }