diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9d1f7ab..5e3b9de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,4 +69,18 @@ jobs: - name: Release beskar-static image run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar-static:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" - name: Release beskar-static helm chart - run: ./scripts/mage ci:chart ghcr.io/ctrliq/helm-charts/beskar-static:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file + run: ./scripts/mage ci:chart ghcr.io/ctrliq/helm-charts/beskar-static:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" + + release-beskar-ostree: + name: release beskar-ostree + needs: [lint, tests] + runs-on: ubuntu-22.04 + steps: + - uses: actions/setup-go@v3 + with: + go-version: '1.21' + - uses: actions/checkout@v3 + - name: Release beskar-ostree image + run: ./scripts/mage ci:image ghcr.io/ctrliq/beskar-ostree:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" + - name: Release beskar-ostree helm chart + run: ./scripts/mage ci:chart ghcr.io/ctrliq/helm-charts/beskar-ostree:${{ github.ref_name }} "${{ github.actor }}" "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1f65aa4..9c13cd1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ build/output .envrc.local vendor go.work.sum + + +*/.idea +/.idea/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/beskar.iml b/.idea/beskar.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/beskar.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..50cc4b9 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index df2c668..29b8aec 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,10 @@ It's designed to support various artifacts and expose them through dedicated plu ### Features -* Modular/Extensible via plugins +* Modular/Extensible via [plugins](docs/plugins.md) * Support for YUM repositories (beskar-yum) * Support for static file repositories (beskar-static) +* Support for OSTree repositories ([beskar-ostree](internal/plugins/ostree/README.md)) ### Docker images @@ -41,6 +42,12 @@ For beskar-static helm chart: helm pull oci://ghcr.io/ctrliq/helm-charts/beskar-static --version 0.0.1 --untar ``` +For beskar-static helm chart: + +``` +helm pull oci://ghcr.io/ctrliq/helm-charts/beskar-ostree --version 0.0.1 --untar +``` + ### Compilation Binaries are not provided as part of releases, you can compile it yourself by running: diff --git a/build/mage/build.go b/build/mage/build.go index 164989d..b32bf1d 100644 --- a/build/mage/build.go +++ b/build/mage/build.go @@ -55,6 +55,8 @@ type binaryConfig struct { buildTags []string baseImage string integrationTest *integrationTest + buildEnv map[string]string + buildExecStmts [][]string } const ( @@ -62,6 +64,7 @@ const ( BeskarctlBinary = "beskarctl" BeskarYUMBinary = "beskar-yum" BeskarStaticBinary = "beskar-static" + BeskarOSTreeBinary = "beskar-ostree" ) var binaries = map[string]binaryConfig{ @@ -101,7 +104,7 @@ var binaries = map[string]binaryConfig{ }, useProto: true, // NOTE: restore in case alpine createrepo_c package is broken again - //baseImage: "debian:bullseye-slim", + // baseImage: "debian:bullseye-slim", integrationTest: &integrationTest{ isPlugin: true, envs: map[string]string{ @@ -130,6 +133,46 @@ var binaries = map[string]binaryConfig{ }, }, }, + BeskarOSTreeBinary: { + configFiles: map[string]string{ + "internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml": "/etc/beskar/beskar-ostree.yaml", + }, + genAPI: &genAPI{ + path: "pkg/plugins/ostree/api/v1", + filename: "api.go", + interfaceName: "OSTree", + }, + useProto: true, + execStmts: [][]string{ + { + "apk", "add", "ostree", "ostree-dev", + }, + }, + buildExecStmts: [][]string{ + { + // pkg-config is needed to compute CFLAGS + "apk", "add", "pkgconfig", + }, + { + // Install gcc. Could have installed gc directly but this seems to be the recommended way for alpine. + "apk", "add", "build-base", + }, + { + // Install ostree development libraries + "apk", "add", "ostree", "ostree-dev", + }, + }, + buildEnv: map[string]string{ + "CGO_ENABLED": "1", + }, + excludedPlatforms: map[dagger.Platform]struct{}{ + "linux/arm64": {}, + "linux/s390x": {}, + "linux/ppc64le": {}, + "linux/arm/v6": {}, + "linux/arm/v7": {}, + }, + }, } type Build mg.Namespace @@ -170,6 +213,7 @@ func (b Build) Plugins(ctx context.Context) { ctx, mg.F(b.Plugin, BeskarYUMBinary), mg.F(b.Plugin, BeskarStaticBinary), + mg.F(b.Plugin, BeskarOSTreeBinary), ) } @@ -274,6 +318,14 @@ func (b Build) build(ctx context.Context, name string) error { golang = golang.WithEnvVariable(key, value) } + for key, value := range binaryConfig.buildEnv { + golang = golang.WithEnvVariable(key, value) + } + + for _, execStmt := range binaryConfig.buildExecStmts { + golang = golang.WithExec(execStmt) + } + path := filepath.Join("/output", binary) inputCmd := filepath.Join("cmd", name) diff --git a/build/mage/lint.go b/build/mage/lint.go index b6cc1ea..12a59d9 100644 --- a/build/mage/lint.go +++ b/build/mage/lint.go @@ -36,6 +36,18 @@ func (Lint) Go(ctx context.Context) error { WithWorkdir("/src"). With(goCache(client)) + // Set up the environment for the linter per the settings of each binary. + // This could lead to conflicts if the binaries have different settings. + for _, config := range binaries { + for key, value := range config.buildEnv { + golangciLint = golangciLint.WithEnvVariable(key, value) + } + + for _, execStmt := range config.buildExecStmts { + golangciLint = golangciLint.WithExec(execStmt) + } + } + golangciLint = golangciLint.WithExec([]string{ "golangci-lint", "-v", "run", "--modules-download-mode", "readonly", "--timeout", "5m", }) diff --git a/build/mage/test.go b/build/mage/test.go index 53f87c0..014b778 100644 --- a/build/mage/test.go +++ b/build/mage/test.go @@ -2,11 +2,10 @@ package mage import ( "context" - "fmt" - "strings" - "dagger.io/dagger" + "fmt" "github.com/magefile/mage/mg" + "strings" ) type Test mg.Namespace @@ -29,8 +28,20 @@ func (Test) Unit(ctx context.Context) error { WithWorkdir("/src"). With(goCache(client)) + for _, config := range binaries { + for key, value := range config.buildEnv { + unitTest = unitTest.WithEnvVariable(key, value) + } + + for _, execStmt := range config.buildExecStmts { + unitTest = unitTest.WithExec(execStmt) + } + } + unitTest = unitTest.WithExec([]string{ "go", "test", "-v", "-count=1", "./...", + }, dagger.ContainerWithExecOpts{ + InsecureRootCapabilities: true, }) return printOutput(ctx, unitTest) diff --git a/charts/beskar-ostree/.helmignore b/charts/beskar-ostree/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/beskar-ostree/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/beskar-ostree/Chart.yaml b/charts/beskar-ostree/Chart.yaml new file mode 100644 index 0000000..44625d7 --- /dev/null +++ b/charts/beskar-ostree/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +description: A Helm chart for Beskar OSTree Repository Plugin +name: beskar-ostree +version: 0.0.1 +appVersion: 0.0.1 +home: https://github.com/ctrliq/beskar +maintainers: +- email: dev@ciq.com + name: CtrlIQ Inc. + url: https://github.com/ctrliq/beskar +sources: +- https://github.com/ctrliq/beskar diff --git a/charts/beskar-ostree/LICENSE b/charts/beskar-ostree/LICENSE new file mode 100644 index 0000000..393b7a3 --- /dev/null +++ b/charts/beskar-ostree/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright The Helm Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/charts/beskar-ostree/templates/NOTES.txt b/charts/beskar-ostree/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/charts/beskar-ostree/templates/_helpers.tpl b/charts/beskar-ostree/templates/_helpers.tpl new file mode 100644 index 0000000..e827ba5 --- /dev/null +++ b/charts/beskar-ostree/templates/_helpers.tpl @@ -0,0 +1,150 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "beskar-ostree.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "beskar-ostree.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "beskar-ostree.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "beskar-ostree.labels" -}} +helm.sh/chart: {{ include "beskar-ostree.chart" . }} +{{ include "beskar-ostree.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "beskar-ostree.selectorLabels" -}} +app.kubernetes.io/name: {{ include "beskar-ostree.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "beskar-ostree.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "beskar-ostree.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "beskar-ostree.envs" -}} +- name: BESKAROSTREE_GOSSIP_KEY + valueFrom: + secretKeyRef: + name: beskar-gossip-secret + key: gossipKey +{{- if eq .Values.configData.storage.driver "filesystem" }} +- name: BESKAROSTREE_STORAGE_FILESYSTEM_DIRECTORY + value: {{ .Values.configData.storage.filesystem.directory }} +{{- else if eq .Values.configData.storage.driver "azure" }} +- name: BESKAROSTREE_STORAGE_AZURE_ACCOUNTNAME + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: azureAccountName +- name: BESKAROSTREE_STORAGE_AZURE_ACCOUNTKEY + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: azureAccountKey +{{- else if eq .Values.configData.storage.driver "s3" }} + {{- if and .Values.secrets.s3.secretKey .Values.secrets.s3.accessKey }} +- name: BESKAROSTREE_STORAGE_S3_ACCESSKEYID + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: s3AccessKey +- name: BESKAROSTREE_STORAGE_S3_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: {{ template "beskar-ostree.fullname" . }}-secret + key: s3SecretKey + {{- end }} +{{- else if eq .Values.configData.storage.driver "gcs" }} +- name: BESKAROSTREE_STORAGE_GCS_KEYFILE + value: /etc/gcs-keyfile +{{- end -}} + +{{- with .Values.extraEnvVars }} +{{ toYaml . }} +{{- end -}} + +{{- end -}} + +{{- define "beskar-ostree.volumeMounts" -}} +- name: config + mountPath: "/etc/beskar" + +{{- if eq .Values.configData.storage.driver "filesystem" }} +- name: data + mountPath: {{ .Values.configData.storage.filesystem.directory }} +{{- else if eq .Values.configData.storage.driver "gcs" }} +- name: gcs + mountPath: "/etc/gcs-keyfile" + subPath: gcsKeyfile + readOnly: true +{{- end }} + +{{- with .Values.extraVolumeMounts }} +{{ toYaml . }} +{{- end }} + +{{- end -}} + +{{- define "beskar-ostree.volumes" -}} +- name: config + configMap: + name: {{ template "beskar-ostree.fullname" . }}-config + +{{- if eq .Values.configData.storage.driver "filesystem" }} +- name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "beskar-ostree.fullname" . }}{{- end }} + {{- else }} + emptyDir: {} + {{- end -}} +{{- else if eq .Values.configData.storage.driver "gcs" }} +- name: gcs + secret: + secretName: {{ template "beskar-ostree.fullname" . }}-secret +{{- end }} + +{{- with .Values.extraVolumes }} +{{ toYaml . }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/configmap.yaml b/charts/beskar-ostree/templates/configmap.yaml new file mode 100644 index 0000000..3426be5 --- /dev/null +++ b/charts/beskar-ostree/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "beskar-ostree.fullname" . }}-config + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +data: + beskar-ostree.yaml: |- +{{ toYaml .Values.configData | indent 4 }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/hpa.yaml b/charts/beskar-ostree/templates/hpa.yaml new file mode 100644 index 0000000..022bfd1 --- /dev/null +++ b/charts/beskar-ostree/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "beskar-ostree.fullname" . }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "beskar-ostree.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/beskar-ostree/templates/pvc.yaml b/charts/beskar-ostree/templates/pvc.yaml new file mode 100644 index 0000000..77c0621 --- /dev/null +++ b/charts/beskar-ostree/templates/pvc.yaml @@ -0,0 +1,26 @@ +{{- if .Values.persistence.enabled }} +{{- if not .Values.persistence.existingClaim -}} +{{- if eq .Values.configData.storage.driver "filesystem" }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "beskar-ostree.fullname" . }} + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- if .Values.persistence.storageClass }} +{{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/beskar-ostree/templates/role.yaml b/charts/beskar-ostree/templates/role.yaml new file mode 100644 index 0000000..5e2a930 --- /dev/null +++ b/charts/beskar-ostree/templates/role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "beskar-ostree.fullname" . }} +rules: + - apiGroups: + - '' + resources: + - endpoints + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "beskar-ostree.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name | default (include "beskar-ostree.fullname" .) }} + apiGroup: "" + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ template "beskar-ostree.fullname" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/charts/beskar-ostree/templates/secret.yaml b/charts/beskar-ostree/templates/secret.yaml new file mode 100644 index 0000000..fc5e2c8 --- /dev/null +++ b/charts/beskar-ostree/templates/secret.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "beskar-ostree.fullname" . }}-secret + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +type: Opaque +data: + {{- if eq .Values.configData.storage.driver "azure" }} + {{- if and .Values.secrets.azure.accountName .Values.secrets.azure.accountKey .Values.secrets.azure.container }} + azureAccountName: {{ .Values.secrets.azure.accountName | b64enc | quote }} + azureAccountKey: {{ .Values.secrets.azure.accountKey | b64enc | quote }} + {{- end }} + {{- else if eq .Values.configData.storage.driver "s3" }} + {{- if and .Values.secrets.s3.secretKey .Values.secrets.s3.accessKey }} + s3AccessKey: {{ .Values.secrets.s3.accessKey | b64enc | quote }} + s3SecretKey: {{ .Values.secrets.s3.secretKey | b64enc | quote }} + {{- end }} + {{- else if eq .Values.configData.storage.driver "gcs" }} + gcsKeyfile: {{ .Values.secrets.gcs.keyfile | b64enc | quote }} + {{- end }} + registryUsername: {{ .Values.secrets.registry.username | b64enc | quote }} + registryPassword: {{ .Values.secrets.registry.password | b64enc | quote }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/service.yaml b/charts/beskar-ostree/templates/service.yaml new file mode 100644 index 0000000..378d4da --- /dev/null +++ b/charts/beskar-ostree/templates/service.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "beskar-ostree.fullname" . }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.service.type }} +{{- if .Values.service.sessionAffinity }} + sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- if .Values.service.sessionAffinityConfig }} + sessionAffinityConfig: + {{ toYaml .Values.service.sessionAffinityConfig | nindent 4 }} + {{- end -}} +{{- end }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + protocol: TCP + name: http + selector: + {{- include "beskar-ostree.selectorLabels" . | nindent 4 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "beskar-ostree.fullname" . }}-gossip + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} + go.ciq.dev/beskar-gossip: "true" +{{- if .Values.gossip.annotations }} + annotations: +{{ toYaml .Values.gossip.annotations | indent 4 }} +{{- end }} +spec: + type: ClusterIP +{{- if .Values.gossip.sessionAffinity }} + sessionAffinity: {{ .Values.gossip.sessionAffinity }} + {{- if .Values.gossip.sessionAffinityConfig }} + sessionAffinityConfig: + {{ toYaml .Values.gossip.sessionAffinityConfig | nindent 4 }} + {{- end -}} +{{- end }} + ports: + - port: {{ .Values.gossip.port }} + protocol: TCP + name: gossip-tcp + targetPort: {{ .Values.gossip.port }} + - port: {{ .Values.gossip.port }} + protocol: UDP + name: gossip-udp + targetPort: {{ .Values.gossip.port }} + selector: + {{- include "beskar-ostree.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/serviceaccount.yaml b/charts/beskar-ostree/templates/serviceaccount.yaml new file mode 100644 index 0000000..e6ad1a9 --- /dev/null +++ b/charts/beskar-ostree/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: +{{- if .Values.serviceAccount.name }} + name: {{ .Values.serviceAccount.name }} +{{- else }} + name: {{ include "beskar-ostree.fullname" . }} +{{- end }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} \ No newline at end of file diff --git a/charts/beskar-ostree/templates/statefulset.yaml b/charts/beskar-ostree/templates/statefulset.yaml new file mode 100644 index 0000000..36947fb --- /dev/null +++ b/charts/beskar-ostree/templates/statefulset.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "beskar-ostree.fullname" . }} + labels: + {{- include "beskar-ostree.labels" . | nindent 4 }} +spec: + serviceName: {{ .Chart.Name }} + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "beskar-ostree.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "beskar-ostree.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{ toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.serviceAccount.name | default (include "beskar-ostree.fullname" .) }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /usr/bin/beskar-ostree + - -config-dir=/etc/beskar + ports: + - containerPort: {{ .Values.service.port }} + name: http + protocol: TCP + - containerPort: {{ .Values.gossip.port }} + name: gossip-tcp + protocol: TCP + - containerPort: {{ .Values.gossip.port }} + name: gossip-udp + protocol: UDP + livenessProbe: + tcpSocket: + port: http + readinessProbe: + tcpSocket: + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: {{ include "beskar-ostree.envs" . | nindent 12 }} + volumeMounts: {{ include "beskar-ostree.volumeMounts" . | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: {{ include "beskar-ostree.volumes" . | nindent 8 }} diff --git a/charts/beskar-ostree/values.yaml b/charts/beskar-ostree/values.yaml new file mode 100644 index 0000000..223c957 --- /dev/null +++ b/charts/beskar-ostree/values.yaml @@ -0,0 +1,130 @@ +# Default values for beskar-ostree. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/ctrliq/beskar-ostree + # Overrides the image tag whose default is the chart appVersion. + tag: 0.0.1 + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + # sessionAffinity: None + # sessionAffinityConfig: {} + type: ClusterIP + port: 5200 + annotations: {} + +gossip: + # sessionAffinity: None + # sessionAffinityConfig: {} + port: 5201 + annotations: {} + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +extraVolumeMounts: [] + +extraVolumes: [] + +extraEnvVars: [] + +persistence: + accessMode: 'ReadWriteOnce' + enabled: false + size: 10Gi + # storageClass: '-' + +secrets: + registry: + username: beskar + password: beskar + + s3: + accessKey: "" + secretKey: "" + + gcs: + keyfile: "" + + azure: + accountName: "" + # base64_encoded_account_key + accountKey: "" + +configData: + version: "1.0" + addr: :5200 + profiling: false + datadir: /tmp/beskar-ostree + + log: + level: debug + format: json + + gossip: + addr: :5201 + + storage: + driver: filesystem + prefix: "" + s3: + endpoint: 127.0.0.1:9100 + bucket: beskar-ostree + region: us-east-1 + filesystem: + directory: /tmp/beskar-ostree + gcs: + bucket: beskar-ostree + azure: + container: beskar-ostree \ No newline at end of file diff --git a/cmd/beskar-ostree/main.go b/cmd/beskar-ostree/main.go new file mode 100644 index 0000000..6fd4399 --- /dev/null +++ b/cmd/beskar-ostree/main.go @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "syscall" + + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/ostreerepository" + + "go.ciq.dev/beskar/internal/pkg/pluginsrv" + "go.ciq.dev/beskar/internal/plugins/ostree" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" + "go.ciq.dev/beskar/pkg/sighandler" + "go.ciq.dev/beskar/pkg/version" +) + +var configDir string + +func serve(beskarOSTreeCmd *flag.FlagSet) error { + if err := beskarOSTreeCmd.Parse(os.Args[1:]); err != nil { + return err + } + + errCh := make(chan error) + + ctx, wait := sighandler.New(errCh, syscall.SIGTERM, syscall.SIGINT) + + beskarOSTreeConfig, err := config.ParseBeskarOSTreeConfig(configDir) + if err != nil { + return err + } + + ln, err := net.Listen("tcp", beskarOSTreeConfig.Addr) + if err != nil { + return err + } + defer func() { + if err := ln.Close(); err != nil { + fmt.Println(err) + } + }() + + plugin, err := ostree.New(ctx, beskarOSTreeConfig) + if err != nil { + return err + } + + go func() { + errCh <- pluginsrv.Serve[*ostreerepository.Handler](ln, plugin) + }() + + return wait(false) +} + +func main() { + beskarOSTreeCmd := flag.NewFlagSet("beskar-ostree", flag.ExitOnError) + beskarOSTreeCmd.StringVar(&configDir, "config-dir", "", "configuration directory") + + subCommand := "" + if len(os.Args) > 1 { + subCommand = os.Args[1] + } + + switch subCommand { + case "version": + fmt.Println(version.Semver) + default: + if err := serve(beskarOSTreeCmd); err != nil { + log.Fatal(err) + } + } +} diff --git a/cmd/beskar-static/main.go b/cmd/beskar-static/main.go index 23b97ce..8939518 100644 --- a/cmd/beskar-static/main.go +++ b/cmd/beskar-static/main.go @@ -11,10 +11,11 @@ import ( "os" "syscall" + "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/static" "go.ciq.dev/beskar/internal/plugins/static/pkg/config" - "go.ciq.dev/beskar/internal/plugins/static/pkg/staticrepository" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" ) diff --git a/cmd/beskar-yum/main.go b/cmd/beskar-yum/main.go index 353089d..22ed28c 100644 --- a/cmd/beskar-yum/main.go +++ b/cmd/beskar-yum/main.go @@ -11,10 +11,11 @@ import ( "os" "syscall" + "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" "go.ciq.dev/beskar/internal/plugins/yum" "go.ciq.dev/beskar/internal/plugins/yum/pkg/config" - "go.ciq.dev/beskar/internal/plugins/yum/pkg/yumrepository" "go.ciq.dev/beskar/pkg/sighandler" "go.ciq.dev/beskar/pkg/version" ) diff --git a/cmd/beskarctl/README.md b/cmd/beskarctl/README.md new file mode 100644 index 0000000..22f3db0 --- /dev/null +++ b/cmd/beskarctl/README.md @@ -0,0 +1,32 @@ +# beskarctl +`beskarctl` is a command line tool for interacting with Beskar Artifact Registries. + +## Installation +``` +go install go.ciq.dev/beskar/cmd/beskarctl@latest +``` + +## Usage +`beskarctl` is very similar to `kubectl` in that it provide various subcommands for interacting with Beskar repositories. +The following subcommands are available: + ``` +beskarctl yum [flags] +beskarctl static [flags] +beskarctl ostree [flags] + ``` +For more information on a specific subcommand, run `beskarctl --help`. + +## Adding a new subcommand +Adding a new subcommand is fairly straightforward. Feel free to use the existing subcommands as a template, e.g., +`cmd/beskarctl/static/`. The following steps should be followed: + +1. Create a new file in `cmd/beskarctl//root.go`. +2. Add a new `cobra.Command` to the `rootCmd` variable in `cmd/beskarctl//root.go`. +3. Add an accessor function to `cmd/beskarctl//root.go` that returns the new `cobra.Command`. +4. Register the new subcommand in `cmd/beskarctl/ctl/root.go` by calling the accessor function. + +### Implementation Notes +- The `cobra.Command` you create should not be exported. Rather, your package should export an accessor function that +returns the `cobra.Command`. The accessor function is your chance to set up any flags or subcommands that your +`cobra.Command` needs. Please avoid the use of init functi +- helper functions are available for common values such as `--repo` and `--registry`. See `cmd/beskarctl/ctl/helpers.go` \ No newline at end of file diff --git a/cmd/beskarctl/ctl/error.go b/cmd/beskarctl/ctl/error.go new file mode 100644 index 0000000..706dc5f --- /dev/null +++ b/cmd/beskarctl/ctl/error.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ctl + +import "fmt" + +type Err string + +func (e Err) Error() string { + return string(e) +} + +func Errf(str string, a ...any) Err { + return Err(fmt.Sprintf(str, a...)) +} diff --git a/cmd/beskarctl/ctl/helpers.go b/cmd/beskarctl/ctl/helpers.go new file mode 100644 index 0000000..445a5e8 --- /dev/null +++ b/cmd/beskarctl/ctl/helpers.go @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ctl + +import ( + "os" + + "github.com/spf13/cobra" +) + +const ( + ErrMissingFlagRepo = Err("missing repo flag") + ErrMissingFlagRegistry = Err("missing registry flag") +) + +const ( + FlagNameRepo = "repo" + FlagNameRegistry = "registry" +) + +// RegisterFlags registers the flags that are common to all commands. +func RegisterFlags(cmd *cobra.Command) { + // Flags that are common to all commands. + cmd.PersistentFlags().String(FlagNameRepo, "", "The repository to operate on.") + cmd.PersistentFlags().String(FlagNameRegistry, "", "The registry to operate on.") +} + +// Repo returns the repository name from the command line. +// If the repository is not specified, the command will exit with an error. +func Repo() string { + repo, err := rootCmd.Flags().GetString(FlagNameRepo) + if err != nil || repo == "" { + rootCmd.PrintErrln(ErrMissingFlagRepo) + os.Exit(1) + } + + return repo +} + +// Registry returns the registry name from the command line. +// If the registry is not specified, the command will exit with an error. +func Registry() string { + registry, err := rootCmd.Flags().GetString(FlagNameRegistry) + if err != nil || registry == "" { + rootCmd.PrintErrln(ErrMissingFlagRegistry) + os.Exit(1) + } + + return registry +} diff --git a/cmd/beskarctl/ctl/root.go b/cmd/beskarctl/ctl/root.go new file mode 100644 index 0000000..fcb6efe --- /dev/null +++ b/cmd/beskarctl/ctl/root.go @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ctl + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "beskarctl", + Short: "Operations related to beskar.", +} + +func Execute(cmds ...*cobra.Command) { + RegisterFlags(rootCmd) + + rootCmd.AddCommand( + cmds..., + ) + + err := rootCmd.Execute() + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/cmd/beskarctl/main.go b/cmd/beskarctl/main.go index 9ee9f02..c96a5da 100644 --- a/cmd/beskarctl/main.go +++ b/cmd/beskarctl/main.go @@ -4,95 +4,16 @@ package main import ( - "flag" - "fmt" - "os" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - "go.ciq.dev/beskar/pkg/oras" - "go.ciq.dev/beskar/pkg/orasrpm" - "go.ciq.dev/beskar/pkg/version" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/cmd/beskarctl/ostree" + "go.ciq.dev/beskar/cmd/beskarctl/static" + "go.ciq.dev/beskar/cmd/beskarctl/yum" ) -func fatal(format string, a ...any) { - fmt.Printf(format+"\n", a...) - os.Exit(1) -} - func main() { - pushCmd := flag.NewFlagSet("push", flag.ExitOnError) - pushRepo := pushCmd.String("repo", "", "repo") - pushRegistry := pushCmd.String("registry", "", "registry") - - pushMetadataCmd := flag.NewFlagSet("push-metadata", flag.ExitOnError) - pushMetadataRepo := pushMetadataCmd.String("repo", "", "repo") - pushMetadataRegistry := pushMetadataCmd.String("registry", "", "registry") - pushMetadataType := pushMetadataCmd.String("type", "", "type") - - if len(os.Args) == 1 { - fatal("missing subcommand") - } - - switch os.Args[1] { - case "version": - fmt.Println(version.Semver) - case "push": - if err := pushCmd.Parse(os.Args[2:]); err != nil { - fatal("while parsing command arguments: %w", err) - } - rpm := pushCmd.Arg(0) - if rpm == "" { - fatal("an RPM package must be specified") - } else if pushRegistry == nil || *pushRegistry == "" { - fatal("a registry must be specified") - } else if pushRepo == nil || *pushRepo == "" { - fatal("a repo must be specified") - } - if err := push(rpm, *pushRepo, *pushRegistry); err != nil { - fatal("while pushing RPM package: %s", err) - } - case "push-metadata": - if err := pushMetadataCmd.Parse(os.Args[2:]); err != nil { - fatal("while parsing command arguments: %w", err) - } - metadata := pushMetadataCmd.Arg(0) - if metadata == "" { - fatal("a metadata file must be specified") - } else if pushMetadataRegistry == nil || *pushMetadataRegistry == "" { - fatal("a registry must be specified") - } else if pushMetadataRepo == nil || *pushMetadataRepo == "" { - fatal("a repo must be specified") - } else if pushMetadataType == nil || *pushMetadataType == "" { - fatal("a metadata type must be specified") - } - if err := pushMetadata(metadata, *pushMetadataType, *pushMetadataRepo, *pushMetadataRegistry); err != nil { - fatal("while pushing metadata: %s", err) - } - default: - fatal("unknown %q subcommand", os.Args[1]) - } -} - -func push(rpmPath string, repo, registry string) error { - pusher, err := orasrpm.NewRPMPusher(rpmPath, repo, name.WithDefaultRegistry(registry)) - if err != nil { - return fmt.Errorf("while creating RPM pusher: %w", err) - } - - fmt.Printf("Pushing %s to %s\n", rpmPath, pusher.Reference()) - - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) -} - -func pushMetadata(metadataPath string, dataType, repo, registry string) error { - pusher, err := orasrpm.NewRPMExtraMetadataPusher(metadataPath, repo, dataType, name.WithDefaultRegistry(registry)) - if err != nil { - return fmt.Errorf("while creating RPM metadata pusher: %w", err) - } - - fmt.Printf("Pushing %s to %s\n", metadataPath, pusher.Reference()) - - return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + ctl.Execute( + yum.RootCmd(), + static.RootCmd(), + ostree.RootCmd(), + ) } diff --git a/cmd/beskarctl/ostree/file.go b/cmd/beskarctl/ostree/file.go new file mode 100644 index 0000000..8de9bca --- /dev/null +++ b/cmd/beskarctl/ostree/file.go @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostree + +import ( + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "ostree", + Aliases: []string{ + "o", + }, + Short: "Operations related to ostree repositories.", +} + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + ) + + return rootCmd +} diff --git a/cmd/beskarctl/ostree/repo.go b/cmd/beskarctl/ostree/repo.go new file mode 100644 index 0000000..412c85e --- /dev/null +++ b/cmd/beskarctl/ostree/repo.go @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostree + +import ( + "context" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/orasostree" +) + +var ( + pushCmd = &cobra.Command{ + Use: "push [directory]", + Short: "Push an ostree repository.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := args[0] + if dir == "" { + return ctl.Err("a directory must be specified") + } + + repoPusher := orasostree.NewOSTreeRepositoryPusher(context.Background(), dir, ctl.Repo(), jobCount) + repoPusher = repoPusher.WithNameOptions(name.WithDefaultRegistry(ctl.Registry())) + repoPusher = repoPusher.WithRemoteOptions(remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err := repoPusher.Push(); err != nil { + return ctl.Errf("while pushing ostree repository: %s", err) + } + return nil + }, + } + jobCount int +) + +func PushCmd() *cobra.Command { + pushCmd.Flags().IntVarP( + &jobCount, + "jobs", + "j", + 10, + "The number of concurrent jobs to use for pushing the repository.", + ) + return pushCmd +} diff --git a/cmd/beskarctl/static/push.go b/cmd/beskarctl/static/push.go new file mode 100644 index 0000000..56e7d88 --- /dev/null +++ b/cmd/beskarctl/static/push.go @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package static + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasfile" +) + +var pushCmd = &cobra.Command{ + Use: "push [file]", + Short: "Push a file.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + file := args[0] + if file == "" { + return ctl.Err("file must be specified") + } + + if err := push(file, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing static file: %s", err) + } + return nil + }, +} + +func PushCmd() *cobra.Command { + return pushCmd +} + +func push(filepath, repo, registry string) error { + pusher, err := orasfile.NewStaticFilePusher(filepath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating StaticFile pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", filepath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/static/root.go b/cmd/beskarctl/static/root.go new file mode 100644 index 0000000..f917157 --- /dev/null +++ b/cmd/beskarctl/static/root.go @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package static + +import ( + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "static", + Aliases: []string{ + "file", + "s", + }, + Short: "Operations related to static files.", +} + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + ) + + return rootCmd +} diff --git a/cmd/beskarctl/yum/push.go b/cmd/beskarctl/yum/push.go new file mode 100644 index 0000000..a8b6379 --- /dev/null +++ b/cmd/beskarctl/yum/push.go @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package yum + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasrpm" +) + +var pushCmd = &cobra.Command{ + Use: "push [rpm filepath]", + Short: "Push a yum repository to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rpm := args[0] + if rpm == "" { + return ctl.Err("an RPM package must be specified") + } + + if err := push(rpm, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing RPM package: %s", err) + } + return nil + }, +} + +func PushCmd() *cobra.Command { + return pushCmd +} + +func push(rpmPath, repo, registry string) error { + pusher, err := orasrpm.NewRPMPusher(rpmPath, repo, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating RPM pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", rpmPath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/yum/pushmetadata.go b/cmd/beskarctl/yum/pushmetadata.go new file mode 100644 index 0000000..6f1015c --- /dev/null +++ b/cmd/beskarctl/yum/pushmetadata.go @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package yum + +import ( + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/pkg/oras" + "go.ciq.dev/beskar/pkg/orasrpm" +) + +// yum push-metadata +var ( + pushMetadataCmd = &cobra.Command{ + Use: "push-metadata [metadata filepath]", + Short: "Push yum repository metadata to a registry.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + metadata := args[0] + if metadata == "" { + return ctl.Err("a metadata file must be specified") + } else if registry == "" { + return MissingRequiredFlagRegistry + } else if repo == "" { + return MissingRequiredFlagRepo + } else if pushMetadataType == "" { + return ctl.Err("a metadata type must be specified") + } + + if err := pushMetadata(metadata, pushMetadataType, ctl.Repo(), ctl.Registry()); err != nil { + return ctl.Errf("while pushing metadata: %s", err) + } + return nil + }, + } + pushMetadataType string +) + +func PushMetadataCmd() *cobra.Command { + pushMetadataCmd.Flags().StringVarP(&pushMetadataType, "type", "t", "", "type") + return pushMetadataCmd +} + +func pushMetadata(metadataPath, dataType, repo, registry string) error { + pusher, err := orasrpm.NewRPMExtraMetadataPusher(metadataPath, repo, dataType, name.WithDefaultRegistry(registry)) + if err != nil { + return fmt.Errorf("while creating RPM metadata pusher: %w", err) + } + + fmt.Printf("Pushing %s to %s\n", metadataPath, pusher.Reference()) + + return oras.Push(pusher, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} diff --git a/cmd/beskarctl/yum/root.go b/cmd/beskarctl/yum/root.go new file mode 100644 index 0000000..3c2ea4b --- /dev/null +++ b/cmd/beskarctl/yum/root.go @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package yum + +import ( + "github.com/spf13/cobra" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" +) + +const ( + MissingRequiredFlagRepo ctl.Err = "a repo must be specified" + MissingRequiredFlagRegistry ctl.Err = "a registry must be specified" +) + +var ( + repo string + registry string + rootCmd = &cobra.Command{ + Use: "yum", + Aliases: []string{ + "y", + "rpm", + "dnf", + }, + Short: "Operations related to yum repositories.", + } +) + +func RootCmd() *cobra.Command { + rootCmd.AddCommand( + PushCmd(), + PushMetadataCmd(), + ) + + return rootCmd +} diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 0000000..322cdb8 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,166 @@ +# Beskar Plugins +A Beskar plugin is a binary that is deployed alongside Beskar and is responsible for managing a specific type of +artifact. For example, the `yum` plugin is responsible for managing RPMs. In it's very basic form a plugin is responsible +for mapping an incoming request to an artifact in the registry. Plugins may contain additional logic to support other actions +such as uploading, deleting, etc. For example, the `yum` plugin supports mirroring a remote repository. + +## How To Use This Document +This document is intended to be a guide for writing a Beskar plugin. It is not intended to be a complete reference. Use +the information provided here to get started and then refer to the code for more details. `internal/plugins/static` is a +simple plugin to use as a reference. It is recommended that you read through the code and then use it as a starting point +for your own plugin. + +## Plugin Architecture +A Beskar plugin is written in Go and will be deployed so that it can be accessed by Beskar. There are a few mechanisms that +Beskar uses to discover and communicate with plugins. The first of which is a gossip protocol that is used to discover +plugins. The second is the Events API that is used to keep plugins in sync with Beskar, such as when an artifact is uploaded +or deleted. The third is the plugin service that is used to serve the plugin's API. We will cover these in more detail below, +but luckily Beskar provides a few interfaces, as well as a series of helper methods, to make writing a plugin easier. + +### Plugin Discovery and API Request Routing +Beskar uses [a gossip protocol](https://github.com/hashicorp/memberlist) to discover plugins. Early in its startup process a plugin will register itself +with a known peer, generally one of the main Beskar instances, and the plugin's info will be shared with the rest of the cluster. +This info includes the plugin's name, version, and the address of the plugin's API. Beskar will then use this info to route +requests to the plugin's API using a [Rego policy](https://www.openpolicyagent.org/) provided by the plugin. + +**Note that you do not need to do anything special to register your plugin. Beskar will handle this for you.** All you need +to do is provide the plugin's info, which includes the rego policy, and a router. We will cover this in more detail later. + +### Repository Handler +In some cases your plugin may need to be informed when an artifact is uploaded or deleted. This is accomplished by +implementing the [Handler interface](../internal/pkg/repository/handler.go). The object you implement will be used to receive events from Beskar and will +enable your plugin to keep its internal state up to date. + +#### Implementation Notes +When implementing your `repository.Handler` there are a few things to keep in mind. + +First, the `QueueEvent()` method is not intended to be used to perform long-running operations. Instead, you should +queue the event for processing in another goroutine. The static plugin provides a good example of this by spinning +up a goroutine in its `Start()` that listens for events and processes them, while the `QueueEvent()` method simply queues +the event for processing in the run loop. + +Second, Beskar provides a [RepoHandler struct](../internal/pkg/repository/handler.go) that partially implements the +`Handler` interface and provides some helper methods that reduce your implementation burden to only `Start()` and +`QueueEvent()`. This is exemplified below as well as in the [Static plugin](../internal/plugins/static/pkg/staticrepository/handler.go). + +Third, we recommend that you create a constructor for your handler that conforms to the `repository.HandlerFactory` type. +This will come in handy later when creating the plugin service. + +#### Example Implementation of `repository.Handler` +``` + +type ExampleHandler struct { + *repository.RepoHandler +} + +func NewExampleHandler(*slog.Logger, repoHandler *repository.RepoHandler) *ExampleHandler { + return &ExampleHandler{ + RepoHandler: repoHandler, + } +} + +func (h *ExampleHandler) Start(ctx context.Context) { + // Process stored events + // Start goroutine to dequeue and process new events +} + +func (h *ExampleHandler) QueueEvent(event *eventv1.EventPayload, store bool) error { + // Store event if store is true + // Queue event for processing + return nil +} +``` + +#### Plugins without internal state +Not all plugins will have internal state, for example, the [Static plugin](../internal/plugins/ostree/plugin.go). simply +maps an incoming request to an artifact in the registry. In these cases, it is not required to implement a +`repository.Handler`. You can simply return `nil` from the `RepositoryManager()` method of your plugin service and leave +your plugin's `Info.MediaTypes` empty. This will tell Beskar that your plugin does not need to receive events. More on +this in the next section. + + +### Plugin Service +The [Plugin Service](../internal/pkg/pluginsrv/service.go) is responsible for serving the plugin's API, registering your +`repository.Handler` and providing the info Beskar needs about your plugin. We recommend that your implementation of +`pluginsrv.Service` have a constructor that accepts a config object and returns a pointer to your service. For example: +``` + +//go:embed embedded/router.rego +var routerRego []byte + +//go:embed embedded/data.json +var routerData []byte + +const ( + // PluginName is the name of the plugin + PluginName = "example" +) + +type ExamplePlugin struct { + ctx context.Context + config pluginsrv.Config + + repositoryManager *repository.Manager + handlerParams *repository.HandlerParams +} + +type ExamplePluginConfig struct { + Gossip gossip.Config +} + +func NewExamplePlugin(ctx context.Context, exampleConfig ExamplePluginConfig) (*ExamplePlugin, error) { + config := pluginsrv.Config{} + + router := chi.NewRouter() + // for kubernetes probes + router.Handle("/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + config.Router = router + config.Gossip = exampleConfig.Gossip + config.Info = &pluginv1.Info{ + Name: PluginName, + Version: version.Semver, + Mediatypes: []string{ + "application/vnd.ciq.example.file.v1.config+json", + }, + Router: &pluginv1.Router{ + Rego: routerRego, + Data: routerData, + }, + } + + plugin := ExamplePlugin{ + ctx: ctx, + config: config, + } + + plugin.repositoryManager = repository.NewManager(plugin.handlerParams, NewExampleHandler) + + return &plugin, nil +} + +func (p *ExamplePlugin) Start(http.RoundTripper, *mtls.CAPEM, *gossip.BeskarMeta) error { + // Register handlers with p.config.Router + return nil +} + +func (p *ExamplePlugin) Context() context.Context { + return p.ctx +} + +func (p *ExamplePlugin) Config() Config { + return p.config +} + +func (p *ExamplePlugin) RepositoryManager() *repository.Manager { + return nil +} +``` + + +#### Your Plugin's API +The `Start(...)` method is called when the server is about to serve your plugin's api and is your chance to register your +plugin's handlers with the server. + +The `Config()` method is used to return your plugin's configuration. This is used by Beskar to generate the plugin's + diff --git a/internal/pkg/beskar/plugin.go b/internal/pkg/beskar/plugin.go index d26bd10..b382a85 100644 --- a/internal/pkg/beskar/plugin.go +++ b/internal/pkg/beskar/plugin.go @@ -100,12 +100,16 @@ func (pm *pluginManager) setClientTLSConfig(tlsConfig *tls.Config) { } func (pm *pluginManager) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // We expect the request to be of the form /artifacts/{plugin_name}/... + // If it is not, we return a 404. matches := artifactsMatch.FindStringSubmatch(r.URL.Path) if len(matches) < 2 { w.WriteHeader(http.StatusNotFound) return } + // Check if the plugin is registered with a name matching the second path component. + // If it is not, we return a 404. pm.pluginsMutex.RLock() pl := pm.plugins[matches[1]] pm.pluginsMutex.RUnlock() @@ -115,6 +119,7 @@ func (pm *pluginManager) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + // Forward the request to the plugin. (The in memory representation of the plugin not the plugin application itself) pl.ServeHTTP(w, r) } @@ -271,6 +276,9 @@ type plugin struct { func (p *plugin) ServeHTTP(w http.ResponseWriter, r *http.Request) { key := r.RemoteAddr + // If the request is for a repository, we need to check if the router has a decision for it. + // If it does, we need to redirect the request to the appropriate location. + // If it does not, we need to use the node hash to find the appropriate node to forward the request to. result, err := p.router.Load().Decision(r, p.registry) if err != nil { p.logger.Errorf("%s router decision error: %s", p.name, err) diff --git a/internal/pkg/pluginsrv/service.go b/internal/pkg/pluginsrv/service.go index 505268c..e631b11 100644 --- a/internal/pkg/pluginsrv/service.go +++ b/internal/pkg/pluginsrv/service.go @@ -37,9 +37,17 @@ type Config struct { } type Service[H repository.Handler] interface { + // Start starts the service's HTTP server. Start(http.RoundTripper, *mtls.CAPEM, *gossip.BeskarMeta) error + + // Context returns the service's context. Context() context.Context + + // Config returns the service's configuration. Config() Config + + // RepositoryManager returns the service's repository manager. + // For plugin's without a repository manager, this method should return nil. RepositoryManager() *repository.Manager[H] } @@ -96,6 +104,23 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err } repoManager := service.RepositoryManager() + if repoManager != nil { + // Gracefully shutdown repository handlers + defer func() { + var wg sync.WaitGroup + for name, handler := range repoManager.GetAll() { + wg.Add(1) + + go func(name string, handler repository.Handler) { + logger.Info("stopping repository handler", "repository", name) + handler.Stop() + logger.Info("repository handler stopped", "repository", name) + wg.Done() + }(name, handler) + } + wg.Wait() + }() + } ticker := time.NewTicker(time.Second * 5) @@ -112,8 +137,8 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err manager: repoManager, } - serviceConfig.Router.HandleFunc("/event", http.HandlerFunc(wh.event)) - serviceConfig.Router.HandleFunc("/info", http.HandlerFunc(wh.info)) + serviceConfig.Router.With(IsTLSMiddleware).HandleFunc("/event", wh.event) + serviceConfig.Router.With(IsTLSMiddleware).HandleFunc("/info", wh.info) transport, err := getBeskarTransport(caPEM, beskarMeta) if err != nil { @@ -142,21 +167,6 @@ func Serve[H repository.Handler](ln net.Listener, service Service[H]) (errFn err _ = server.Shutdown(ctx) - var wg sync.WaitGroup - - for name, handler := range repoManager.GetAll() { - wg.Add(1) - - go func(name string, handler repository.Handler) { - logger.Info("stopping repository handler", "repository", name) - handler.Stop() - logger.Info("repository handler stopped", "repository", name) - wg.Done() - }(name, handler) - } - - wg.Wait() - return serverErr } diff --git a/internal/pkg/pluginsrv/webhandler.go b/internal/pkg/pluginsrv/webhandler.go index eb2cc76..97c1767 100644 --- a/internal/pkg/pluginsrv/webhandler.go +++ b/internal/pkg/pluginsrv/webhandler.go @@ -29,8 +29,19 @@ func IsTLS(w http.ResponseWriter, r *http.Request) bool { return true } +// IsTLSMiddleware is a middleware that checks if the request is TLS. This is a convenience wrapper around IsTLS. +func IsTLSMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !IsTLS(w, r) { + return + } + next.ServeHTTP(w, r) + }) +} + func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { - if !IsTLS(w, r) { + if wh.manager == nil { + w.WriteHeader(http.StatusNotImplemented) return } @@ -85,10 +96,6 @@ func (wh *webHandler[H]) event(w http.ResponseWriter, r *http.Request) { } func (wh *webHandler[H]) info(w http.ResponseWriter, r *http.Request) { - if !IsTLS(w, r) { - return - } - if r.Method != http.MethodGet { w.WriteHeader(http.StatusNotImplemented) return diff --git a/internal/pkg/repository/handler.go b/internal/pkg/repository/handler.go index f5fe7fa..e3f8b47 100644 --- a/internal/pkg/repository/handler.go +++ b/internal/pkg/repository/handler.go @@ -7,12 +7,16 @@ import ( "context" "errors" "io" + "net" "os" "path/filepath" + "strconv" "sync" "sync/atomic" "time" + "go.ciq.dev/beskar/internal/pkg/gossip" + "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" eventv1 "go.ciq.dev/beskar/pkg/api/event/v1" @@ -25,19 +29,41 @@ type HandlerParams struct { RemoteOptions []remote.Option NameOptions []name.Option remove func(string) + BeskarMeta *gossip.BeskarMeta } func (hp HandlerParams) Remove(repository string) { hp.remove(repository) } +func (hp HandlerParams) GetBeskarServiceHostPort() string { + return net.JoinHostPort(hp.BeskarMeta.Hostname, strconv.Itoa(int(hp.BeskarMeta.ServicePort))) +} + +func (hp HandlerParams) GetBeskarRegistryHostPort() string { + return net.JoinHostPort(hp.BeskarMeta.Hostname, strconv.Itoa(int(hp.BeskarMeta.RegistryPort))) +} + +// Handler - Interface for handling events for a repository. type Handler interface { + // QueueEvent - Called when a new event is received. If store is true, the event should be stored in the database. + // Note: Avoid performing any long-running operations in this function. QueueEvent(event *eventv1.EventPayload, store bool) error + + // Started - Returns true if the handler has started. Started() bool + + // Start - Called when the handler should start processing events. + // This is your chance to set up any resources, e.g., database connections, run loops, etc. + // This will only be called once. Start(context.Context) + + // Stop - Called when the handler should stop processing events and clean up resources. Stop() } +// RepoHandler - A partial default implementation of the Handler interface that provides some common functionality. +// You can embed this in your own handler to get some default functionality, e.g., an event queue. type RepoHandler struct { Repository string Params *HandlerParams @@ -169,6 +195,14 @@ func (rh *RepoHandler) DeleteManifest(ref string) (errFn error) { return remote.Delete(namedRef, rh.Params.RemoteOptions...) } +func (rh *RepoHandler) PullManifest(ref string) (errFn error) { + namedRef, err := name.ParseReference(ref, rh.Params.NameOptions...) + if err != nil { + return err + } + return remote.Delete(namedRef, rh.Params.RemoteOptions...) +} + func (rh *RepoHandler) SyncArtifact(ctx context.Context, name string, timeout time.Duration) (chan error, func() error) { errCh := make(chan error, 1) diff --git a/internal/plugins/ostree/README.md b/internal/plugins/ostree/README.md new file mode 100644 index 0000000..8ed1e84 --- /dev/null +++ b/internal/plugins/ostree/README.md @@ -0,0 +1,20 @@ +# OSTree Plugin + +## Overview +The ostree plugin is responsible for mapping the ostree repository to the OCI registry. This is done in the router.rego +and no routing execution happens within the plugin itself at runtime. The plugin does, however, provide an API for mirroring +ostree repositories into beskar. + +## File Tagging +The ostree plugin maps the ostree repository filepaths to the OCI registry tags. Most files are simply mapped by hashing +the full filepath relative to the ostree root. For example, `objects/ab/abcd1234.filez` becomes `file:b8458bd029a97ca5e03f272a6b7bd0d1`. +There are a few exceptions to this rule, however. The following files are considered "special" and are tagged as follows: +1. `summary` -> `file:summary` +2. `summary.sig` -> `file:summary.sig` +3. `config` -> `file:config` + +There is no technical reason for this and is only done to make the mapping more human-readable in the case of "special" +files. + +## Mirroring +TBD \ No newline at end of file diff --git a/internal/plugins/ostree/api.go b/internal/plugins/ostree/api.go new file mode 100644 index 0000000..9bbaa2f --- /dev/null +++ b/internal/plugins/ostree/api.go @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostree + +import ( + "context" + + "github.com/RussellLuo/kun/pkg/werror" + "github.com/RussellLuo/kun/pkg/werror/gcode" + + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" +) + +func checkRepository(repository string) error { + if !apiv1.RepositoryMatch(repository) { + return werror.Wrapf(gcode.ErrInvalidArgument, "invalid repository name, must match expression %q", apiv1.RepositoryRegex) + } + return nil +} + +func (p *Plugin) CreateRepository(ctx context.Context, repository string, properties *apiv1.OSTreeRepositoryProperties) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).CreateRepository(ctx, properties) +} + +func (p *Plugin) DeleteRepository(ctx context.Context, repository string) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).DeleteRepository(ctx) +} + +func (p *Plugin) AddRemote(ctx context.Context, repository string, properties *apiv1.OSTreeRemoteProperties) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).AddRemote(ctx, properties) +} + +func (p *Plugin) SyncRepository(ctx context.Context, repository string, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { + if err := checkRepository(repository); err != nil { + return err + } + + return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx, properties) +} + +func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) { + if err := checkRepository(repository); err != nil { + return nil, err + } + + return p.repositoryManager.Get(ctx, repository).GetRepositorySyncStatus(ctx) +} diff --git a/internal/plugins/ostree/embedded/data.json b/internal/plugins/ostree/embedded/data.json new file mode 100644 index 0000000..337cbc3 --- /dev/null +++ b/internal/plugins/ostree/embedded/data.json @@ -0,0 +1,28 @@ +{ + "routes": [ + { + "pattern": "^/(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)/repo/([a-z0-9]+(?:[/._-][a-z0-9]+)*)$", + "methods": [ + "GET", + "HEAD" + ] + }, + { + "pattern": "^/artifacts/ostree/api/v1/doc/(.*)$", + "body": false + }, + { + "pattern": "^/artifacts/ostree/api/v1/(.*)$", + "body": true, + "body_key": "repository" + } + ], + "mediatype": { + "file": "application/vnd.ciq.ostree.v1.file" + }, + "tags": [ + "summary", + "summary.sig", + "config" + ] +} \ No newline at end of file diff --git a/internal/plugins/ostree/embedded/router.rego b/internal/plugins/ostree/embedded/router.rego new file mode 100644 index 0000000..08f365e --- /dev/null +++ b/internal/plugins/ostree/embedded/router.rego @@ -0,0 +1,64 @@ +package router + +import future.keywords.if +import future.keywords.in + +default output = {"repository": "", "redirect_url": "", "found": false} + +makeTag(filename) = checksum if { + filename in data.tags + checksum := filename +} else = checksum { + checksum := crypto.md5(filename) +} + +blob_url(repo, filename) = url { + digest := oci.blob_digest(sprintf("%s:%s", [repo, makeTag(filename)]), "mediatype", data.mediatype.file) + url := { + "url": sprintf("/v2/%s/blobs/sha256:%s", [repo, digest]), + "found": digest != "", + } +} + +output = obj { + some index + input.method in data.routes[index].methods + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + redirect := blob_url( + sprintf("%s/file", [match[1]]), + match[2], + ) + obj := { + "repository": match[1], + "redirect_url": redirect.url, + "found": redirect.found + } +} else = obj if { + data.routes[index].body == true + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + repo := object.get(request.body(), data.routes[index].body_key, "") + obj := { + "repository": repo, + "redirect_url": "", + "found": repo != "" + } +} else = obj { + match := regex.find_all_string_submatch_n( + data.routes[index].pattern, + input.path, + 1 + )[0] + obj := { + "repository": "", + "redirect_url": "", + "found": true + } +} \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/config/beskar-ostree.go b/internal/plugins/ostree/pkg/config/beskar-ostree.go new file mode 100644 index 0000000..10df529 --- /dev/null +++ b/internal/plugins/ostree/pkg/config/beskar-ostree.go @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + + "github.com/distribution/distribution/v3/configuration" + "go.ciq.dev/beskar/internal/pkg/config" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" +) + +const ( + BeskarOSTreeConfigFile = "beskar-ostree.yaml" + DefaultBeskarOSTreeDataDir = "/tmp/beskar-ostree" +) + +//go:embed default/beskar-ostree.yaml +var defaultBeskarOSTreeConfig string + +type BeskarOSTreeConfig struct { + Version string `yaml:"version"` + Log log.Config `yaml:"log"` + Addr string `yaml:"addr"` + Gossip gossip.Config `yaml:"gossip"` + Profiling bool `yaml:"profiling"` + DataDir string `yaml:"datadir"` + ConfigDirectory string `yaml:"-"` +} + +type BeskarOSTreeConfigV1 BeskarOSTreeConfig + +func ParseBeskarOSTreeConfig(dir string) (*BeskarOSTreeConfig, error) { + customDir := false + filename := filepath.Join(config.DefaultConfigDir, BeskarOSTreeConfigFile) + if dir != "" { + filename = filepath.Join(dir, BeskarOSTreeConfigFile) + customDir = true + } + + configDir := filepath.Dir(filename) + + var configReader io.Reader + + f, err := os.Open(filename) + if err != nil { + if !errors.Is(err, os.ErrNotExist) || customDir { + return nil, err + } + configReader = strings.NewReader(defaultBeskarOSTreeConfig) + configDir = "" + } else { + defer func() { + if err := f.Close(); err != nil { + fmt.Println(err) + } + }() + configReader = f + } + + configBuffer := new(bytes.Buffer) + if _, err := io.Copy(configBuffer, configReader); err != nil { + return nil, err + } + + configParser := configuration.NewParser("beskarostree", []configuration.VersionedParseInfo{ + { + Version: configuration.MajorMinorVersion(1, 0), + ParseAs: reflect.TypeOf(BeskarOSTreeConfigV1{}), + ConversionFunc: func(c interface{}) (interface{}, error) { + if v1, ok := c.(*BeskarOSTreeConfigV1); ok { + v1.ConfigDirectory = configDir + return (*BeskarOSTreeConfig)(v1), nil + } + return nil, fmt.Errorf("expected *BeskarOSTreeConfigV1, received %#v", c) + }, + }, + }) + + beskarOSTreeConfig := new(BeskarOSTreeConfig) + + if err := configParser.Parse(configBuffer.Bytes(), beskarOSTreeConfig); err != nil { + return nil, err + } + + if beskarOSTreeConfig.DataDir == "" { + beskarOSTreeConfig.DataDir = DefaultBeskarOSTreeDataDir + } + + return beskarOSTreeConfig, nil +} diff --git a/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml b/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml new file mode 100644 index 0000000..043fb9b --- /dev/null +++ b/internal/plugins/ostree/pkg/config/default/beskar-ostree.yaml @@ -0,0 +1,16 @@ +version: 1.0 + +addr: 0.0.0.0:5200 + +log: + level: debug + format: json + +profiling: true +datadir: /tmp/beskar-ostree + +gossip: + addr: 0.0.0.0:5201 + key: XD1IOhcp0HWFgZJ/HAaARqMKJwfMWtz284Yj7wxmerA= + peers: + - 127.0.0.1:5102 diff --git a/internal/plugins/ostree/pkg/libostree/README.md b/internal/plugins/ostree/pkg/libostree/README.md new file mode 100644 index 0000000..785c327 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/README.md @@ -0,0 +1,29 @@ +# ostree + +ostree is a wrapper around [libostree](https://github.com/ostreedev/ostree) that aims to provide an idiomatic API. + + +### Notes +1. A minimal glib implementation exists within the ostree pkg. This is to avoid a dependency on glib for the time being. + - This implementation is not complete and will be expanded as needed. + - The glib implementation is not intended to be used outside of the ostree pkg. + - `GCancellable` is not implemented on some functions. If the func accepts a context.Context it most likely implements a GCancellable. +2. Not all of libostree is wrapped. Only the parts that are needed for beskar are wrapped. Which is basically everything + need to perform pull operations. + - `OstreeAsyncProgress` is not implemented. Just send nil. + + +### Developer Warnings +- `glib/gobject` are used here and add a layer of complexity to the code, specifically with regard to memory management. +glib/gobject are reference counted and objects are freed when the reference count reaches 0. Therefore, you will see +`C.g_XXX_ref_sink` or `C.g_XXX_ref` (increases reference count) and `C.g_XXX_unref()` (decrease reference count) in some +places and `C.free()` in others. A good rule of thumb is that if you see a `g_` prefix you are dealing with a reference +counted object and should not call `C.free()`. See [glib](https://docs.gtk.org/glib/index.html) for more information. +See [gobject](https://docs.gtk.org/gobject/index.html) for more information. + + +### Testdata +The testdata directory contains a simple ostree repo that can be used for testing. It was created using the generate-testdata.sh +script. testdata has been committed to this git repo so that it remains static. If you need to regenerate the testdata you can, +however, keep in mind that newer versions of ostree may produce different results and may cause tests to fail. The version +of ostree used to generate the testdata is 2023.7. \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/errors.go b/internal/plugins/ostree/pkg/libostree/errors.go new file mode 100644 index 0000000..f942f8f --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/errors.go @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package libostree + +type Err string + +func (e Err) Error() string { + return string(e) +} + +const ( + ErrInvalidPath = Err("invalid path") +) diff --git a/internal/plugins/ostree/pkg/libostree/generate-testdata.sh b/internal/plugins/ostree/pkg/libostree/generate-testdata.sh new file mode 100755 index 0000000..d5dd3d0 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/generate-testdata.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +# This script generates test data for libostree. + +# Clean up any existing test data +rm -rf testdata + +mkdir -p testdata/{repo,tree} + +# Create a simple ostree repo with two branches and a series of commits. +ostree --repo=testdata/repo init --mode=archive + +echo "Test file in a simple ostree repo - branch test1" > ./testdata/tree/testfile.txt +ostree --repo=testdata/repo commit --branch=test1 ./testdata/tree/ + +echo "Test file in a simple ostree repo - branch test2" > ./testdata/tree/testfile.txt +ostree --repo=testdata/repo commit --branch=test2 ./testdata/tree/ + +echo "Another test file" > ./testdata/tree/another_testfile.txt +ostree --repo=testdata/repo commit --branch=test2 ./testdata/tree/ + +ostree --repo=testdata/repo summary --update + +# We don't actually need the tree directory, just the repo. +rm -rf testdata/tree \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go b/internal/plugins/ostree/pkg/libostree/glib_helpers.go new file mode 100644 index 0000000..e6fda4a --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go @@ -0,0 +1,23 @@ +//nolint:goheader +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package libostree + +// #include "glib_helpers.go.h" +import "C" + +import ( + "errors" +) + +// GoError converts a C glib error to a Go error. +// The C error is freed after conversion. +func GoError(e *C.GError) error { + defer C.g_error_free(e) + + if e == nil { + return nil + } + return errors.New(C.GoString(C._g_error_get_message(e))) +} diff --git a/internal/plugins/ostree/pkg/libostree/glib_helpers.go.h b/internal/plugins/ostree/pkg/libostree/glib_helpers.go.h new file mode 100644 index 0000000..4aceb8f --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/glib_helpers.go.h @@ -0,0 +1,11 @@ +#include +#include +#include +#include + +static char * +_g_error_get_message (GError *error) +{ + g_assert (error != NULL); + return error->message; +} \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/options.go b/internal/plugins/ostree/pkg/libostree/options.go new file mode 100644 index 0000000..90f0735 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/options.go @@ -0,0 +1,65 @@ +//nolint:goheader +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package libostree + +// #include "options.go.h" +import "C" +import "unsafe" + +// Option defines an option for pulling ostree repos. +// It is used to build a *C.GVariant via a *C.GVariantBuilder. +// deferFree is an optional function that frees the memory allocated by the option. deferFree may be called more than once. +type ( + Option func(builder *C.GVariantBuilder, deferFree deferredFreeFn) + deferredFreeFn func(...unsafe.Pointer) +) + +// ToGVariant converts the given Options to a GVariant using a GVaraintBuilder. +func toGVariant(opts ...Option) *C.GVariant { + typeStr := C.CString("a{sv}") + defer C.free(unsafe.Pointer(typeStr)) + + variantType := C.g_variant_type_new(typeStr) + + // The builder is freed by g_variant_builder_end below. + // See https://docs.gtk.org/glib/method.VariantBuilder.init.html + var builder C.GVariantBuilder + C.g_variant_builder_init(&builder, variantType) + + // Collect pointers to free later + var toFree []unsafe.Pointer + deferFreeFn := func(ptrs ...unsafe.Pointer) { + toFree = append(toFree, ptrs...) + } + + for _, opt := range opts { + opt(&builder, deferFreeFn) + } + defer func() { + for i := 0; i < len(toFree); i++ { + C.free(toFree[i]) + } + }() + + variant := C.g_variant_builder_end(&builder) + return C.g_variant_ref_sink(variant) +} + +func gVariantBuilderAddVariant(builder *C.GVariantBuilder, key *C.gchar, variant *C.GVariant) { + C.g_variant_builder_add_variant(builder, key, variant) +} + +// NoGPGVerify sets the gpg-verify option to false in the pull options. +func NoGPGVerify() Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + key := C.CString("gpg-verify") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(0))), + ) + } +} diff --git a/internal/plugins/ostree/pkg/libostree/options.go.h b/internal/plugins/ostree/pkg/libostree/options.go.h new file mode 100644 index 0000000..20b89d9 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/options.go.h @@ -0,0 +1,14 @@ +#include +#include +#include +#include + +// This exists because cGo doesn't support variadic functions +void +g_variant_builder_add_variant( + GVariantBuilder *builder, + const gchar *key, + GVariant *value +) { + g_variant_builder_add(builder, "{s@v}", key, value); +} diff --git a/internal/plugins/ostree/pkg/libostree/ostree.go b/internal/plugins/ostree/pkg/libostree/ostree.go new file mode 100644 index 0000000..f59b83c --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/ostree.go @@ -0,0 +1,10 @@ +//nolint:goheader +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package libostree + +// This file is used to generate the cgo pkg-config flags for libostree. + +// #cgo pkg-config: ostree-1 glib-2.0 +import "C" diff --git a/internal/plugins/ostree/pkg/libostree/pull.go b/internal/plugins/ostree/pkg/libostree/pull.go new file mode 100644 index 0000000..bb05bf3 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/pull.go @@ -0,0 +1,213 @@ +//nolint:goheader +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package libostree + +// #include +// #include "pull.go.h" +import "C" + +import ( + "context" + "unsafe" +) + +// Pull pulls refs from the named remote. +// Returns an error if the refs could not be fetched. +func (r *Repo) Pull(ctx context.Context, remote string, opts ...Option) error { + cremote := C.CString(remote) + defer C.free(unsafe.Pointer(cremote)) + + options := toGVariant(opts...) + defer C.g_variant_unref(options) + + var cErr *C.GError + + cCancel := C.g_cancellable_new() + go func() { + //nolint:gosimple + for { + select { + case <-ctx.Done(): + C.g_cancellable_cancel(cCancel) + return + } + } + }() + + // Pull refs from remote + if C.ostree_repo_pull_with_options( + r.native, + cremote, + options, + nil, + cCancel, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +type FlagSet int + +const ( + // Mirror - Write out refs suitable for mirrors and fetch all refs if none requested + Mirror = 1 << iota + + // CommitOnly - Fetch only the commit metadata + CommitOnly + + // Untrusted - Do verify checksums of local (filesystem-accessible) repositories (defaults on for HTTP) + Untrusted + + // BaseUserOnlyFiles - Since 2017.7. Reject writes of content objects with modes outside of 0775. + BaseUserOnlyFiles + + // TrustedHTTP - Don't verify checksums of objects HTTP repositories (Since: 2017.12) + TrustedHTTP + + // None - No special options for pull + None = 0 +) + +// Flags adds the given flags to the pull options. +func Flags(flags FlagSet) Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + key := C.CString("flags") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_int32(C.gint32(flags))), + ) + } +} + +// Refs adds the given refs to the pull options. +// When pulling refs from a remote, only the specified refs will be pulled. +func Refs(refs ...string) Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + cRefs := C.MakeRefArray(C.int(len(refs))) + deferFree(unsafe.Pointer(cRefs)) + for i := 0; i < len(refs); i++ { + cRef := C.CString(refs[i]) + deferFree(unsafe.Pointer(cRef)) + C.AppendRef(cRefs, C.int(i), cRef) + } + C.g_variant_builder_add_refs( + builder, + cRefs, + ) + } +} + +// NoGPGVerifySummary sets the gpg-verify-summary option to false in the pull options. +func NoGPGVerifySummary() Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + key := C.CString("gpg-verify-summary") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(0))), + ) + } +} + +// Depth sets the depth option to the given value in the pull options. +// How far in the history to traverse; default is 0, -1 means infinite +func Depth(depth int) Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + // 0 is the default depth so there is no need to add it to the builder. + if depth == 0 { + return + } + key := C.CString("depth") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_int32(C.gint32(depth))), + ) + } +} + +// DisableStaticDelta sets the disable-static-deltas option to true in the pull options. +// Do not use static deltas. +func DisableStaticDelta() Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + key := C.CString("disable-static-deltas") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(1))), + ) + } +} + +// RequireStaticDelta sets the require-static-deltas option to true in the pull options. +// Require static deltas. +func RequireStaticDelta() Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + key := C.CString("require-static-deltas") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(1))), + ) + } +} + +// DryRun sets the dry-run option to true in the pull options. +// Only print information on what will be downloaded (requires static deltas). +func DryRun() Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + key := C.CString("dry-run") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_boolean(C.gboolean(1))), + ) + } +} + +// AppendUserAgent sets the append-user-agent option to the given value in the pull options. +// Additional string to append to the user agent. +func AppendUserAgent(appendUserAgent string) Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + // "" is the default so there is no need to add it to the builder. + if appendUserAgent == "" { + return + } + + key := C.CString("append-user-agent") + deferFree(unsafe.Pointer(key)) + cAppendUserAgent := C.CString(appendUserAgent) + deferFree(unsafe.Pointer(cAppendUserAgent)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_string(cAppendUserAgent)), + ) + } +} + +// NetworkRetries sets the n-network-retries option to the given value in the pull options. +// Number of times to retry each download on receiving. +func NetworkRetries(n int) Option { + return func(builder *C.GVariantBuilder, deferFree deferredFreeFn) { + key := C.CString("n-network-retries") + deferFree(unsafe.Pointer(key)) + gVariantBuilderAddVariant( + builder, + key, + C.g_variant_new_variant(C.g_variant_new_int32(C.gint32(n))), + ) + } +} diff --git a/internal/plugins/ostree/pkg/libostree/pull.go.h b/internal/plugins/ostree/pkg/libostree/pull.go.h new file mode 100644 index 0000000..a78004f --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/pull.go.h @@ -0,0 +1,34 @@ +#include +#include +#include +#include + +// The following is a mechanism for converting a Go slice of strings to a char**. This could have been done in Go, but +// it's easier and less error prone to do it here. +char** MakeRefArray(int size) { + return calloc(sizeof(char*), size); +} + +void AppendRef(char** refs, int index, char* ref) { + refs[index] = ref; +} + +void FreeRefArray(char** refs) { + int i; + for (i = 0; refs[i] != NULL; i++) { + free(refs[i]); + } + free(refs); +} + +// This exists because cGo doesn't provide a way to cast char** to const char *const *. +void g_variant_builder_add_refs(GVariantBuilder *builder, char** refs) { + g_variant_builder_add( + builder, + "{s@v}", + "refs", + g_variant_new_variant( + g_variant_new_strv((const char *const *) refs, -1) + ) + ); +} \ No newline at end of file diff --git a/internal/plugins/ostree/pkg/libostree/pull_test.go b/internal/plugins/ostree/pkg/libostree/pull_test.go new file mode 100644 index 0000000..f62965e --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/pull_test.go @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package libostree + +import ( + "context" + "fmt" + "log" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +/* +The testdata directory contains a simple ostree repo that can be used for testing. It was created using the generate-testdata.sh +script. testdata has been committed to this git repo so that it remains static. If you need to regenerate the testdata you can, +however, keep in mind that newer versions of ostree may produce different results and may cause tests to fail. The version +of ostree used to generate the testdata is 2023.7. +*/ +func TestMain(m *testing.M) { + _, err := os.Stat("testdata/repo/summary") + if os.IsNotExist(err) { + log.Fatalln("testdata/repo does not exist: please run ./generate-testdata.sh") + } + + if err := os.MkdirAll("/tmp/libostree-pull_test", 0755); err != nil { + log.Fatalf("failed to create test directory: %s", err.Error()) + } + + os.Exit(m.Run()) +} + +func TestRepo_Pull(t *testing.T) { + fmt.Println(os.Getwd()) + svr := httptest.NewServer(http.FileServer(http.Dir("testdata/repo"))) + defer svr.Close() + + remoteName := "local" + remoteURL := svr.URL + //refs := []string{ + // "test1", + // "test2", + //} + + modes := []RepoMode{ + RepoModeArchive, + RepoModeArchiveZ2, + RepoModeBare, + RepoModeBareUser, + RepoModeBareUserOnly, + // RepoModeBareSplitXAttrs, + } + + // Test pull for each mode + for _, mode := range modes { + mode := mode + repoName := fmt.Sprintf("repo-%s", mode) + repoPath := fmt.Sprintf("/tmp/libostree-pull_test/%s", repoName) + + t.Run(repoName, func(t *testing.T) { + t.Cleanup(func() { + _ = os.RemoveAll(repoPath) + }) + + t.Run(fmt.Sprintf("should create repo in %s mode", mode), func(t *testing.T) { + repo, err := Init(repoPath, mode) + assert.NotNil(t, repo) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to initialize repo", "err: %s", err.Error()) + } + + t.Run("should not fail to init twice", func(t *testing.T) { + repo, err := Init(repoPath, mode) + assert.NotNil(t, repo) + assert.NoError(t, err) + }) + }) + + var repo *Repo + t.Run("should open repo", func(t *testing.T) { + var err error + repo, err = Open(repoPath) + assert.NotNil(t, repo) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to open repo", "err: %s", err.Error()) + } + }) + + t.Run("should create remote", func(t *testing.T) { + err := repo.AddRemote(remoteName, remoteURL, NoGPGVerify()) + assert.NoError(t, err) + + // Manually check the config file to ensure the remote was added + configData, err := os.ReadFile(fmt.Sprintf("%s/config", repoPath)) + if err != nil { + t.Errorf("failed to read config file: %s", err.Error()) + } + assert.Contains(t, string(configData), fmt.Sprintf(`[remote "%s"]`, remoteName)) + assert.Contains(t, string(configData), fmt.Sprintf(`url=%s`, remoteURL)) + }) + + t.Run("should error - remote already exists", func(t *testing.T) { + err := repo.AddRemote(remoteName, remoteURL) + assert.Error(t, err) + }) + + t.Run("should list remotes", func(t *testing.T) { + remotes := repo.ListRemotes() + assert.Equal(t, remotes, []string{remoteName}) + }) + + t.Run("should cancel pull", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := repo.Pull( + ctx, + remoteName, + Flags(Mirror|TrustedHTTP), + ) + assert.Error(t, err) + if err == nil { + assert.Failf(t, "failed to cancel pull", "err: %s", err.Error()) + } + }) + + // TODO: Repeat the following tests for only a specific ref + t.Run("should pull entire repo", func(t *testing.T) { + err := repo.Pull( + context.TODO(), + remoteName, + Flags(Mirror|TrustedHTTP), + ) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to pull repo", "err: %s", err.Error()) + } + }) + + t.Run("should list refs from original repo", func(t *testing.T) { + expectedChecksums := map[string]bool{} + test1Data, err := os.ReadFile("testdata/repo/refs/heads/test1") + test2Data, err := os.ReadFile("testdata/repo/refs/heads/test2") + if err != nil { + t.Errorf("failed to read refs file: %s", err.Error()) + } + + // Update in case of changes to testdata + expectedChecksums[strings.TrimRight(string(test1Data), "\n")] = false + expectedChecksums[strings.TrimRight(string(test2Data), "\n")] = false + + refs, err := repo.ListRefsExt(ListRefsExtFlagsNone) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to list refs", "err: %s", err.Error()) + } + assert.NotEmpty(t, refs) + + for _, ref := range refs { + checksum := ref.Checksum + assert.NotEmpty(t, checksum) + for sum := range expectedChecksums { + if sum == checksum { + expectedChecksums[sum] = true + } + } + } + + for sum, exists := range expectedChecksums { + assert.True(t, exists, "checksum %s not found", sum) + } + }) + + t.Run("should generate summary file", func(t *testing.T) { + err := repo.RegenerateSummary() + assert.NoError(t, err) + _, err = os.Stat(fmt.Sprintf("%s/summary", repoPath)) + assert.NoError(t, err) + if err != nil { + assert.Failf(t, "failed to stat summary file", "err: %s", err.Error()) + } + }) + }) + } +} diff --git a/internal/plugins/ostree/pkg/libostree/repo.go b/internal/plugins/ostree/pkg/libostree/repo.go new file mode 100644 index 0000000..0f4a291 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/repo.go @@ -0,0 +1,319 @@ +//nolint:goheader +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package libostree + +// #include +// #include +import "C" + +import ( + "runtime" + "unsafe" +) + +// RepoMode - The mode to use when creating a new repo. +// If an unknown mode is passed, RepoModeBare will be used silently. +// +// See https://ostreedev.github.io/ostree/formats/#the-archive-format +// See https://ostreedev.github.io/ostree/formats/#aside-bare-formats +type RepoMode string + +func (r RepoMode) toC() C.OstreeRepoMode { + switch r { + case RepoModeBare: + return C.OSTREE_REPO_MODE_BARE + case RepoModeArchive: + return C.OSTREE_REPO_MODE_ARCHIVE + case RepoModeArchiveZ2: + return C.OSTREE_REPO_MODE_ARCHIVE_Z2 + case RepoModeBareUser: + return C.OSTREE_REPO_MODE_BARE_USER + case RepoModeBareUserOnly: + return C.OSTREE_REPO_MODE_BARE_USER_ONLY + case RepoModeBareSplitXAttrs: + return C.OSTREE_REPO_MODE_BARE_SPLIT_XATTRS + default: + return C.OSTREE_REPO_MODE_BARE + } +} + +const ( + // RepoModeBare - The default mode. Keeps all file metadata. May require elevated privileges. + // The bare repository format is the simplest one. In this mode regular files are directly stored to disk, and all + // metadata (e.g. uid/gid and xattrs) is reflected to the filesystem. It allows further direct access to content and + // metadata, but it may require elevated privileges when writing objects to the repository. + RepoModeBare RepoMode = "bare" + + // RepoModeArchive - The archive format. Best for small storage footprint. Mostly used for server-side repositories. + // The archive format simply gzip-compresses each content object. Metadata objects are stored uncompressed. This + // means that it’s easy to serve via static HTTP. Note: the repo config file still uses the historical term + // archive-z2 as mode. But this essentially indicates the modern archive format. + // + // When you commit new content, you will see new .filez files appearing in `objects/`. + RepoModeArchive RepoMode = "archive" + + // RepoModeArchiveZ2 - Functionally equivalent to RepoModeArchive. Only useful for backwards compatibility. + RepoModeArchiveZ2 RepoMode = "archive-z2" + + // RepoModeBareUser - Like RepoModeBare but ignore incoming uid/gid and xattrs. + // The bare-user format is a bit special in that the uid/gid and xattrs from the content are ignored. This is + // primarily useful if you want to have the same OSTree-managed content that can be run on a host system or an + // unprivileged container. + RepoModeBareUser RepoMode = "bare-user" + + // RepoModeBareUserOnly - Like RepoModeBareUser. No metadata stored. Only useful for checkouts. Does not need xattrs. + // Same as BARE_USER, but all metadata is not stored, so it can only be used for user checkouts. Does not need xattrs. + RepoModeBareUserOnly RepoMode = "bare-user-only" + + // RepoModeBareSplitXAttrs - Like RepoModeBare but store xattrs in a separate file. + // Similarly, the bare-split-xattrs format is a special mode where xattrs are stored as separate repository objects, + // and not directly reflected to the filesystem. This is primarily useful when transporting xattrs through lossy + // environments (e.g. tar streams and containerized environments). It also allows carrying security-sensitive xattrs + // (e.g. SELinux labels) out-of-band without involving OS filesystem logic. + RepoModeBareSplitXAttrs RepoMode = "bare-split-xattrs" +) + +type Repo struct { + native *C.OstreeRepo +} + +func fromNative(cRepo *C.OstreeRepo) *Repo { + repo := &Repo{ + native: cRepo, + } + + // Let the GB trigger free the cRepo for us when repo is freed. + runtime.SetFinalizer(repo, func(r *Repo) { + C.free(unsafe.Pointer(r.native)) + }) + + return repo +} + +// Init initializes & opens a new ostree repository at the given path. +// +// Create the underlying structure on disk for the repository, and call +// ostree_repo_open() on the result, preparing it for use. +// +// Since version 2016.8, this function will succeed on an existing +// repository, and finish creating any necessary files in a partially +// created repository. However, this function cannot change the mode +// of an existing repository, and will silently ignore an attempt to +// do so. +// +// Since 2017.9, "existing repository" is defined by the existence of an +// `objects` subdirectory. +// +// This function predates ostree_repo_create_at(). It is an error to call +// this function on a repository initialized via ostree_repo_open_at(). +func Init(path string, mode RepoMode) (repo *Repo, err error) { + if path == "" { + return nil, ErrInvalidPath + } + + cPathStr := C.CString(path) + defer C.free(unsafe.Pointer(cPathStr)) + cPath := C.g_file_new_for_path(cPathStr) + defer C.g_object_unref(C.gpointer(cPath)) + + // Create a *C.OstreeRepo from the path + cRepo := C.ostree_repo_new(cPath) + defer func() { + if err != nil { + C.free(unsafe.Pointer(cRepo)) + } + }() + + var cErr *C.GError + + if r := C.ostree_repo_create(cRepo, mode.toC(), nil, &cErr); r == C.gboolean(0) { + return nil, GoError(cErr) + } + return fromNative(cRepo), nil +} + +// Open opens an ostree repository at the given path. +func Open(path string) (*Repo, error) { + if path == "" { + return nil, ErrInvalidPath + } + + cPathStr := C.CString(path) + defer C.free(unsafe.Pointer(cPathStr)) + cPath := C.g_file_new_for_path(cPathStr) + defer C.g_object_unref(C.gpointer(cPath)) + + // Create a *C.OstreeRepo from the path + cRepo := C.ostree_repo_new(cPath) + + var cErr *C.GError + + if r := C.ostree_repo_open(cRepo, nil, &cErr); r == C.gboolean(0) { + return nil, GoError(cErr) + } + + return fromNative(cRepo), nil +} + +// AddRemote adds a remote to the repository. +func (r *Repo) AddRemote(name, url string, opts ...Option) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + cURL := C.CString(url) + defer C.free(unsafe.Pointer(cURL)) + + options := toGVariant(opts...) + defer C.g_variant_unref(options) + + var cErr *C.GError + + /* + gboolean + ostree_repo_remote_add(OstreeRepo *self, + const char *name, + const char *url, + GVariant *options, + GCancellable *cancellable, + GError **error) + */ + if C.ostree_repo_remote_add( + r.native, + cName, + cURL, + options, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +// DeleteRemote deletes a remote from the repository. +func (r *Repo) DeleteRemote(name string) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + var cErr *C.GError + if C.ostree_repo_remote_delete( + r.native, + cName, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +// ReloadRemoteConfig reloads the remote configuration. +func (r *Repo) ReloadRemoteConfig() error { + var cErr *C.GError + + if C.ostree_repo_reload_config( + r.native, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} + +// ListRemotes lists the remotes in the repository. +func (r *Repo) ListRemotes() []string { + var n C.guint + remotes := C.ostree_repo_remote_list( + r.native, + &n, + ) + + var ret []string + for { + if *remotes == nil { + break + } + ret = append(ret, C.GoString(*remotes)) + remotes = (**C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(remotes)) + unsafe.Sizeof(uintptr(0)))) + } + + return ret +} + +type ListRefsExtFlags int + +const ( + ListRefsExtFlagsAliases = 1 << iota + ListRefsExtFlagsExcludeRemotes + ListRefsExtFlagsExcludeMirrors + ListRefsExtFlagsNone ListRefsExtFlags = 0 +) + +type Ref struct { + Name string + Checksum string +} + +func (r *Repo) ListRefsExt(flags ListRefsExtFlags, prefix ...string) ([]Ref, error) { + var cPrefix *C.char + if len(prefix) > 0 { + cPrefix = C.CString(prefix[0]) + defer C.free(unsafe.Pointer(cPrefix)) + } + + cFlags := (C.OstreeRepoListRefsExtFlags)(C.int(flags)) + + var cErr *C.GError + var outAllRefs *C.GHashTable + if C.ostree_repo_list_refs_ext( + r.native, + cPrefix, + &outAllRefs, + cFlags, + nil, + &cErr, + ) == C.gboolean(0) { + return nil, GoError(cErr) + } + + // iter is freed when g_hash_table_iter_next returns false + var iter C.GHashTableIter + C.g_hash_table_iter_init(&iter, outAllRefs) + + var cRef, cChecksum C.gpointer + var ret []Ref + for C.g_hash_table_iter_next(&iter, &cRef, &cChecksum) == C.gboolean(1) { + if cRef == nil { + break + } + + ref := (*C.OstreeCollectionRef)(unsafe.Pointer(&cRef)) + + ret = append(ret, Ref{ + Name: C.GoString(ref.ref_name), + Checksum: C.GoString((*C.char)(cChecksum)), + }) + } + + return ret, nil +} + +func (r *Repo) RegenerateSummary() error { + var cErr *C.GError + if C.ostree_repo_regenerate_summary( + r.native, + nil, + nil, + &cErr, + ) == C.gboolean(0) { + return GoError(cErr) + } + + return nil +} diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/.lock b/internal/plugins/ostree/pkg/libostree/testdata/repo/.lock new file mode 100644 index 0000000..e69de29 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/config b/internal/plugins/ostree/pkg/libostree/testdata/repo/config new file mode 100644 index 0000000..d289d74 --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/testdata/repo/config @@ -0,0 +1,4 @@ +[core] +repo_version=1 +mode=archive-z2 +indexed-deltas=true diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/05/b8c96e271b1d932571374a811918927eb06f8354d88a4b670e0588d10a628b.filez b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/05/b8c96e271b1d932571374a811918927eb06f8354d88a4b670e0588d10a628b.filez new file mode 100644 index 0000000..e95c4e9 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/05/b8c96e271b1d932571374a811918927eb06f8354d88a4b670e0588d10a628b.filez differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/07/10a0f5925abdefc9816329711011c5791a6ce28584127d68b4452321be431c.dirtree b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/07/10a0f5925abdefc9816329711011c5791a6ce28584127d68b4452321be431c.dirtree new file mode 100644 index 0000000..7f67df1 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/07/10a0f5925abdefc9816329711011c5791a6ce28584127d68b4452321be431c.dirtree differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/08/9d8abd9ca632baabbaf908c825eb709b260bb8c688c87890f34b13b96565c2.dirtree b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/08/9d8abd9ca632baabbaf908c825eb709b260bb8c688c87890f34b13b96565c2.dirtree new file mode 100644 index 0000000..365ec28 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/08/9d8abd9ca632baabbaf908c825eb709b260bb8c688c87890f34b13b96565c2.dirtree differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/0e/1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22.commit b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/0e/1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22.commit new file mode 100644 index 0000000..93b2837 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/0e/1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22.commit differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/43/f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41.commit b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/43/f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41.commit new file mode 100644 index 0000000..6000091 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/43/f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41.commit differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/6b/1f7b40b1be1309032c26d50438bac61853554c724cbd7c5de2bbfb224e9779.dirmeta b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/6b/1f7b40b1be1309032c26d50438bac61853554c724cbd7c5de2bbfb224e9779.dirmeta new file mode 100644 index 0000000..162d631 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/6b/1f7b40b1be1309032c26d50438bac61853554c724cbd7c5de2bbfb224e9779.dirmeta differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/93/7a454371f7f367ba8d168932d593549b625507b8e2350b24b0e172d90e4000.filez b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/93/7a454371f7f367ba8d168932d593549b625507b8e2350b24b0e172d90e4000.filez new file mode 100644 index 0000000..cdb2348 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/93/7a454371f7f367ba8d168932d593549b625507b8e2350b24b0e172d90e4000.filez differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/aa/659a25dc52d863e4017c35110e95391dd4a31217025a8fb0acebf7ea8db88b.commit b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/aa/659a25dc52d863e4017c35110e95391dd4a31217025a8fb0acebf7ea8db88b.commit new file mode 100644 index 0000000..5b8f996 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/aa/659a25dc52d863e4017c35110e95391dd4a31217025a8fb0acebf7ea8db88b.commit differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/e1/95f272f46c434d26c5c7499e38faf21919fdfdeceb09bf64c862a6abfbca46.dirtree b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/e1/95f272f46c434d26c5c7499e38faf21919fdfdeceb09bf64c862a6abfbca46.dirtree new file mode 100644 index 0000000..08ef717 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/e1/95f272f46c434d26c5c7499e38faf21919fdfdeceb09bf64c862a6abfbca46.dirtree differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/ee/3d4763e0ff1f327e102da7e2ca268041c2abe2e73fdb76aea896bc74b5a638.filez b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/ee/3d4763e0ff1f327e102da7e2ca268041c2abe2e73fdb76aea896bc74b5a638.filez new file mode 100644 index 0000000..2443490 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/objects/ee/3d4763e0ff1f327e102da7e2ca268041c2abe2e73fdb76aea896bc74b5a638.filez differ diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test1 b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test1 new file mode 100644 index 0000000..7fc311c --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test1 @@ -0,0 +1 @@ +0e1518ee5f0421ad34685958b662c4947ea18a96be03b406bc8eb9ccf913ff22 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test2 b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test2 new file mode 100644 index 0000000..d4b9ffd --- /dev/null +++ b/internal/plugins/ostree/pkg/libostree/testdata/repo/refs/heads/test2 @@ -0,0 +1 @@ +43f7cd809a7b783da5bb1769b099c79e795ac1d667aed5b9121a0c75f9b38e41 diff --git a/internal/plugins/ostree/pkg/libostree/testdata/repo/summary b/internal/plugins/ostree/pkg/libostree/testdata/repo/summary new file mode 100644 index 0000000..b464f03 Binary files /dev/null and b/internal/plugins/ostree/pkg/libostree/testdata/repo/summary differ diff --git a/internal/plugins/ostree/pkg/ostreerepository/api.go b/internal/plugins/ostree/pkg/ostreerepository/api.go new file mode 100644 index 0000000..d25522d --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/api.go @@ -0,0 +1,227 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostreerepository + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" + "go.ciq.dev/beskar/pkg/orasostree" + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" + "go.ciq.dev/beskar/pkg/utils" + "golang.org/x/sync/errgroup" +) + +func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.OSTreeRepositoryProperties) (err error) { + h.logger.Debug("creating repository", "repository", h.Repository) + // Validate request + if len(properties.Remotes) == 0 { + return ctl.Errf("at least one remote is required") + } + + // Check if repo already exists + if h.checkRepoExists(ctx) { + return ctl.Err("repository already exists") + } + + // Transition to provisioning state + if err := h.setState(StateProvisioning); err != nil { + return err + } + defer h.clearState() + + return h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { + // Add user provided remotes + // We do not need to add beskar remote here + for _, remote := range properties.Remotes { + var opts []libostree.Option + if remote.NoGPGVerify { + opts = append(opts, libostree.NoGPGVerify()) + } + if err := repo.AddRemote(remote.Name, remote.RemoteURL, opts...); err != nil { + return false, ctl.Errf("adding remote to ostree repository %s: %s", remote.Name, err) + } + } + + if err := repo.RegenerateSummary(); err != nil { + return false, ctl.Errf("regenerating summary for ostree repository %s: %s", h.repoDir, err) + } + + return true, nil + }, SkipPull()) +} + +// DeleteRepository deletes the repository from beskar and the local filesystem. +// +// This could lead to an invalid _state if the repository fails to completely deleting from beskar. +func (h *Handler) DeleteRepository(ctx context.Context) (err error) { + // Transition to deleting state + if err := h.setState(StateDeleting); err != nil { + return err + } + + // Check if repo already exists + if !h.checkRepoExists(ctx) { + defer h.clearState() + return ctl.Err("repository does not exist") + } + + go func() { + defer func() { + if err == nil { + // stop the repo handler and trigger cleanup + h.Stop() + } + h.clearState() + }() + h.logger.Debug("deleting repository") + + err := h.BeginLocalRepoTransaction(context.Background(), func(ctx context.Context, repo *libostree.Repo) (bool, error) { + // Create a worker pool to deleting each file in the repository concurrently. + // ctx will be cancelled on error, and the error will be returned. + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(100) + + // Walk the directory tree, skipping directories and deleting each file. + if err := filepath.WalkDir(h.repoDir, func(path string, d os.DirEntry, err error) error { + // If there was an error with the file, return it. + if err != nil { + return fmt.Errorf("walking %s: %w", path, err) + } + // Skip directories. + if d.IsDir() { + return nil + } + // Skip the rest of the files if the context has been cancelled. + if ctx.Err() != nil { + // Skip remaining files because our context has been cancelled. + // We could return the error here, but we want to exclusively handle that error in our call to eg.Wait(). + // This is because we would never be able to handle an error returned from the last job. + return filepath.SkipAll + } + // Schedule deletion to run in a worker. + eg.Go(func() error { + // Delete the file from the repository + filename := filepath.Base(path) + h.logger.Debug("deleting file from beskar", "file", filename) + digest := orasostree.MakeTag(filename) + digestRef := filepath.Join(h.Repository, "file:"+digest) + if err := h.DeleteManifest(digestRef); err != nil { + h.logger.Error("deleting file from beskar", "error", err.Error()) + } + + return nil + }) + + return nil + }); err != nil { + return false, err + } + + // We don't want to push any changes to beskar. + return false, eg.Wait() + }) + if err != nil { + h.logger.Error("deleting repository", "error", err.Error()) + } + }() + + return nil +} + +func (h *Handler) AddRemote(ctx context.Context, remote *apiv1.OSTreeRemoteProperties) (err error) { + // Transition to provisioning state + if err := h.setState(StateProvisioning); err != nil { + return err + } + defer h.clearState() + + if !h.checkRepoExists(ctx) { + return ctl.Errf("repository does not exist") + } + + return h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { + // Add user provided remote + var opts []libostree.Option + if remote.NoGPGVerify { + opts = append(opts, libostree.NoGPGVerify()) + } + if err := repo.AddRemote(remote.Name, remote.RemoteURL, opts...); err != nil { + // No need to make error pretty, it is already pretty + return false, err + } + + return true, nil + }, SkipPull()) +} + +func (h *Handler) SyncRepository(_ context.Context, properties *apiv1.OSTreeRepositorySyncRequest) (err error) { + // Transition to syncing state + if err := h.setState(StateSyncing); err != nil { + return err + } + + // Spin up pull worker + go func() { + h.logger.Debug("syncing repository") + + var err error + defer func() { + if err != nil { + h.logger.Error("repository sync failed", "properties", properties, "error", err.Error()) + repoSync := *h.repoSync.Load() + repoSync.SyncError = err.Error() + h.setRepoSync(&repoSync) + } else { + h.logger.Debug("repository sync complete", "properties", properties) + } + h.clearState() + }() + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + err = h.BeginLocalRepoTransaction(ctx, func(ctx context.Context, repo *libostree.Repo) (bool, error) { + // Pull the latest changes from the remote. + opts := []libostree.Option{ + libostree.Depth(properties.Depth), + libostree.Flags(libostree.Mirror | libostree.TrustedHTTP), + } + if len(properties.Refs) > 0 { + opts = append(opts, libostree.Refs(properties.Refs...)) + } + + // pull remote content into local repo + if err := repo.Pull(ctx, properties.Remote, opts...); err != nil { + return false, ctl.Errf("pulling ostree repository: %s", err) + } + + if err := repo.RegenerateSummary(); err != nil { + return false, ctl.Errf("regenerating summary for ostree repository %s: %s", h.repoDir, err) + } + + return true, nil + }) + }() + + return nil +} + +func (h *Handler) GetRepositorySyncStatus(_ context.Context) (syncStatus *apiv1.SyncStatus, err error) { + repoSync := h.repoSync.Load() + if repoSync == nil { + return nil, ctl.Errf("repository sync status not available") + } + return &apiv1.SyncStatus{ + Syncing: repoSync.Syncing, + StartTime: utils.TimeToString(repoSync.StartTime), + EndTime: utils.TimeToString(repoSync.EndTime), + SyncError: repoSync.SyncError, + }, nil +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/handler.go b/internal/plugins/ostree/pkg/ostreerepository/handler.go new file mode 100644 index 0000000..59d0e47 --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/handler.go @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostreerepository + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path" + "path/filepath" + "sync" + "sync/atomic" + + "github.com/RussellLuo/kun/pkg/werror" + "github.com/RussellLuo/kun/pkg/werror/gcode" + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/internal/pkg/repository" + eventv1 "go.ciq.dev/beskar/pkg/api/event/v1" +) + +const ( + beskarRemoteName = "_beskar_" +) + +type State int32 + +const ( + // StateStopped - The repository _state is unknown. + StateStopped State = iota + // StateReady - The repository is ready. + StateReady + // StateProvisioning - The repository is being provisioned. + StateProvisioning + // StateSyncing - The repository is being synced. + StateSyncing + // StateDeleting - The repository is being deleted. + StateDeleting +) + +func (s State) String() string { + switch s { + case StateStopped: + return "stopped" + case StateReady: + return "ready" + case StateProvisioning: + return "provisioning" + case StateSyncing: + return "syncing" + case StateDeleting: + return "deleting" + default: + return "unknown" + } +} + +type Handler struct { + *repository.RepoHandler + logger *slog.Logger + repoDir string + repoLock sync.RWMutex + repoSync atomic.Pointer[RepoSync] + + _state atomic.Int32 +} + +func NewHandler(logger *slog.Logger, repoHandler *repository.RepoHandler) *Handler { + return &Handler{ + RepoHandler: repoHandler, + repoDir: filepath.Join(repoHandler.Params.Dir, repoHandler.Repository), + logger: logger, + } +} + +func (h *Handler) setState(state State) error { + current := h.getState() + if current != StateReady && state != StateReady { + return werror.Wrap(gcode.ErrUnavailable, fmt.Errorf("repository is busy: %s", current)) + } + h._state.Swap(int32(state)) + if state == StateSyncing || current == StateSyncing { + _ = h.updateSyncing(state == StateSyncing) + } + return nil +} + +func (h *Handler) clearState() { + h._state.Swap(int32(StateReady)) + _ = h.updateSyncing(false) +} + +func (h *Handler) getState() State { + if !h.Started() { + return StateStopped + } + return State(h._state.Load()) +} + +func (h *Handler) cleanup() { + h.logger.Debug("repository cleanup", "repository", h.Repository) + h.repoLock.Lock() + defer h.repoLock.Unlock() + + close(h.Queued) + h.Params.Remove(h.Repository) + _ = os.RemoveAll(h.repoDir) +} + +func (h *Handler) QueueEvent(_ *eventv1.EventPayload, _ bool) error { + return nil +} + +func (h *Handler) Start(ctx context.Context) { + h.logger.Debug("starting repository", "repository", h.Repository) + h.clearState() + + go func() { + for !h.Stopped.Load() { + //nolint: gosimple + select { + case <-ctx.Done(): + h.Stopped.Store(true) + } + } + h.cleanup() + }() +} + +// pullConfig pulls the config file from beskar. +func (h *Handler) pullFile(ctx context.Context, filename string) error { + // TODO: Replace with appropriate puller mechanism + url := "http://" + h.Params.GetBeskarRegistryHostPort() + path.Join("/", h.Repository, "repo", filename) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Check Content-Length + if resp.ContentLength <= 0 { + return ctl.Errf("content-length is 0") + } + + // Create the file + out, err := os.Create(path.Join(h.repoDir, filename)) + if err != nil { + return err + } + + // Write the body to file + _, err = io.Copy(out, resp.Body) + return err +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/local.go b/internal/plugins/ostree/pkg/ostreerepository/local.go new file mode 100644 index 0000000..c8f4eff --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/local.go @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostreerepository + +import ( + "context" + "os" + "path" + "path/filepath" + + "go.ciq.dev/beskar/cmd/beskarctl/ctl" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/libostree" + "go.ciq.dev/beskar/pkg/orasostree" +) + +// checkRepoExists checks if the ostree repository exists in beskar. +func (h *Handler) checkRepoExists(_ context.Context) bool { + // Check if repo already exists + configTag := orasostree.MakeTag(orasostree.FileConfig) + configRef := filepath.Join(h.Repository, "file:"+configTag) + _, err := h.GetManifestDigest(configRef) + return err == nil +} + +type ( + TransactionFn func(ctx context.Context, repo *libostree.Repo) (commit bool, err error) + TransactionOptions struct { + skipPull bool + } +) + +type TransactionOption func(*TransactionOptions) + +func SkipPull() TransactionOption { + return func(opts *TransactionOptions) { + opts.skipPull = true + } +} + +// BeginLocalRepoTransaction executes a transaction against the local ostree repository. +// The transaction is executed in a temporary directory in which the following steps are performed: +// 1. The local ostree repository is opened. +// 2. The beskar remote is added to the local ostree repository. +// 3. The beskar version of the repo is pulled into the local ostree repository. +// 4. The transactorFn is executed. +// 5. If the transactorFn returns true, the local ostree repository is pushed to beskar. If false, all local changes are discarded. +// 6. The temporary directory is removed. +func (h *Handler) BeginLocalRepoTransaction(ctx context.Context, tFn TransactionFn, opts ...TransactionOption) error { + options := TransactionOptions{} + for _, opt := range opts { + opt(&options) + } + + // We control the local repo lifecycle here, so we need to lock it. + h.repoLock.Lock() + defer h.repoLock.Unlock() + + // Open the local repo + // Create the repository directory + if err := os.MkdirAll(h.repoDir, 0o700); err != nil { + // If the directory already exists, we can continue + if !os.IsExist(err) { + return ctl.Errf("create repository dir: %s", err) + } + } + + // Clean up the disk when we are done + defer func() { + if err := os.RemoveAll(h.repoDir); err != nil { + h.logger.Error("removing local repo", "repo", h.repoDir, "error", err) + } + }() + + // We will always use archive mode here + // Note that we are not using the returned repo pointer here. We will re-open the repo later. + _, err := libostree.Init(h.repoDir, libostree.RepoModeArchive) + if err != nil { + return ctl.Errf("initializing ostree repository %s: %s", h.repoDir, err) + } + + // It is necessary to pull the config from beskar before we can add the beskar remote. This is because config files + // are unique the instance of a repo you are interacting with. Meaning, remotes are not pulled with the repo's data. + if err := h.pullFile(ctx, orasostree.FileConfig); err != nil { + h.logger.Debug("no config found in beskar", "error", err) + } + + // Re-open the local repo + // We need to re-open the repo here because we just pulled the config from beskar. If we don't re-open the repo, the + // config we just manually pulled down will not be loaded into memory. + repo, err := libostree.Open(h.repoDir) + if err != nil { + return ctl.Errf("opening ostree repository %s: %s", h.repoDir, err) + } + + // Add beskar as a remote so that we can pull from it + beskarServiceURL := "http://" + h.Params.GetBeskarRegistryHostPort() + path.Join("/", h.Repository, "repo") + if err := repo.AddRemote(beskarRemoteName, beskarServiceURL, libostree.NoGPGVerify()); err != nil { + return ctl.Errf("adding remote to ostree repository %s: %s", beskarRemoteName, err) + } + + // pull remote content into local repo from beskar + if !options.skipPull && h.checkRepoExists(ctx) { + if err := repo.Pull( + ctx, + beskarRemoteName, + libostree.NoGPGVerify(), + libostree.Flags(libostree.Mirror|libostree.TrustedHTTP), + ); err != nil { + return ctl.Errf("pulling ostree repository from %s: %s", beskarRemoteName, err) + } + } + + // Execute the transaction + commit, err := tFn(ctx, repo) + if err != nil { + return ctl.Errf("executing transaction: %s", err) + } + + // Commit the changes to beskar if the transaction deems it necessary + if commit { + // Remove the internal beskar remote so that external clients can't pull from it, not that it would work. + if err := repo.DeleteRemote(beskarRemoteName); err != nil { + return ctl.Errf("deleting remote %s: %s", beskarRemoteName, err) + } + + // Close the local + // Push local repo to beskar using OSTreePusher + repoPusher := orasostree.NewOSTreeRepositoryPusher( + ctx, + h.repoDir, + h.Repository, + 100, + ) + repoPusher = repoPusher.WithLogger(h.logger) + repoPusher = repoPusher.WithNameOptions(h.Params.NameOptions...) + repoPusher = repoPusher.WithRemoteOptions(h.Params.RemoteOptions...) + if err := repoPusher.Push(); err != nil { + return ctl.Errf("pushing ostree repository: %s", err) + } + } + + return nil +} diff --git a/internal/plugins/ostree/pkg/ostreerepository/sync.go b/internal/plugins/ostree/pkg/ostreerepository/sync.go new file mode 100644 index 0000000..74f6cf9 --- /dev/null +++ b/internal/plugins/ostree/pkg/ostreerepository/sync.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostreerepository + +import ( + "time" +) + +type RepoSync struct { + Syncing bool + StartTime int64 + EndTime int64 + SyncError string +} + +func (h *Handler) setRepoSync(repoSync *RepoSync) { + rs := *repoSync + h.repoSync.Store(&rs) +} + +//nolint:unparam +func (h *Handler) updateSyncing(syncing bool) *RepoSync { + if h.repoSync.Load() == nil { + h.repoSync.Store(&RepoSync{}) + } + + repoSync := *h.repoSync.Load() + previousSyncing := repoSync.Syncing + repoSync.Syncing = syncing + if syncing && !previousSyncing { + repoSync.StartTime = time.Now().UTC().Unix() + repoSync.SyncError = "" + } else if !syncing && previousSyncing { + repoSync.EndTime = time.Now().UTC().Unix() + } + h.repoSync.Store(&repoSync) + return h.repoSync.Load() +} diff --git a/internal/plugins/ostree/plugin.go b/internal/plugins/ostree/plugin.go new file mode 100644 index 0000000..398730d --- /dev/null +++ b/internal/plugins/ostree/plugin.go @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package ostree + +import ( + "context" + _ "embed" + "net" + "net/http" + "net/http/pprof" + "path/filepath" + "strconv" + + "github.com/RussellLuo/kun/pkg/httpcodec" + "github.com/go-chi/chi" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "go.ciq.dev/beskar/internal/pkg/gossip" + "go.ciq.dev/beskar/internal/pkg/log" + "go.ciq.dev/beskar/internal/pkg/pluginsrv" + "go.ciq.dev/beskar/internal/pkg/repository" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/config" + "go.ciq.dev/beskar/internal/plugins/ostree/pkg/ostreerepository" + pluginv1 "go.ciq.dev/beskar/pkg/api/plugin/v1" + "go.ciq.dev/beskar/pkg/mtls" + apiv1 "go.ciq.dev/beskar/pkg/plugins/ostree/api/v1" + "go.ciq.dev/beskar/pkg/version" +) + +const ( + PluginName = "ostree" + PluginAPIPathPattern = "/artifacts/ostree/api/v1" +) + +//go:embed embedded/router.rego +var routerRego []byte + +//go:embed embedded/data.json +var routerData []byte + +type Plugin struct { + ctx context.Context + config pluginsrv.Config + + repositoryManager *repository.Manager[*ostreerepository.Handler] + handlerParams *repository.HandlerParams +} + +func New(ctx context.Context, beskarOSTreeConfig *config.BeskarOSTreeConfig) (*Plugin, error) { + logger, err := beskarOSTreeConfig.Log.Logger(log.ContextHandler) + if err != nil { + return nil, err + } + ctx = log.SetContextLogger(ctx, logger) + + router := chi.NewRouter() + + // for kubernetes probes + router.Handle("/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + + if beskarOSTreeConfig.Profiling { + router.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) + router.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + router.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + router.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + router.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) + router.Handle("/debug/pprof/{cmd}", http.HandlerFunc(pprof.Index)) // special handling for Gorilla mux + } + + params := &repository.HandlerParams{ + Dir: filepath.Join(beskarOSTreeConfig.DataDir, "_repohandlers_"), + } + + return &Plugin{ + ctx: ctx, + config: pluginsrv.Config{ + Router: router, + Gossip: beskarOSTreeConfig.Gossip, + Info: &pluginv1.Info{ + Name: PluginName, + // Not registering media types so that Beskar doesn't send events. + Mediatypes: []string{}, + Version: version.Semver, + Router: &pluginv1.Router{ + Rego: routerRego, + Data: routerData, + }, + }, + }, + handlerParams: params, + repositoryManager: repository.NewManager[*ostreerepository.Handler]( + params, + ostreerepository.NewHandler, + ), + }, nil +} + +func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *gossip.BeskarMeta) error { + // Collection beskar http service endpoint for later pulls + p.handlerParams.BeskarMeta = beskarMeta + + hostport := net.JoinHostPort(beskarMeta.Hostname, strconv.Itoa(int(beskarMeta.RegistryPort))) + p.handlerParams.NameOptions = []name.Option{ + name.WithDefaultRegistry(hostport), + } + p.handlerParams.RemoteOptions = []remote.Option{ + remote.WithTransport(transport), + } + + p.config.Router.Route( + PluginAPIPathPattern, + func(r chi.Router) { + r.Use(pluginsrv.IsTLSMiddleware) + r.Mount("/", apiv1.NewHTTPRouter( + p, + httpcodec.NewDefaultCodecs(nil), + )) + }, + ) + + return nil +} + +func (p *Plugin) Context() context.Context { + return p.ctx +} + +func (p *Plugin) Config() pluginsrv.Config { + return p.config +} + +func (p *Plugin) RepositoryManager() *repository.Manager[*ostreerepository.Handler] { + return p.repositoryManager +} diff --git a/internal/plugins/static/pkg/staticrepository/api.go b/internal/plugins/static/pkg/staticrepository/api.go index 6a3eea9..c8acd05 100644 --- a/internal/plugins/static/pkg/staticrepository/api.go +++ b/internal/plugins/static/pkg/staticrepository/api.go @@ -10,6 +10,8 @@ import ( "path/filepath" "time" + "go.ciq.dev/beskar/pkg/utils" + "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" "github.com/hashicorp/go-multierror" @@ -90,7 +92,7 @@ func (h *Handler) ListRepositoryLogs(ctx context.Context, _ *apiv1.Page) (logs [ logs = append(logs, apiv1.RepositoryLog{ Level: log.Level, Message: log.Message, - Date: timeToString(log.Date), + Date: utils.TimeToString(log.Date), }) return nil }) @@ -221,16 +223,7 @@ func toRepositoryFileAPI(pkg *staticdb.RepositoryFile) *apiv1.RepositoryFile { Tag: pkg.Tag, ID: pkg.ID, Name: pkg.Name, - UploadTime: timeToString(pkg.UploadTime), + UploadTime: utils.TimeToString(pkg.UploadTime), Size: pkg.Size, } } - -const timeFormat = time.DateTime + " MST" - -func timeToString(t int64) string { - if t == 0 { - return "" - } - return time.Unix(t, 0).Format(timeFormat) -} diff --git a/internal/plugins/static/plugin.go b/internal/plugins/static/plugin.go index 23a1983..435d534 100644 --- a/internal/plugins/static/plugin.go +++ b/internal/plugins/static/plugin.go @@ -124,7 +124,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/static/api/v1", func(r chi.Router) { - r.Use(p.apiMiddleware) + r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -146,12 +146,3 @@ func (p *Plugin) Context() context.Context { func (p *Plugin) RepositoryManager() *repository.Manager[*staticrepository.Handler] { return p.repositoryManager } - -func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !pluginsrv.IsTLS(w, r) { - return - } - next.ServeHTTP(w, r) - }) -} diff --git a/internal/plugins/yum/pkg/yumrepository/api.go b/internal/plugins/yum/pkg/yumrepository/api.go index 6192028..d10a9b9 100644 --- a/internal/plugins/yum/pkg/yumrepository/api.go +++ b/internal/plugins/yum/pkg/yumrepository/api.go @@ -12,6 +12,8 @@ import ( "path/filepath" "time" + "go.ciq.dev/beskar/pkg/utils" + "github.com/RussellLuo/kun/pkg/werror" "github.com/RussellLuo/kun/pkg/werror/gcode" "github.com/google/go-containerregistry/pkg/v1/remote/transport" @@ -24,8 +26,6 @@ import ( var dbCtx = context.Background() -const timeFormat = time.DateTime + " MST" - func (h *Handler) CreateRepository(ctx context.Context, properties *apiv1.RepositoryProperties) (err error) { if !h.Started() { return werror.Wrap(gcode.ErrUnavailable, err) @@ -338,8 +338,8 @@ func (h *Handler) GetRepositorySyncStatus(context.Context) (syncStatus *apiv1.Sy reposync := h.getReposync() return &apiv1.SyncStatus{ Syncing: reposync.Syncing, - StartTime: timeToString(reposync.StartTime), - EndTime: timeToString(reposync.EndTime), + StartTime: utils.TimeToString(reposync.StartTime), + EndTime: utils.TimeToString(reposync.EndTime), TotalPackages: reposync.TotalPackages, SyncedPackages: reposync.SyncedPackages, SyncError: reposync.SyncError, @@ -361,7 +361,7 @@ func (h *Handler) ListRepositoryLogs(ctx context.Context, _ *apiv1.Page) (logs [ logs = append(logs, apiv1.RepositoryLog{ Level: log.Level, Message: log.Message, - Date: timeToString(log.Date), + Date: utils.TimeToString(log.Date), }) return nil }) @@ -563,8 +563,8 @@ func toRepositoryPackageAPI(pkg *yumdb.RepositoryPackage) *apiv1.RepositoryPacka Tag: pkg.Tag, ID: pkg.ID, Name: pkg.Name, - UploadTime: timeToString(pkg.UploadTime), - BuildTime: timeToString(pkg.BuildTime), + UploadTime: utils.TimeToString(pkg.UploadTime), + BuildTime: utils.TimeToString(pkg.BuildTime), Size: pkg.Size, Architecture: pkg.Architecture, SourceRPM: pkg.SourceRPM, @@ -579,10 +579,3 @@ func toRepositoryPackageAPI(pkg *yumdb.RepositoryPackage) *apiv1.RepositoryPacka GPGSignature: pkg.GPGSignature, } } - -func timeToString(t int64) string { - if t == 0 { - return "" - } - return time.Unix(t, 0).Format(timeFormat) -} diff --git a/internal/plugins/yum/plugin.go b/internal/plugins/yum/plugin.go index 5f3f61d..4ce7b50 100644 --- a/internal/plugins/yum/plugin.go +++ b/internal/plugins/yum/plugin.go @@ -127,7 +127,7 @@ func (p *Plugin) Start(transport http.RoundTripper, _ *mtls.CAPEM, beskarMeta *g p.config.Router.Route( "/artifacts/yum/api/v1", func(r chi.Router) { - r.Use(p.apiMiddleware) + r.Use(pluginsrv.IsTLSMiddleware) r.Mount("/", apiv1.NewHTTPRouter( p, httpcodec.NewDefaultCodecs(nil), @@ -149,12 +149,3 @@ func (p *Plugin) Context() context.Context { func (p *Plugin) RepositoryManager() *repository.Manager[*yumrepository.Handler] { return p.repositoryManager } - -func (p *Plugin) apiMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !pluginsrv.IsTLS(w, r) { - return - } - next.ServeHTTP(w, r) - }) -} diff --git a/pkg/orasostree/ostree.go b/pkg/orasostree/ostree.go new file mode 100644 index 0000000..a41f5c3 --- /dev/null +++ b/pkg/orasostree/ostree.go @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package orasostree + +import ( + "crypto/md5" //nolint:gosec + "fmt" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "go.ciq.dev/beskar/pkg/oras" +) + +const ( + ArtifactsPathPrefix = "artifacts" + OSTreePathPrefix = "ostree" + + OSTreeConfigType = "application/vnd.ciq.ostree.file.v1.config+json" + OSTreeLayerType = "application/vnd.ciq.ostree.v1.file" + + FileSummary = "summary" + FileSummarySig = "summary.sig" + FileConfig = "config" +) + +func NewOSTreeFilePusher(repoRootDir, path, repo string, opts ...name.Option) (oras.Pusher, error) { + if !strings.HasPrefix(repo, ArtifactsPathPrefix+"/") { + if !strings.HasPrefix(repo, OSTreePathPrefix+"/") { + repo = filepath.Join(OSTreePathPrefix, repo) + } + + repo = filepath.Join(ArtifactsPathPrefix, repo) + } + + // Sanitize the path to match the format of the tag. See internal/plugins/ostree/embedded/data.json. + // In this case the file path needs to be relative to the repository root and not contain a leading slash. + path = strings.TrimPrefix(path, repoRootDir) + path = strings.TrimPrefix(path, "/") + + fileTag := MakeTag(path) + rawRef := filepath.Join(repo, "file:"+fileTag) + ref, err := name.ParseReference(rawRef, opts...) + if err != nil { + return nil, fmt.Errorf("while parsing reference %s: %w", rawRef, err) + } + + absolutePath := filepath.Join(repoRootDir, path) + + return oras.NewGenericPusher( + ref, + oras.NewManifestConfig(OSTreeConfigType, nil), + oras.NewLocalFileLayer(absolutePath, oras.WithLayerMediaType(OSTreeLayerType)), + ), nil +} + +// specialTags are tags that are not md5 hashes of the filename. +// These files are meant to stand out in the registry. +// Note: Values are not limited to the repo's root directory, but at the moment on the following have been identified. +var specialTags = []string{ + FileSummary, + FileSummarySig, + FileConfig, +} + +// MakeTag creates a tag for a file. +// If the filename starts with a special tag, the tag is returned as-is. +// Otherwise, the tag is the md5 hash of the filename. +func MakeTag(filename string) string { + for _, tag := range specialTags { + if filename == tag { + return tag + } + } + + //nolint:gosec + return fmt.Sprintf("%x", md5.Sum([]byte(filename))) +} diff --git a/pkg/orasostree/push.go b/pkg/orasostree/push.go new file mode 100644 index 0000000..e06c21b --- /dev/null +++ b/pkg/orasostree/push.go @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package orasostree + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "go.ciq.dev/beskar/pkg/oras" + "golang.org/x/sync/errgroup" +) + +type OSTreeRepositoryPusher struct { + ctx context.Context + dir string + repo string + jobCount int + nameOpts []name.Option + remoteOpts []remote.Option + logger *slog.Logger +} + +func NewOSTreeRepositoryPusher(ctx context.Context, dir, repo string, jobCount int) *OSTreeRepositoryPusher { + return &OSTreeRepositoryPusher{ + ctx: ctx, + dir: dir, + repo: repo, + jobCount: jobCount, + } +} + +func (p *OSTreeRepositoryPusher) WithNameOptions(opts ...name.Option) *OSTreeRepositoryPusher { + p.nameOpts = opts + return p +} + +func (p *OSTreeRepositoryPusher) WithRemoteOptions(opts ...remote.Option) *OSTreeRepositoryPusher { + p.remoteOpts = opts + return p +} + +func (p *OSTreeRepositoryPusher) WithLogger(logger *slog.Logger) *OSTreeRepositoryPusher { + p.logger = logger + return p +} + +// Push walks a local ostree repository and pushes each file to the given registry. +// dir is the root directory of the ostree repository, i.e., the directory containing the summary file. +// repo is the name of the ostree repository. +// registry is the registry to push to. +func (p *OSTreeRepositoryPusher) Push() error { + // Prove that we were given the root directory of an ostree repository + // by checking for the existence of the config file. + // Typically, libostree will check for the "objects" directory, but this will do just the same. + fileInfo, err := os.Stat(filepath.Join(p.dir, FileConfig)) + if os.IsNotExist(err) || fileInfo.IsDir() { + return fmt.Errorf("%s file not found in %s: you may need to call ostree init", FileConfig, p.dir) + } else if err != nil { + return fmt.Errorf("error accessing %s in %s: %w", FileConfig, p.dir, err) + } + + // Create a worker pool to push each file in the repository concurrently. + // ctx will be cancelled on error, and the error will be returned. + eg, ctx := errgroup.WithContext(p.ctx) + eg.SetLimit(p.jobCount) + + // Walk the directory tree, skipping directories and pushing each file. + if err := filepath.WalkDir(p.dir, func(path string, d os.DirEntry, err error) error { + // If there was an error with the file, return it. + if err != nil { + return fmt.Errorf("while walking %s: %w", path, err) + } + + // Skip directories. + if d.IsDir() { + return nil + } + + if ctx.Err() != nil { + // Skip remaining files because our context has been cancelled. + // We could return the error here, but we want to exclusively handle that error in our call to eg.Wait(). + // This is because we would never be able to handle an error returned from the last job. + return filepath.SkipAll + } + + eg.Go(func() error { + if err := p.push(path); err != nil { + return fmt.Errorf("while pushing %s: %w", path, err) + } + return nil + }) + + return nil + }); err != nil { + // We should only receive here if filepath.WalkDir() returns an error. + // Push errors are handled below. + return fmt.Errorf("while walking %s: %w", p.dir, err) + } + + // Wait for all workers to finish. + // If any worker returns an error, eg.Wait() will return that error. + return eg.Wait() +} + +func (p *OSTreeRepositoryPusher) push(path string) error { + pusher, err := NewOSTreeFilePusher(p.dir, path, p.repo, p.nameOpts...) + if err != nil { + return fmt.Errorf("while creating OSTree pusher: %w", err) + } + + if p.logger != nil { + path = strings.TrimPrefix(path, p.dir) + path = strings.TrimPrefix(path, "/") + p.logger.Debug("pushing file to beskar", "file", path, "reference", pusher.Reference()) + } + + return oras.Push(pusher, p.remoteOpts...) +} diff --git a/pkg/plugins/ostree/api/v1/api.go b/pkg/plugins/ostree/api/v1/api.go new file mode 100644 index 0000000..46ab6c8 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/api.go @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package apiv1 + +import ( + "context" + "regexp" +) + +const ( + RepositoryRegex = "^(artifacts/ostree/[a-z0-9]+(?:[/._-][a-z0-9]+)*)$" + URLPath = "/artifacts/ostree/api/v1" +) + +var repositoryMatcher = regexp.MustCompile(RepositoryRegex) + +func RepositoryMatch(repository string) bool { + return repositoryMatcher.MatchString(repository) +} + +type Page struct { + Size int + Token string +} + +type OSTreeRepositoryProperties struct { + // Remotes - The remote repositories to mirror. + Remotes []OSTreeRemoteProperties `json:"remotes"` +} + +type OSTreeRemoteProperties struct { + // Name - The name of the remote repository. + Name string `json:"name"` + + // RemoteURL - The http url of the remote repository. + RemoteURL string `json:"remote_url"` + + // GPGVerify - Whether to verify the GPG signature of the repository. + NoGPGVerify bool `json:"no_gpg_verify"` +} + +type OSTreeRepositorySyncRequest struct { + // Remote - The name of the remote to sync. + Remote string `json:"remote"` + + // Refs - The branches/refs to mirror. Leave empty to mirror all branches/refs. + Refs []string `json:"refs"` + + // Depth - The depth of the mirror. Defaults is 0, -1 means infinite. + Depth int `json:"depth"` +} + +// Mirror sync status. +type SyncStatus struct { + Syncing bool `json:"syncing"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time"` + SyncError string `json:"sync_error"` + + // TODO: Implement these + // The data for these is present when performing a pull via the ostree cli, so it is in the libostree code base. + // SyncedMetadata int `json:"synced_metadata"` + // SyncedObjects int `json:"synced_objects"` +} + +// OSTree is used for managing ostree repositories. +// This is the API documentation of OSTree. +// +//kun:oas title=OSTree Repository Management API +//kun:oas version=1.0.0 +//kun:oas basePath=/artifacts/ostree/api/v1 +//kun:oas docsPath=/doc/swagger.yaml +//kun:oas tags=ostree +type OSTree interface { + // Create an OSTree repository. + //kun:op POST /repository + //kun:success statusCode=200 + CreateRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) + + // Delete a OSTree repository. + //kun:op DELETE /repository + //kun:success statusCode=202 + DeleteRepository(ctx context.Context, repository string) (err error) + + // Add a new remote to the OSTree repository. + //kun:op POST /repository/remote + //kun:success statusCode=200 + AddRemote(ctx context.Context, repository string, properties *OSTreeRemoteProperties) (err error) + + // Sync an ostree repository with one of the configured remotes. + //kun:op POST /repository/sync + //kun:success statusCode=202 + SyncRepository(ctx context.Context, repository string, properties *OSTreeRepositorySyncRequest) (err error) + + // Get OSTree repository sync status. + //kun:op GET /repository/sync + //kun:success statusCode=200 + GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *SyncStatus, err error) +} diff --git a/pkg/plugins/ostree/api/v1/endpoint.go b/pkg/plugins/ostree/api/v1/endpoint.go new file mode 100644 index 0000000..cabf265 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/endpoint.go @@ -0,0 +1,195 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + + "github.com/RussellLuo/kun/pkg/httpoption" + "github.com/RussellLuo/validating/v3" + "github.com/go-kit/kit/endpoint" +) + +type AddRemoteRequest struct { + Repository string `json:"repository"` + Properties *OSTreeRemoteProperties `json:"properties"` +} + +// ValidateAddRemoteRequest creates a validator for AddRemoteRequest. +func ValidateAddRemoteRequest(newSchema func(*AddRemoteRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*AddRemoteRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type AddRemoteResponse struct { + Err error `json:"-"` +} + +func (r *AddRemoteResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *AddRemoteResponse) Failed() error { return r.Err } + +// MakeEndpointOfAddRemote creates the endpoint for s.AddRemote. +func MakeEndpointOfAddRemote(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*AddRemoteRequest) + err := s.AddRemote( + ctx, + req.Repository, + req.Properties, + ) + return &AddRemoteResponse{ + Err: err, + }, nil + } +} + +type CreateRepositoryRequest struct { + Repository string `json:"repository"` + Properties *OSTreeRepositoryProperties `json:"properties"` +} + +// ValidateCreateRepositoryRequest creates a validator for CreateRepositoryRequest. +func ValidateCreateRepositoryRequest(newSchema func(*CreateRepositoryRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*CreateRepositoryRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type CreateRepositoryResponse struct { + Err error `json:"-"` +} + +func (r *CreateRepositoryResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *CreateRepositoryResponse) Failed() error { return r.Err } + +// MakeEndpointOfCreateRepository creates the endpoint for s.CreateRepository. +func MakeEndpointOfCreateRepository(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*CreateRepositoryRequest) + err := s.CreateRepository( + ctx, + req.Repository, + req.Properties, + ) + return &CreateRepositoryResponse{ + Err: err, + }, nil + } +} + +type DeleteRepositoryRequest struct { + Repository string `json:"repository"` +} + +// ValidateDeleteRepositoryRequest creates a validator for DeleteRepositoryRequest. +func ValidateDeleteRepositoryRequest(newSchema func(*DeleteRepositoryRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*DeleteRepositoryRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type DeleteRepositoryResponse struct { + Err error `json:"-"` +} + +func (r *DeleteRepositoryResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *DeleteRepositoryResponse) Failed() error { return r.Err } + +// MakeEndpointOfDeleteRepository creates the endpoint for s.DeleteRepository. +func MakeEndpointOfDeleteRepository(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*DeleteRepositoryRequest) + err := s.DeleteRepository( + ctx, + req.Repository, + ) + return &DeleteRepositoryResponse{ + Err: err, + }, nil + } +} + +type GetRepositorySyncStatusRequest struct { + Repository string `json:"repository"` +} + +// ValidateGetRepositorySyncStatusRequest creates a validator for GetRepositorySyncStatusRequest. +func ValidateGetRepositorySyncStatusRequest(newSchema func(*GetRepositorySyncStatusRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*GetRepositorySyncStatusRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type GetRepositorySyncStatusResponse struct { + SyncStatus *SyncStatus `json:"sync_status"` + Err error `json:"-"` +} + +func (r *GetRepositorySyncStatusResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *GetRepositorySyncStatusResponse) Failed() error { return r.Err } + +// MakeEndpointOfGetRepositorySyncStatus creates the endpoint for s.GetRepositorySyncStatus. +func MakeEndpointOfGetRepositorySyncStatus(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*GetRepositorySyncStatusRequest) + syncStatus, err := s.GetRepositorySyncStatus( + ctx, + req.Repository, + ) + return &GetRepositorySyncStatusResponse{ + SyncStatus: syncStatus, + Err: err, + }, nil + } +} + +type SyncRepositoryRequest struct { + Repository string `json:"repository"` + Properties *OSTreeRepositorySyncRequest `json:"properties"` +} + +// ValidateSyncRepositoryRequest creates a validator for SyncRepositoryRequest. +func ValidateSyncRepositoryRequest(newSchema func(*SyncRepositoryRequest) validating.Schema) httpoption.Validator { + return httpoption.FuncValidator(func(value interface{}) error { + req := value.(*SyncRepositoryRequest) + return httpoption.Validate(newSchema(req)) + }) +} + +type SyncRepositoryResponse struct { + Err error `json:"-"` +} + +func (r *SyncRepositoryResponse) Body() interface{} { return r } + +// Failed implements endpoint.Failer. +func (r *SyncRepositoryResponse) Failed() error { return r.Err } + +// MakeEndpointOfSyncRepository creates the endpoint for s.SyncRepository. +func MakeEndpointOfSyncRepository(s OSTree) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(*SyncRepositoryRequest) + err := s.SyncRepository( + ctx, + req.Repository, + req.Properties, + ) + return &SyncRepositoryResponse{ + Err: err, + }, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http.go b/pkg/plugins/ostree/api/v1/http.go new file mode 100644 index 0000000..fb8eace --- /dev/null +++ b/pkg/plugins/ostree/api/v1/http.go @@ -0,0 +1,178 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + "net/http" + + "github.com/RussellLuo/kun/pkg/httpcodec" + "github.com/RussellLuo/kun/pkg/httpoption" + "github.com/RussellLuo/kun/pkg/oas2" + "github.com/go-chi/chi" + kithttp "github.com/go-kit/kit/transport/http" +) + +func NewHTTPRouter(svc OSTree, codecs httpcodec.Codecs, opts ...httpoption.Option) chi.Router { + r := chi.NewRouter() + options := httpoption.NewOptions(opts...) + + r.Method("GET", "/doc/swagger.yaml", oas2.Handler(OASv2APIDoc, options.ResponseSchema())) + + var codec httpcodec.Codec + var validator httpoption.Validator + var kitOptions []kithttp.ServerOption + + codec = codecs.EncodeDecoder("AddRemote") + validator = options.RequestValidator("AddRemote") + r.Method( + "POST", "/repository/remote", + kithttp.NewServer( + MakeEndpointOfAddRemote(svc), + decodeAddRemoteRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("CreateRepository") + validator = options.RequestValidator("CreateRepository") + r.Method( + "POST", "/repository", + kithttp.NewServer( + MakeEndpointOfCreateRepository(svc), + decodeCreateRepositoryRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("DeleteRepository") + validator = options.RequestValidator("DeleteRepository") + r.Method( + "DELETE", "/repository", + kithttp.NewServer( + MakeEndpointOfDeleteRepository(svc), + decodeDeleteRepositoryRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 202), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("GetRepositorySyncStatus") + validator = options.RequestValidator("GetRepositorySyncStatus") + r.Method( + "GET", "/repository/sync", + kithttp.NewServer( + MakeEndpointOfGetRepositorySyncStatus(svc), + decodeGetRepositorySyncStatusRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 200), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + codec = codecs.EncodeDecoder("SyncRepository") + validator = options.RequestValidator("SyncRepository") + r.Method( + "POST", "/repository/sync", + kithttp.NewServer( + MakeEndpointOfSyncRepository(svc), + decodeSyncRepositoryRequest(codec, validator), + httpcodec.MakeResponseEncoder(codec, 202), + append(kitOptions, + kithttp.ServerErrorEncoder(httpcodec.MakeErrorEncoder(codec)), + )..., + ), + ) + + return r +} + +func decodeAddRemoteRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req AddRemoteRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeCreateRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req CreateRepositoryRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeDeleteRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req DeleteRepositoryRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeGetRepositorySyncStatusRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req GetRepositorySyncStatusRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} + +func decodeSyncRepositoryRequest(codec httpcodec.Codec, validator httpoption.Validator) kithttp.DecodeRequestFunc { + return func(_ context.Context, r *http.Request) (interface{}, error) { + var _req SyncRepositoryRequest + + if err := codec.DecodeRequestBody(r, &_req); err != nil { + return nil, err + } + + if err := validator.Validate(&_req); err != nil { + return nil, err + } + + return &_req, nil + } +} diff --git a/pkg/plugins/ostree/api/v1/http_client.go b/pkg/plugins/ostree/api/v1/http_client.go new file mode 100644 index 0000000..989ec36 --- /dev/null +++ b/pkg/plugins/ostree/api/v1/http_client.go @@ -0,0 +1,281 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "context" + "net/http" + "net/url" + "strings" + + "github.com/RussellLuo/kun/pkg/httpcodec" +) + +type HTTPClient struct { + codecs httpcodec.Codecs + httpClient *http.Client + scheme string + host string + pathPrefix string +} + +func NewHTTPClient(codecs httpcodec.Codecs, httpClient *http.Client, baseURL string) (*HTTPClient, error) { + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + return &HTTPClient{ + codecs: codecs, + httpClient: httpClient, + scheme: u.Scheme, + host: u.Host, + pathPrefix: strings.TrimSuffix(u.Path, "/"), + }, nil +} + +func (c *HTTPClient) AddRemote(ctx context.Context, repository string, properties *OSTreeRemoteProperties) (err error) { + codec := c.codecs.EncodeDecoder("AddRemote") + + path := "/repository/remote" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + Properties *OSTreeRemoteProperties `json:"properties"` + }{ + Repository: repository, + Properties: properties, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "POST", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} + +func (c *HTTPClient) CreateRepository(ctx context.Context, repository string, properties *OSTreeRepositoryProperties) (err error) { + codec := c.codecs.EncodeDecoder("CreateRepository") + + path := "/repository" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + Properties *OSTreeRepositoryProperties `json:"properties"` + }{ + Repository: repository, + Properties: properties, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "POST", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} + +func (c *HTTPClient) DeleteRepository(ctx context.Context, repository string) (err error) { + codec := c.codecs.EncodeDecoder("DeleteRepository") + + path := "/repository" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + }{ + Repository: repository, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "DELETE", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} + +func (c *HTTPClient) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *SyncStatus, err error) { + codec := c.codecs.EncodeDecoder("GetRepositorySyncStatus") + + path := "/repository/sync" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + }{ + Repository: repository, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return nil, err + } + + _req, err := http.NewRequestWithContext(ctx, "GET", u.String(), reqBodyReader) + if err != nil { + return nil, err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return nil, err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return nil, err + } + + respBody := &GetRepositorySyncStatusResponse{} + err = codec.DecodeSuccessResponse(_resp.Body, respBody.Body()) + if err != nil { + return nil, err + } + return respBody.SyncStatus, nil +} + +func (c *HTTPClient) SyncRepository(ctx context.Context, repository string, properties *OSTreeRepositorySyncRequest) (err error) { + codec := c.codecs.EncodeDecoder("SyncRepository") + + path := "/repository/sync" + u := &url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: c.pathPrefix + path, + } + + reqBody := struct { + Repository string `json:"repository"` + Properties *OSTreeRepositorySyncRequest `json:"properties"` + }{ + Repository: repository, + Properties: properties, + } + reqBodyReader, headers, err := codec.EncodeRequestBody(&reqBody) + if err != nil { + return err + } + + _req, err := http.NewRequestWithContext(ctx, "POST", u.String(), reqBodyReader) + if err != nil { + return err + } + + for k, v := range headers { + _req.Header.Set(k, v) + } + + _resp, err := c.httpClient.Do(_req) + if err != nil { + return err + } + defer _resp.Body.Close() + + if _resp.StatusCode < http.StatusOK || _resp.StatusCode > http.StatusNoContent { + var respErr error + err := codec.DecodeFailureResponse(_resp.Body, &respErr) + if err == nil { + err = respErr + } + return err + } + + return nil +} diff --git a/pkg/plugins/ostree/api/v1/oas2.go b/pkg/plugins/ostree/api/v1/oas2.go new file mode 100644 index 0000000..ba7f80b --- /dev/null +++ b/pkg/plugins/ostree/api/v1/oas2.go @@ -0,0 +1,145 @@ +// Code generated by kun; DO NOT EDIT. +// github.com/RussellLuo/kun + +package apiv1 + +import ( + "reflect" + + "github.com/RussellLuo/kun/pkg/oas2" +) + +var ( + base = `swagger: "2.0" +info: + title: "OSTree Repository Management API" + version: "1.0.0" + description: "OSTree is used for managing ostree repositories.\nThis is the API documentation of OSTree.\n//" + license: + name: "MIT" +host: "example.com" +basePath: "/artifacts/ostree/api/v1" +schemes: + - "https" +consumes: + - "application/json" +produces: + - "application/json" +` + + paths = ` +paths: + /repository/remote: + post: + description: "Add a new remote to the OSTree repository." + operationId: "AddRemote" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/AddRemoteRequestBody" + %s + /repository: + post: + description: "Create an OSTree repository." + operationId: "CreateRepository" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/CreateRepositoryRequestBody" + %s + delete: + description: "Delete a OSTree repository." + operationId: "DeleteRepository" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/DeleteRepositoryRequestBody" + %s + /repository/sync: + get: + description: "Get OSTree repository sync status." + operationId: "GetRepositorySyncStatus" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/GetRepositorySyncStatusRequestBody" + %s + post: + description: "Sync an ostree repository with one of the configured remotes." + operationId: "SyncRepository" + tags: + - ostree + parameters: + - name: body + in: body + schema: + $ref: "#/definitions/SyncRepositoryRequestBody" + %s +` +) + +func getResponses(schema oas2.Schema) []oas2.OASResponses { + return []oas2.OASResponses{ + oas2.GetOASResponses(schema, "AddRemote", 200, &AddRemoteResponse{}), + oas2.GetOASResponses(schema, "CreateRepository", 200, &CreateRepositoryResponse{}), + oas2.GetOASResponses(schema, "DeleteRepository", 202, &DeleteRepositoryResponse{}), + oas2.GetOASResponses(schema, "GetRepositorySyncStatus", 200, &GetRepositorySyncStatusResponse{}), + oas2.GetOASResponses(schema, "SyncRepository", 202, &SyncRepositoryResponse{}), + } +} + +func getDefinitions(schema oas2.Schema) map[string]oas2.Definition { + defs := make(map[string]oas2.Definition) + + oas2.AddDefinition(defs, "AddRemoteRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + Properties *OSTreeRemoteProperties `json:"properties"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "AddRemote", 200, (&AddRemoteResponse{}).Body()) + + oas2.AddDefinition(defs, "CreateRepositoryRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + Properties *OSTreeRepositoryProperties `json:"properties"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "CreateRepository", 200, (&CreateRepositoryResponse{}).Body()) + + oas2.AddDefinition(defs, "DeleteRepositoryRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "DeleteRepository", 202, (&DeleteRepositoryResponse{}).Body()) + + oas2.AddDefinition(defs, "GetRepositorySyncStatusRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "GetRepositorySyncStatus", 200, (&GetRepositorySyncStatusResponse{}).Body()) + + oas2.AddDefinition(defs, "SyncRepositoryRequestBody", reflect.ValueOf(&struct { + Repository string `json:"repository"` + Properties *OSTreeRepositorySyncRequest `json:"properties"` + }{})) + oas2.AddResponseDefinitions(defs, schema, "SyncRepository", 202, (&SyncRepositoryResponse{}).Body()) + + return defs +} + +func OASv2APIDoc(schema oas2.Schema) string { + resps := getResponses(schema) + paths := oas2.GenPaths(resps, paths) + + defs := getDefinitions(schema) + definitions := oas2.GenDefinitions(defs) + + return base + paths + definitions +} diff --git a/pkg/utils/time.go b/pkg/utils/time.go new file mode 100644 index 0000000..16981bd --- /dev/null +++ b/pkg/utils/time.go @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2023-2024, CIQ, Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import "time" + +const timeFormat = "2006-01-02 15:04:05 MST" + +func TimeToString(t int64) string { + if t == 0 { + return "" + } + return time.Unix(t, 0).Format(timeFormat) +}