From b39b6d01ce43878b72f290b5f3ba689bae415dc9 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 15:03:55 -0400 Subject: [PATCH 01/14] fix: multiplatform sbom and vulnscan --- .gitignore | 1 + Task/Taskfile.yml | 109 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 90 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index fcfc3082..0ea8fffb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .task/* sbom.*.json vulns.*.json +seiso_goat:*.tar # Created by https://www.toptal.com/developers/gitignore/api/vim,emacs,vs,python,node,macos # Edit at https://www.toptal.com/developers/gitignore?templates=vim,emacs,vs,python,node,macos diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index bd4a3bc9..69cd6588 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -9,6 +9,7 @@ set: - pipefail vars: + # Inspired by https://github.com/containerd/containerd/blob/e0912c068b131b33798ae45fd447a1624a6faf0a/platforms/database.go#L76 LOCAL_PLATFORM: sh: | os="linux" @@ -91,7 +92,6 @@ tasks: VERSION: '{{.VERSION}}' PLATFORM: '{{if eq .PLATFORM "all"}}{{.SUPPORTED_PLATFORMS}}{{else if .PLATFORM}}{{.PLATFORM}}{{else}}{{.LOCAL_PLATFORM}}{{end}}' PUBLISH: '{{.PUBLISH | default "false"}}' - DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' TAG_COMMIT_HASH: sh: git rev-list -1 "v{{.VERSION}}" COMMIT_HASH: @@ -116,6 +116,11 @@ tasks: else: build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" print(build_version)' + OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_"}}:{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_"}}.tar' + DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' + DOCKER_BUILDX_DOCKERFILE: '{{.DOCKER_BUILDX_DOCKERFILE | default "."}}' + DOCKER_BUILDX_BUILDARGS: '{{.DOCKER_BUILDX_BUILDARGS | default "--build-arg VERSION=\"{{.BUILD_VERSION}}\" --build-arg COMMIT_HASH=\"{{.COMMIT_HASH}}\""}}' + DOCKER_BUILDX_TAGS: '{{.DOCKER_BUILDX_TAGS | default "--tag \"{{.IMAGE_NAME}}:latest\" --tag \"{{.IMAGE_NAME}}:{{.BUILD_VERSION}}\""}}' cmds: # We only load when the provided platform equals the detected local platform. This is for two reasons: # 1. We assume you don't want to load a cross-platform build @@ -123,18 +128,18 @@ tasks: # # Also, we make load and push mutually exclusive because docker says "ERROR: push and load may not be set together at the moment" # - # Finally, we combine this all together in one `docker buildx build` with `--push` when {{.PUBLISH}} is true so that it handles the multi-platform - # manifest creation for us. Otherwise we'd need to push per-platform tags and artisanally craft the manifest with `crane`, `docker manifest`, or similar + # If we aren't loading or pushing, we dump an OCI-formatted artifact out to disk + # + # We leverage `docker buildx build` with `--push` to make a multi-platform manifest when {{.PUBLISH}} is true. Otherwise we'd need to push per-platform + # tags and artisanally craft the multi-platform manifest with a tool like `crane`, `docker manifest`, or similar - | docker buildx build --platform="{{.PLATFORM}}" \ - {{if eq .PUBLISH "true"}}--push{{else if eq .PLATFORM .LOCAL_PLATFORM}}--load{{end}} \ + {{if eq .PUBLISH "true"}}--push{{else if eq .PLATFORM .LOCAL_PLATFORM}}--load{{else}}-o type=oci,dest="{{.OUTPUT_FILE}}"{{end}} \ {{if .DOCKER_BUILDX_CUSTOM_ARGS}}{{.DOCKER_BUILDX_CUSTOM_ARGS}}{{end}} \ - --build-arg VERSION="{{.BUILD_VERSION}}" \ - --build-arg COMMIT_HASH="{{.COMMIT_HASH}}" \ - --tag {{.IMAGE_NAME}}:latest \ - --tag {{.IMAGE_NAME}}:{{.BUILD_VERSION}} \ - "${PWD}/." - - '{{if ne .PLATFORM .LOCAL_PLATFORM}}{{if ne .PUBLISH "true"}}echo "WARNING: Avoided loading {{.IMAGE_NAME}}:latest and {{.IMAGE_NAME}}:{{.BUILD_VERSION}} into your docker daemon because you built a cross-platform image of {{.PLATFORM}}"{{end}}{{end}}' + {{.DOCKER_BUILDX_BUILDARGS}} \ + {{.DOCKER_BUILDX_TAGS}} \ + "{{.DOCKER_BUILDX_DOCKERFILE}}" + - '{{if ne .PLATFORM .LOCAL_PLATFORM}}{{if ne .PUBLISH "true"}}echo "WARNING: Avoided loading {{.IMAGE_NAME}}:latest and {{.IMAGE_NAME}}:{{.BUILD_VERSION}} into your docker daemon because you built a cross-platform image of {{.PLATFORM}}.{{if ne .PUBLISH "true"}} See {{.OUTPUT_FILE}} for the OCI artifact.{{end}}"{{end}}{{end}}' release: desc: Cut a project release @@ -241,6 +246,7 @@ tasks: - find {{.ROOT_DIR}} -type d -name '.task' -exec rm -rf {} + - find {{.ROOT_DIR}} -type f -name 'sbom.*.json' -delete - find {{.ROOT_DIR}} -type f -name 'vulns.*.json' -delete + - find {{.ROOT_DIR}} -type f -name 'seiso_*:*.tar' -delete sbom: desc: Generate project SBOMs @@ -249,25 +255,88 @@ tasks: - sh: which syft msg: "Syft must be installed and reasonably current" vars: - IMAGE_AND_TAG: '{{.IMAGE_NAME}}:{{.VERSION}}' PLATFORM: '{{if eq .PLATFORM "all"}}{{.SUPPORTED_PLATFORMS}}{{else if .PLATFORM}}{{.PLATFORM}}{{else}}{{.LOCAL_PLATFORM}}{{end}}' + # This duplicates some build logic; consider centralizing + TAG_COMMIT_HASH: + sh: git rev-list -1 "v{{.VERSION}}" + COMMIT_HASH: + sh: git rev-parse HEAD + COMMIT_HASH_SHORT: + sh: git rev-parse --short HEAD + REPO_TAGS: + sh: git tag -l + BUILD_VERSION: + sh: | + pipenv run python -c ' + version_string = "v{{.VERSION}}" + repo_tags = [] + {{range $tag := .REPO_TAGS | splitLines -}} + repo_tags.append("{{$tag}}") + {{end}} + if ( + version_string in repo_tags + and "{{.TAG_COMMIT_HASH}}" == "{{.COMMIT_HASH}}" + ): + build_version = "{{.VERSION}}" + else: + build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" + print(build_version)' + IMAGE_AND_TAG: '{{.IMAGE_NAME}}:{{.BUILD_VERSION}}' + SANITIZED_IMAGE_AND_TAG: '{{.IMAGE_AND_TAG | replace "/" "_"}}' cmds: + - for: + var: PLATFORM + split: ',' + as: platform + task: build + vars: + PLATFORM: '{{.platform}}' + DOCKER_BUILDX_TAGS: '--tag {{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}' - for: var: PLATFORM split: ',' as: platform cmd: | - export sanitized_platform=$(echo "{{.platform}}" | sed "s%/%_%g") \ - && syft docker:{{.IMAGE_AND_TAG}} --platform {{.platform}} \ - -o json=sbom.{{.PROJECT_SLUG}}.{{.VERSION}}.${sanitized_platform}.json \ - -o spdx-json=sbom.{{.PROJECT_SLUG}}.{{.VERSION}}.${sanitized_platform}.spdx.json \ - -o cyclonedx-json=sbom.{{.PROJECT_SLUG}}.{{.VERSION}}.${sanitized_platform}.cyclonedx.json + export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" }}' \ + && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}{{end}}" \ + && syft "${syft_command}" {{if eq .PLATFORM .LOCAL_PLATFORM}}--platform {{.platform}}{{end}} \ + -o json=sbom.${base_name}.syft.json \ + -o spdx-json=sbom.${base_name}.spdx.json \ + -o cyclonedx-json=sbom.${base_name}.cyclonedx.json + vulnscan: desc: Vuln scan the SBOM dir: ../../.. vars: PLATFORM: '{{if eq .PLATFORM "all"}}{{.SUPPORTED_PLATFORMS}}{{else if .PLATFORM}}{{.PLATFORM}}{{else}}{{.LOCAL_PLATFORM}}{{end}}' + # This duplicates some build logic; consider centralizing + TAG_COMMIT_HASH: + sh: git rev-list -1 "v{{.VERSION}}" + COMMIT_HASH: + sh: git rev-parse HEAD + COMMIT_HASH_SHORT: + sh: git rev-parse --short HEAD + REPO_TAGS: + sh: git tag -l + BUILD_VERSION: + sh: | + pipenv run python -c ' + version_string = "v{{.VERSION}}" + repo_tags = [] + {{range $tag := .REPO_TAGS | splitLines -}} + repo_tags.append("{{$tag}}") + {{end}} + if ( + version_string in repo_tags + and "{{.TAG_COMMIT_HASH}}" == "{{.COMMIT_HASH}}" + ): + build_version = "{{.VERSION}}" + else: + build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" + print(build_version)' + IMAGE_AND_TAG: '{{.IMAGE_NAME}}:{{.BUILD_VERSION}}' + SANITIZED_IMAGE_AND_TAG: '{{.IMAGE_AND_TAG | replace "/" "_"}}' preconditions: - sh: which grype msg: "Grype must be installed and reasonably current" @@ -277,7 +346,7 @@ tasks: split: ',' as: platform cmd: | - export sanitized_platform=$(echo "{{.platform}}" | sed "s%/%_%g") \ - && grype sbom:sbom.{{.PROJECT_SLUG}}.{{.VERSION}}.${sanitized_platform}.json \ - --output json \ - --file vulns.{{.PROJECT_SLUG}}.{{.VERSION}}.${sanitized_platform}.json + export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" }}' \ + && grype "sbom:sbom.${base_name}.syft.json" \ + --output json \ + --file "vulns.${base_name}.json" From c8cec85d3c79f3eb799a2c874e45f4f980d783da Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 15:14:01 -0400 Subject: [PATCH 02/14] Undo quoting --- Task/Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index 69cd6588..8d53e530 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -120,7 +120,7 @@ tasks: DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' DOCKER_BUILDX_DOCKERFILE: '{{.DOCKER_BUILDX_DOCKERFILE | default "."}}' DOCKER_BUILDX_BUILDARGS: '{{.DOCKER_BUILDX_BUILDARGS | default "--build-arg VERSION=\"{{.BUILD_VERSION}}\" --build-arg COMMIT_HASH=\"{{.COMMIT_HASH}}\""}}' - DOCKER_BUILDX_TAGS: '{{.DOCKER_BUILDX_TAGS | default "--tag \"{{.IMAGE_NAME}}:latest\" --tag \"{{.IMAGE_NAME}}:{{.BUILD_VERSION}}\""}}' + DOCKER_BUILDX_TAGS: '{{.DOCKER_BUILDX_TAGS | default "--tag {{.IMAGE_NAME}}:latest --tag {{.IMAGE_NAME}}:{{.BUILD_VERSION}}"}}' cmds: # We only load when the provided platform equals the detected local platform. This is for two reasons: # 1. We assume you don't want to load a cross-platform build From 74a4be5e231681edaad953425d99d5e822c2ec09 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 21:19:14 -0400 Subject: [PATCH 03/14] Can't next go tempaltes in default --- Task/Taskfile.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index 8d53e530..4ffb97ab 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -118,9 +118,7 @@ tasks: print(build_version)' OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_"}}:{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_"}}.tar' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' - DOCKER_BUILDX_DOCKERFILE: '{{.DOCKER_BUILDX_DOCKERFILE | default "."}}' - DOCKER_BUILDX_BUILDARGS: '{{.DOCKER_BUILDX_BUILDARGS | default "--build-arg VERSION=\"{{.BUILD_VERSION}}\" --build-arg COMMIT_HASH=\"{{.COMMIT_HASH}}\""}}' - DOCKER_BUILDX_TAGS: '{{.DOCKER_BUILDX_TAGS | default "--tag {{.IMAGE_NAME}}:latest --tag {{.IMAGE_NAME}}:{{.BUILD_VERSION}}"}}' + DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE | default "."}}' cmds: # We only load when the provided platform equals the detected local platform. This is for two reasons: # 1. We assume you don't want to load a cross-platform build @@ -134,11 +132,11 @@ tasks: # tags and artisanally craft the multi-platform manifest with a tool like `crane`, `docker manifest`, or similar - | docker buildx build --platform="{{.PLATFORM}}" \ - {{if eq .PUBLISH "true"}}--push{{else if eq .PLATFORM .LOCAL_PLATFORM}}--load{{else}}-o type=oci,dest="{{.OUTPUT_FILE}}"{{end}} \ - {{if .DOCKER_BUILDX_CUSTOM_ARGS}}{{.DOCKER_BUILDX_CUSTOM_ARGS}}{{end}} \ - {{.DOCKER_BUILDX_BUILDARGS}} \ - {{.DOCKER_BUILDX_TAGS}} \ - "{{.DOCKER_BUILDX_DOCKERFILE}}" + {{if eq .PUBLISH "true"}}--push{{else if eq .PLATFORM .LOCAL_PLATFORM}}--load{{else}}-o type=oci,dest="{{.OUTPUT_FILE}}"{{end}} \ + {{if .DOCKER_BUILDX_CUSTOM_ARGS}}{{.DOCKER_BUILDX_CUSTOM_ARGS}}{{end}} \ + {{if .DOCKER_BUILDX_CUSTOM_TAGS}}{{.DOCKER_BUILDX_CUSTOM_TAGS}}{{else}}--tag "{{.IMAGE_NAME}}:latest" --tag "{{.IMAGE_NAME}}:{{.BUILD_VERSION}}"{{end}} \ + {{if .DOCKER_BUILDX_CUSTOM_BUILDARGS}}{{.DOCKER_BUILDX_CUSTOM_BUILDARGS}}{{else}}--build-arg VERSION="{{.BUILD_VERSION}}" --build-arg COMMIT_HASH="{{.COMMIT_HASH}}"{{end}} \ + "{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}" - '{{if ne .PLATFORM .LOCAL_PLATFORM}}{{if ne .PUBLISH "true"}}echo "WARNING: Avoided loading {{.IMAGE_NAME}}:latest and {{.IMAGE_NAME}}:{{.BUILD_VERSION}} into your docker daemon because you built a cross-platform image of {{.PLATFORM}}.{{if ne .PUBLISH "true"}} See {{.OUTPUT_FILE}} for the OCI artifact.{{end}}"{{end}}{{end}}' release: From f216cc075b700974b26efcc0ad625ae76dbcdb2e Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 21:39:16 -0400 Subject: [PATCH 04/14] Tweak --- Task/Taskfile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index 4ffb97ab..963054b2 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -289,14 +289,14 @@ tasks: task: build vars: PLATFORM: '{{.platform}}' - DOCKER_BUILDX_TAGS: '--tag {{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}' + DOCKER_BUILDX_CUSTOM_TAGS: '--tag {{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}' - for: var: PLATFORM split: ',' as: platform cmd: | export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" }}' \ - && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}{{end}}" \ + && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}{{end}}" \ && syft "${syft_command}" {{if eq .PLATFORM .LOCAL_PLATFORM}}--platform {{.platform}}{{end}} \ -o json=sbom.${base_name}.syft.json \ -o spdx-json=sbom.${base_name}.spdx.json \ From f857e9b89aee7f83134fec985a95c40964b1854a Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 21:45:33 -0400 Subject: [PATCH 05/14] Try this --- Task/Taskfile.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index 963054b2..51cbf03f 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -116,7 +116,7 @@ tasks: else: build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" print(build_version)' - OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_"}}:{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_"}}.tar' + OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_" | replace ":" "_"}}:{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_" | replace ":" "_"}}.tar' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE | default "."}}' cmds: @@ -280,7 +280,7 @@ tasks: build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" print(build_version)' IMAGE_AND_TAG: '{{.IMAGE_NAME}}:{{.BUILD_VERSION}}' - SANITIZED_IMAGE_AND_TAG: '{{.IMAGE_AND_TAG | replace "/" "_"}}' + SANITIZED_IMAGE_AND_TAG: '{{.IMAGE_AND_TAG | replace "/" "_" | replace ":" "_"}}' cmds: - for: var: PLATFORM @@ -289,14 +289,14 @@ tasks: task: build vars: PLATFORM: '{{.platform}}' - DOCKER_BUILDX_CUSTOM_TAGS: '--tag {{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}' + DOCKER_BUILDX_CUSTOM_TAGS: '--tag {{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_" | replace ":" "_"}}' - for: var: PLATFORM split: ',' as: platform cmd: | - export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" }}' \ - && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}{{end}}" \ + export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" | replace ":" "_"}}' \ + && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_" | replace ":" "_"}}{{end}}" \ && syft "${syft_command}" {{if eq .PLATFORM .LOCAL_PLATFORM}}--platform {{.platform}}{{end}} \ -o json=sbom.${base_name}.syft.json \ -o spdx-json=sbom.${base_name}.spdx.json \ @@ -334,7 +334,7 @@ tasks: build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" print(build_version)' IMAGE_AND_TAG: '{{.IMAGE_NAME}}:{{.BUILD_VERSION}}' - SANITIZED_IMAGE_AND_TAG: '{{.IMAGE_AND_TAG | replace "/" "_"}}' + SANITIZED_IMAGE_AND_TAG: '{{.IMAGE_AND_TAG | replace "/" "_" | replace ":" "_"}}' preconditions: - sh: which grype msg: "Grype must be installed and reasonably current" @@ -344,7 +344,7 @@ tasks: split: ',' as: platform cmd: | - export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" }}' \ + export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" | replace ":" "_" }}' \ && grype "sbom:sbom.${base_name}.syft.json" \ --output json \ --file "vulns.${base_name}.json" From 844bca9a93a884d89599526f2dd93333c46f242b Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 21:50:49 -0400 Subject: [PATCH 06/14] Formatting --- Task/Taskfile.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index 51cbf03f..d5e05bf5 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -296,11 +296,11 @@ tasks: as: platform cmd: | export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" | replace ":" "_"}}' \ - && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_" | replace ":" "_"}}{{end}}" \ + && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_" | replace ":" "_"}}{{end}}" \ && syft "${syft_command}" {{if eq .PLATFORM .LOCAL_PLATFORM}}--platform {{.platform}}{{end}} \ - -o json=sbom.${base_name}.syft.json \ - -o spdx-json=sbom.${base_name}.spdx.json \ - -o cyclonedx-json=sbom.${base_name}.cyclonedx.json + -o json=sbom.${base_name}.syft.json \ + -o spdx-json=sbom.${base_name}.spdx.json \ + -o cyclonedx-json=sbom.${base_name}.cyclonedx.json vulnscan: From 8003e6611533a41971fa861a9364eb3da522dfcb Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 22:00:27 -0400 Subject: [PATCH 07/14] Simplify --- Task/Taskfile.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index d5e05bf5..54a19f04 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -116,7 +116,7 @@ tasks: else: build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" print(build_version)' - OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_" | replace ":" "_"}}:{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_" | replace ":" "_"}}.tar' + OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_"}}_{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_"}}.tar' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE | default "."}}' cmds: @@ -289,14 +289,16 @@ tasks: task: build vars: PLATFORM: '{{.platform}}' - DOCKER_BUILDX_CUSTOM_TAGS: '--tag {{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_" | replace ":" "_"}}' + # This is necessary in order to have a separate tag per platform, and ensure there is only one manifest in the image index due to current + # syft/stereoscope limitations + DOCKER_BUILDX_CUSTOM_TAGS: '--tag {{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}' - for: var: PLATFORM split: ',' as: platform cmd: | - export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" | replace ":" "_"}}' \ - && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_" | replace ":" "_"}}{{end}}" \ + export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_"}}' \ + && export syft_command="{{if ne .platform .LOCAL_PLATFORM}}oci-archive:${base_name}.tar{{else}}docker:{{.IMAGE_AND_TAG}}-{{.platform | replace "/" "_"}}{{end}}" \ && syft "${syft_command}" {{if eq .PLATFORM .LOCAL_PLATFORM}}--platform {{.platform}}{{end}} \ -o json=sbom.${base_name}.syft.json \ -o spdx-json=sbom.${base_name}.spdx.json \ @@ -344,7 +346,7 @@ tasks: split: ',' as: platform cmd: | - export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_" | replace ":" "_" }}' \ + export base_name='{{.SANITIZED_IMAGE_AND_TAG}}_{{.platform | replace "/" "_"}}' \ && grype "sbom:sbom.${base_name}.syft.json" \ --output json \ --file "vulns.${base_name}.json" From 7d3161aa2c3a361ec5ee1a4c0284bde2364108d3 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 22:06:22 -0400 Subject: [PATCH 08/14] fix gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0ea8fffb..9e43f22f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ .task/* sbom.*.json vulns.*.json -seiso_goat:*.tar +seiso_goat_*.tar # Created by https://www.toptal.com/developers/gitignore/api/vim,emacs,vs,python,node,macos # Edit at https://www.toptal.com/developers/gitignore?templates=vim,emacs,vs,python,node,macos From 978258b7e302f71ee7927bac22c6139d83ebbaf6 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Fri, 18 Aug 2023 22:10:06 -0400 Subject: [PATCH 09/14] Fix clena --- Task/Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index 54a19f04..6cf4fb8e 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -244,7 +244,7 @@ tasks: - find {{.ROOT_DIR}} -type d -name '.task' -exec rm -rf {} + - find {{.ROOT_DIR}} -type f -name 'sbom.*.json' -delete - find {{.ROOT_DIR}} -type f -name 'vulns.*.json' -delete - - find {{.ROOT_DIR}} -type f -name 'seiso_*:*.tar' -delete + - find {{.ROOT_DIR}} -type f -name 'seiso_*_*.tar' -delete sbom: desc: Generate project SBOMs From 13adbca1003d73b19f7408905ae9297fb5b4abc5 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Sat, 19 Aug 2023 06:21:18 -0400 Subject: [PATCH 10/14] That should do it --- Task/Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index 6cf4fb8e..fcbba009 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -116,7 +116,7 @@ tasks: else: build_version = f"{{.VERSION}}-{{.COMMIT_HASH_SHORT}}" print(build_version)' - OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_"}}_{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_"}}.tar' + OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_"}}_{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_" | replace "," "_"}}.tar' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE | default "."}}' cmds: From e07c9c5665bd92f0d3f80a171dadae770611949c Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Sat, 19 Aug 2023 14:00:44 -0400 Subject: [PATCH 11/14] Remove unnecessary defaults --- Task/bash/Taskfile.yml | 4 ++-- Task/python/Taskfile.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Task/bash/Taskfile.yml b/Task/bash/Taskfile.yml index f1a69f89..a0f9b424 100644 --- a/Task/bash/Taskfile.yml +++ b/Task/bash/Taskfile.yml @@ -41,7 +41,7 @@ tasks: - task: base:build vars: VERSION: '{{.VERSION}}' - PLATFORM: '{{.PLATFORM | default ""}}' + PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' update: @@ -65,7 +65,7 @@ tasks: - task: base:publish vars: VERSION: '{{.VERSION}}' - PLATFORM: '{{.PLATFORM | default ""}}' + PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' clean: diff --git a/Task/python/Taskfile.yml b/Task/python/Taskfile.yml index 491ff6d6..0f06a22f 100644 --- a/Task/python/Taskfile.yml +++ b/Task/python/Taskfile.yml @@ -47,7 +47,7 @@ tasks: # Unable to make this global due to https://taskfile.dev/usage/#variables see https://github.com/go-task/task/issues/1295 VERSION: sh: pipenv run python -c 'from {{.PROJECT_SLUG}} import __version__; print(__version__)' - PLATFORM: '{{.PLATFORM | default ""}}' + PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' update: @@ -71,7 +71,7 @@ tasks: # Unable to make this global due to https://taskfile.dev/usage/#variables see https://github.com/go-task/task/issues/1295 VERSION: sh: pipenv run python -c 'from {{.PROJECT_SLUG}} import __version__; print(__version__)' - PLATFORM: '{{.PLATFORM | default ""}}' + PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' clean: From 2f125a5684e8ead62b895a7736ca16c9725ced14 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Sat, 19 Aug 2023 17:05:23 -0400 Subject: [PATCH 12/14] Add DOCKER_BUILDX_CUSTOM_DOCKERFILE to publish --- Task/Taskfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index fcbba009..b41d1aff 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -199,6 +199,7 @@ tasks: VERSION: '{{.VERSION}}' PLATFORM: '{{.PLATFORM | default .LOCAL_PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' + DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}' update: desc: > From 0e261b615a9999bcd4e143b2c9d81ba90c80084e Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Sat, 19 Aug 2023 17:08:13 -0400 Subject: [PATCH 13/14] Try this --- Task/bash/Taskfile.yml | 1 + Task/python/Taskfile.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/Task/bash/Taskfile.yml b/Task/bash/Taskfile.yml index a0f9b424..f7c5d0a2 100644 --- a/Task/bash/Taskfile.yml +++ b/Task/bash/Taskfile.yml @@ -67,6 +67,7 @@ tasks: VERSION: '{{.VERSION}}' PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' + DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}' clean: desc: Clean up build artifacts, cache files/directories, temp files, etc. diff --git a/Task/python/Taskfile.yml b/Task/python/Taskfile.yml index 0f06a22f..ac117def 100644 --- a/Task/python/Taskfile.yml +++ b/Task/python/Taskfile.yml @@ -73,6 +73,7 @@ tasks: sh: pipenv run python -c 'from {{.PROJECT_SLUG}} import __version__; print(__version__)' PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' + DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}' clean: desc: Clean up build artifacts, cache files/directories, temp files, etc. From e656e9b464aa3963b1d924257464d0ebd13b1f49 Mon Sep 17 00:00:00 2001 From: Jon Zeolla Date: Sat, 19 Aug 2023 17:11:39 -0400 Subject: [PATCH 14/14] Rename to be more accurate --- Task/Taskfile.yml | 6 +++--- Task/bash/Taskfile.yml | 2 +- Task/python/Taskfile.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Task/Taskfile.yml b/Task/Taskfile.yml index b41d1aff..ec7a2a9c 100644 --- a/Task/Taskfile.yml +++ b/Task/Taskfile.yml @@ -118,7 +118,7 @@ tasks: print(build_version)' OUTPUT_FILE: '{{.IMAGE_NAME | replace "/" "_"}}_{{.BUILD_VERSION}}_{{.PLATFORM | replace "/" "_" | replace "," "_"}}.tar' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' - DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE | default "."}}' + DOCKER_BUILDX_CUSTOM_CONTEXT: '{{.DOCKER_BUILDX_CUSTOM_CONTEXT | default "."}}' cmds: # We only load when the provided platform equals the detected local platform. This is for two reasons: # 1. We assume you don't want to load a cross-platform build @@ -136,7 +136,7 @@ tasks: {{if .DOCKER_BUILDX_CUSTOM_ARGS}}{{.DOCKER_BUILDX_CUSTOM_ARGS}}{{end}} \ {{if .DOCKER_BUILDX_CUSTOM_TAGS}}{{.DOCKER_BUILDX_CUSTOM_TAGS}}{{else}}--tag "{{.IMAGE_NAME}}:latest" --tag "{{.IMAGE_NAME}}:{{.BUILD_VERSION}}"{{end}} \ {{if .DOCKER_BUILDX_CUSTOM_BUILDARGS}}{{.DOCKER_BUILDX_CUSTOM_BUILDARGS}}{{else}}--build-arg VERSION="{{.BUILD_VERSION}}" --build-arg COMMIT_HASH="{{.COMMIT_HASH}}"{{end}} \ - "{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}" + "{{.DOCKER_BUILDX_CUSTOM_CONTEXT}}" - '{{if ne .PLATFORM .LOCAL_PLATFORM}}{{if ne .PUBLISH "true"}}echo "WARNING: Avoided loading {{.IMAGE_NAME}}:latest and {{.IMAGE_NAME}}:{{.BUILD_VERSION}} into your docker daemon because you built a cross-platform image of {{.PLATFORM}}.{{if ne .PUBLISH "true"}} See {{.OUTPUT_FILE}} for the OCI artifact.{{end}}"{{end}}{{end}}' release: @@ -199,7 +199,7 @@ tasks: VERSION: '{{.VERSION}}' PLATFORM: '{{.PLATFORM | default .LOCAL_PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' - DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}' + DOCKER_BUILDX_CUSTOM_CONTEXT: '{{.DOCKER_BUILDX_CUSTOM_CONTEXT}}' update: desc: > diff --git a/Task/bash/Taskfile.yml b/Task/bash/Taskfile.yml index f7c5d0a2..1bc5f54b 100644 --- a/Task/bash/Taskfile.yml +++ b/Task/bash/Taskfile.yml @@ -67,7 +67,7 @@ tasks: VERSION: '{{.VERSION}}' PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' - DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}' + DOCKER_BUILDX_CUSTOM_CONTEXT: '{{.DOCKER_BUILDX_CUSTOM_CONTEXT}}' clean: desc: Clean up build artifacts, cache files/directories, temp files, etc. diff --git a/Task/python/Taskfile.yml b/Task/python/Taskfile.yml index ac117def..048a78db 100644 --- a/Task/python/Taskfile.yml +++ b/Task/python/Taskfile.yml @@ -73,7 +73,7 @@ tasks: sh: pipenv run python -c 'from {{.PROJECT_SLUG}} import __version__; print(__version__)' PLATFORM: '{{.PLATFORM}}' DOCKER_BUILDX_CUSTOM_ARGS: '{{.DOCKER_BUILDX_CUSTOM_ARGS | default ""}}' - DOCKER_BUILDX_CUSTOM_DOCKERFILE: '{{.DOCKER_BUILDX_CUSTOM_DOCKERFILE}}' + DOCKER_BUILDX_CUSTOM_CONTEXT: '{{.DOCKER_BUILDX_CUSTOM_CONTEXT}}' clean: desc: Clean up build artifacts, cache files/directories, temp files, etc.