fix: pulling not respecting "uncompressed" setting in metadata (#3472)

Signed-off-by: Allen Conlon <[email protected]>

go.mod

@@ -311,7 +311,7 @@ require (
 	github.com/fluxcd/pkg/apis/acl v0.3.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fvbommel/sortorder v1.1.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.8
 	github.com/gdamore/encoding v1.0.1 // indirect
 	github.com/github/go-spdx/v2 v2.3.2 // indirect
 	github.com/glebarez/go-sqlite v1.21.2 // indirect

src/internal/packager2/load.go

@@ -51,12 +51,12 @@ func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackageLayout, e
 	isPartial := false
 	switch srcType {
 	case "oci":
-		isPartial, err = pullOCI(ctx, opt.Source, tarPath, opt.Shasum, opt.Filter)
+		isPartial, tarPath, err = pullOCI(ctx, opt.Source, tmpDir, opt.Shasum, opt.Filter)
 		if err != nil {
 			return nil, err
 		}
 	case "http", "https":
-		err = pullHTTP(ctx, opt.Source, tarPath, opt.Shasum)
+		tarPath, err = pullHTTP(ctx, opt.Source, tmpDir, opt.Shasum)
 		if err != nil {
 			return nil, err
 		}

src/internal/packager2/pull.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/defenseunicorns/pkg/helpers/v2"
 	"github.com/defenseunicorns/pkg/oci"
+	"github.com/gabriel-vasile/mimetype"
 	goyaml "github.com/goccy/go-yaml"
 	"github.com/mholt/archiver/v3"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -46,17 +47,17 @@ func Pull(ctx context.Context, src, dir, shasum string, filter filters.Component
 		return err
 	}
 	defer os.Remove(tmpDir)
-	tmpPath := filepath.Join(tmpDir, "data.tar.zst")
+	tmpPath := ""
 
 	isPartial := false
 	switch u.Scheme {
 	case "oci":
-		isPartial, err = pullOCI(ctx, src, tmpPath, shasum, filter)
+		isPartial, tmpPath, err = pullOCI(ctx, src, tmpDir, shasum, filter)
 		if err != nil {
 			return err
 		}
 	case "http", "https":
-		err := pullHTTP(ctx, src, tmpPath, shasum)
+		tmpPath, err = pullHTTP(ctx, src, tmpDir, shasum)
 		if err != nil {
 			return err
 		}
@@ -101,10 +102,10 @@ func Pull(ctx context.Context, src, dir, shasum string, filter filters.Component
 	return nil
 }
 
-func pullOCI(ctx context.Context, src, tarPath, shasum string, filter filters.ComponentFilterStrategy) (bool, error) {
+func pullOCI(ctx context.Context, src, tarDir, shasum string, filter filters.ComponentFilterStrategy) (bool, string, error) {
 	tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory)
 	if err != nil {
-		return false, err
+		return false, "", err
 	}
 	defer os.Remove(tmpDir)
 	if shasum != "" {
@@ -113,54 +114,98 @@ func pullOCI(ctx context.Context, src, tarPath, shasum string, filter filters.Co
 	arch := config.GetArch()
 	remote, err := zoci.NewRemote(ctx, src, oci.PlatformForArch(arch))
 	if err != nil {
-		return false, err
+		return false, "", err
 	}
 	desc, err := remote.ResolveRoot(ctx)
 	if err != nil {
-		return false, fmt.Errorf("could not fetch images index: %w", err)
+		return false, "", fmt.Errorf("could not fetch images index: %w", err)
 	}
 	layersToPull := []ocispec.Descriptor{}
 	isPartial := false
+	tarPath := filepath.Join(tarDir, "data.tar")
+	pkg, err := remote.FetchZarfYAML(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	if !pkg.Metadata.Uncompressed {
+		tarPath = fmt.Sprintf("%s.zst", tarPath)
+	}
 	if supportsFiltering(desc.Platform) {
 		root, err := remote.FetchRoot(ctx)
 		if err != nil {
-			return false, err
+			return false, "", err
 		}
 		if len(root.Layers) != len(layersToPull) {
 			isPartial = true
 		}
-		pkg, err := remote.FetchZarfYAML(ctx)
-		if err != nil {
-			return false, err
-		}
 		pkg.Components, err = filter.Apply(pkg)
 		if err != nil {
-			return false, err
+			return false, "", err
 		}
 		layersToPull, err = remote.LayersFromRequestedComponents(ctx, pkg.Components)
 		if err != nil {
-			return false, err
+			return false, "", err
 		}
 	}
 	_, err = remote.PullPackage(ctx, tmpDir, config.CommonOptions.OCIConcurrency, layersToPull...)
 	if err != nil {
-		return false, err
+		return false, "", err
 	}
 	allTheLayers, err := filepath.Glob(filepath.Join(tmpDir, "*"))
 	if err != nil {
-		return false, err
+		return false, "", err
 	}
 	err = archiver.Archive(allTheLayers, tarPath)
 	if err != nil {
-		return false, err
+		return false, "", err
 	}
-	return isPartial, nil
+	return isPartial, tarPath, nil
 }
 
-func pullHTTP(ctx context.Context, src, tarPath, shasum string) error {
+func pullHTTP(ctx context.Context, src, tarDir, shasum string) (string, error) {
 	if shasum == "" {
-		return errors.New("shasum cannot be empty")
+		return "", errors.New("shasum cannot be empty")
+	}
+	tarPath := filepath.Join(tarDir, "data")
+
+	err := pullHTTPFile(ctx, src, tarPath)
+	if err != nil {
+		return "", err
+	}
+
+	received, err := helpers.GetSHA256OfFile(tarPath)
+	if err != nil {
+		return "", err
+	}
+	if received != shasum {
+		return "", fmt.Errorf("shasum mismatch for file %s, expected %s but got %s", tarPath, shasum, received)
+	}
+
+	mtype, err := mimetype.DetectFile(tarPath)
+	if err != nil {
+		return "", err
+	}
+
+	newPath := filepath.Join(tarDir, "data.tar")
+
+	if mtype.Is("application/x-tar") {
+		err = os.Rename(tarPath, newPath)
+		if err != nil {
+			return "", err
+		}
+		return newPath, nil
+	} else if mtype.Is("application/zstd") {
+		newPath = fmt.Sprintf("%s.zst", newPath)
+		err = os.Rename(tarPath, newPath)
+		if err != nil {
+			return "", err
+		}
+		return newPath, nil
 	}
+	return "", fmt.Errorf("unsupported file type: %s", mtype.Extension())
+}
+
+func pullHTTPFile(ctx context.Context, src, tarPath string) error {
 	f, err := os.Create(tarPath)
 	if err != nil {
 		return err
@@ -186,13 +231,6 @@ func pullHTTP(ctx context.Context, src, tarPath, shasum string) error {
 	if err != nil {
 		return err
 	}
-	received, err := helpers.GetSHA256OfFile(tarPath)
-	if err != nil {
-		return err
-	}
-	if received != shasum {
-		return fmt.Errorf("shasum mismatch for file %s, expected %s but got %s", tarPath, shasum, received)
-	}
 	return nil
 }
 
@@ -236,6 +274,9 @@ func nameFromMetadata(path string) (string, error) {
 	} else if pkg.Metadata.Version != "" {
 		name = fmt.Sprintf("%s-%s", name, pkg.Metadata.Version)
 	}
+	if pkg.Metadata.Uncompressed {
+		return fmt.Sprintf("%s.tar", name), nil
+	}
 	return fmt.Sprintf("%s.tar.zst", name), nil
 }
 

src/internal/packager2/pull_test.go

@@ -50,6 +50,61 @@ func TestPull(t *testing.T) {
 	require.Equal(t, packageData, pulledData)
 }
 
+func TestPullUncompressed(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.TestContext(t)
+	packagePath := "./testdata/uncompressed/zarf-package-test-uncompressed-amd64-0.0.1.tar"
+	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
+		file, err := os.Open(packagePath)
+		if err != nil {
+			rw.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		//nolint:errcheck // ignore
+		io.Copy(rw, file)
+	}))
+	t.Cleanup(func() {
+		srv.Close()
+	})
+
+	dir := t.TempDir()
+	shasum := "a118a4d306acc5dd4eab2c161e78fa3dfd1e08ae1e1794a4393be98c79257f5c"
+	err := Pull(ctx, srv.URL, dir, shasum, filters.Empty(), "", false)
+	require.NoError(t, err)
+
+	packageData, err := os.ReadFile(packagePath)
+	require.NoError(t, err)
+	pulledPath := filepath.Join(dir, "zarf-package-test-uncompressed-amd64-0.0.1.tar")
+	pulledData, err := os.ReadFile(pulledPath)
+	require.NoError(t, err)
+	require.Equal(t, packageData, pulledData)
+}
+
+func TestPullUnsupported(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.TestContext(t)
+	packagePath := "./testdata/uncompressed/zarf.yaml"
+	srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
+		file, err := os.Open(packagePath)
+		if err != nil {
+			rw.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		//nolint:errcheck // ignore
+		io.Copy(rw, file)
+	}))
+	t.Cleanup(func() {
+		srv.Close()
+	})
+
+	dir := t.TempDir()
+	shasum := "6e9dccce07ba9d3c45b7c872fae863c5415d296fd5e2fb72a2583530aa750ccd"
+	err := Pull(ctx, srv.URL, dir, shasum, filters.Empty(), "", false)
+	require.EqualError(t, err, "unsupported file type: .txt", "unsupported file type: .txt")
+}
+
 func TestSupportsFiltering(t *testing.T) {
 	t.Parallel()
 

src/internal/packager2/testdata/uncompressed/zarf.yaml

@@ -0,0 +1,16 @@
+kind: ZarfPackageConfig
+metadata:
+  name: test-uncompressed
+  version: 0.0.1
+  architecture: amd64
+  uncompressed: true
+components:
+  - name: test
+    required: true
+    manifests:
+      - name: deployment
+        namespace: nginx
+        files:
+          - ../deployment.yaml
+    images:
+      - docker.io/library/alpine:3.20