fix(worker): fix unzipping data in prepare-gspatialjp

worker/preparegspatialjp/citygml.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	cms "github.com/reearth/reearth-cms-api/go"
 	"github.com/reearth/reearthx/log"
@@ -67,7 +68,7 @@ func getUdx(ctx context.Context, allFeatureItems map[string]FeatureItem, downloa
 			return fmt.Errorf("failed to download citygml for %s: %w", ft, err)
 		}
 
-		if err := Unzip(ctx, data, outPath, ""); err != nil {
+		if err := Unzip(ctx, data, outPath, "", checkCityGMLZip); err != nil {
 			return fmt.Errorf("failed to unzip citygml for %s: %w", ft, err)
 		}
 	}
@@ -89,7 +90,7 @@ func getAssets(ctx context.Context, cms *cms.CMS, cityItem *CityItem, downloadPa
 			return fmt.Errorf("failed to download assets codeLists: %w", err)
 		}
 
-		if err := Unzip(ctx, data, downloadPath, ""); err != nil {
+		if err := Unzip(ctx, data, downloadPath, "", nil); err != nil {
 			return fmt.Errorf("failed to unzip assets codeLists: %w", err)
 		}
 	}
@@ -108,7 +109,7 @@ func getAssets(ctx context.Context, cms *cms.CMS, cityItem *CityItem, downloadPa
 			return fmt.Errorf("failed to download assets schemas: %w", err)
 		}
 
-		if err := Unzip(ctx, data, downloadPath, ""); err != nil {
+		if err := Unzip(ctx, data, downloadPath, "", nil); err != nil {
 			return fmt.Errorf("failed to unzip assets schemas: %w", err)
 		}
 	}
@@ -127,7 +128,7 @@ func getAssets(ctx context.Context, cms *cms.CMS, cityItem *CityItem, downloadPa
 			return fmt.Errorf("failed to download assets metadata: %w", err)
 		}
 
-		if err := Unzip(ctx, data, downloadPath, ""); err != nil {
+		if err := Unzip(ctx, data, downloadPath, "", nil); err != nil {
 			return fmt.Errorf("failed to unzip assets metadata: %w", err)
 		}
 	}
@@ -146,7 +147,7 @@ func getAssets(ctx context.Context, cms *cms.CMS, cityItem *CityItem, downloadPa
 			return fmt.Errorf("failed to download assets specification: %w", err)
 		}
 
-		if err := Unzip(ctx, data, downloadPath, ""); err != nil {
+		if err := Unzip(ctx, data, downloadPath, "", nil); err != nil {
 			return fmt.Errorf("failed to unzip assets specification: %w", err)
 		}
 	}
@@ -165,10 +166,19 @@ func getAssets(ctx context.Context, cms *cms.CMS, cityItem *CityItem, downloadPa
 			return fmt.Errorf("failed to download assets misc: %w", err)
 		}
 
-		if err := Unzip(ctx, data, downloadPath, "misc"); err != nil {
+		if err := Unzip(ctx, data, downloadPath, "misc/", nil); err != nil {
 			return fmt.Errorf("failed to unzip assets misc: %w", err)
 		}
 	}
 
 	return nil
 }
+
+func checkCityGMLZip(p string) error {
+	if strings.HasPrefix(p, "udx/") ||
+		strings.HasPrefix(p, "/udx/") ||
+		(!strings.Contains(p, "/") && strings.HasSuffix(p, ".gml")) {
+		return fmt.Errorf("invalid citygml zip: %s", p)
+	}
+	return nil
+}

worker/preparegspatialjp/citygml_test.go

@@ -0,0 +1,40 @@
+package preparegspatialjp
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckCityGML(t *testing.T) {
+	tests := []struct {
+		path          string
+		expectedError bool
+	}{
+		{
+			path:          "udx/file.txt",
+			expectedError: true,
+		},
+		{
+			path:          "file.gml",
+			expectedError: true,
+		},
+		{
+			path:          "dir/file.txt",
+			expectedError: false,
+		},
+		{
+			path:          "dir/file.gml",
+			expectedError: false,
+		},
+	}
+
+	for _, test := range tests {
+		result := checkCityGMLZip(test.path)
+		if !test.expectedError {
+			assert.NoError(t, result)
+		} else {
+			assert.Error(t, result)
+		}
+	}
+}

worker/preparegspatialjp/file.go

@@ -68,7 +68,7 @@ func downloadFile(ctx context.Context, url string) (io.ReadCloser, error) {
 	// return bytes.NewReader(b.Bytes()), nil
 }
 
-func Unzip(ctx context.Context, zipFile *bytes.Reader, targetDir string, trimPathSuffix string) error {
+func Unzip(ctx context.Context, zipFile *bytes.Reader, targetDir, prefix string, checkFile func(string) error) error {
 	// Open the zip archive for reading.
 	r, err := zip.NewReader(zipFile, int64(zipFile.Len()))
 	if err != nil {
@@ -77,43 +77,48 @@ func Unzip(ctx context.Context, zipFile *bytes.Reader, targetDir string, trimPat
 
 	// Iterate through the files in the archive.
 	for _, f := range r.File {
-		// Determine the file path ensuring it's within targetDir.
-		filePath := filepath.Join(targetDir, strings.TrimPrefix(f.Name, trimPathSuffix))
-		filePath = strings.TrimSuffix(filePath, `\`)
-		// filePath = filepath.ToSlash(filePath)
-		filePath = strings.ReplaceAll(filePath, `\`, `/`)
-
-		if isInvalid(filePath) {
-			return fmt.Errorf("invalid file path: %s", filePath)
+		filename := f.Name
+		filename = strings.ReplaceAll(filename, `\`, "/")
+		filename = strings.TrimSuffix(filename, "/")
+
+		if prefix != "" {
+			if strings.HasPrefix(filename, prefix) {
+				filename = strings.TrimPrefix(filename, prefix)
+			} else {
+				log.Debugf("skip %s (no prefix: %s)", f.Name, prefix)
+				continue
+			}
 		}
 
-		if strings.HasPrefix(filePath, "__MACOSX/") ||
-			strings.HasPrefix(filePath, "/__MACOSX/") ||
-			strings.HasSuffix(filePath, "/.DS_Store") ||
-			strings.HasSuffix(filePath, "/Thumb.db") ||
-			filePath == ".DS_Store" || filePath == "Thumbs.db" {
-			log.Debugf("skipping %s...", f.Name)
+		if strings.HasPrefix(filename, "__MACOSX/") ||
+			strings.HasSuffix(filename, "/.DS_Store") ||
+			strings.HasSuffix(filename, "/Thumb.db") ||
+			filename == ".DS_Store" || filename == "Thumbs.db" {
+			log.Debugf("skip %s (%s)", f.Name, filename)
 			continue
 		}
 
-		log.Infofc(ctx, "unzipping %s -> %s", f.Name, filePath)
-
-		if trimPathSuffix != "" {
-			filePath = strings.Trim(filePath, trimPathSuffix)
+		if checkFile != nil {
+			if err := checkFile(filename); err != nil {
+				return err
+			}
 		}
 
-		if !strings.HasPrefix(filePath, filepath.Clean(targetDir)) {
-			return fmt.Errorf("invalid file path: %s", filePath)
-		}
+		// Determine the file path ensuring it's within targetDir.
+		filePath := filepath.Join(targetDir,
+			strings.ReplaceAll(filename, "/", string(os.PathSeparator)))
 
 		// If it's a directory, create it.
 		if f.FileInfo().IsDir() {
-			log.Infofc(ctx, "creating dir %s...", f.Name)
+			log.Infofc(ctx, "create dir %s -> %s", f.Name, filePath)
 			if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
 				return fmt.Errorf("failed to create dir: %v", err)
 			}
 			continue
 		}
+
+		log.Infofc(ctx, "extract %s -> %s", f.Name, filePath)
+
 		// Create the enclosing directory if needed.
 		if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
 			return fmt.Errorf("failed to create dir: %v", err)
@@ -146,11 +151,8 @@ func Unzip(ctx context.Context, zipFile *bytes.Reader, targetDir string, trimPat
 		_ = destFile.Close()
 		_ = srcFile.Close()
 	}
-	return nil
-}
 
-func isInvalid(filePath string) bool {
-	return strings.HasPrefix(filePath, "udx/") || (!strings.Contains(filePath, "/") && strings.HasSuffix(filePath, ".gml"))
+	return nil
 }
 
 func ZipDir(ctx context.Context, srcDir string, destZip string) error {

worker/preparegspatialjp/maxlod.go

@@ -58,6 +58,8 @@ func MergeMaxLOD(ctx context.Context, cms *cms.CMS, cityItem *CityItem, allFeatu
 		if err := os.WriteFile(filePath, allData.Bytes(), os.ModePerm); err != nil {
 			return "", "", fmt.Errorf("failed to write data to file: %w", err)
 		}
+	} else {
+		return "", "", nil
 	}
 
 	return fileName, filePath, nil

worker/preparegspatialjp/plateau.go

@@ -41,7 +41,7 @@ func PreparePlateau(ctx context.Context, cms *cms.CMS, cityItem *CityItem, allFe
 				return "", "", fmt.Errorf("failed to download data for %s: %w", ft, err)
 			}
 
-			if err := Unzip(ctx, data, downloadPath, ""); err != nil {
+			if err := Unzip(ctx, data, downloadPath, "", nil); err != nil {
 				return "", "", fmt.Errorf("failed to unzip data for %s: %w", ft, err)
 			}
 		}