From aaa9ed9a19804b2dd83e3ea87f06613be0f0fb24 Mon Sep 17 00:00:00 2001 From: rot1024 Date: Sat, 23 Dec 2023 02:04:44 +0900 Subject: [PATCH] fix(server): related data convertion --- .../cmsintegration/cmsintegrationv3/model.go | 121 +++--- .../cmsintegrationv3/model_test.go | 77 +++- .../cmsintegrationv3/service_related.go | 396 +++++++++++------- .../cmsintegrationv3/service_related_test.go | 89 ++-- .../cmsintegration/cmsintegrationv3/util.go | 6 +- 5 files changed, 437 insertions(+), 252 deletions(-) diff --git a/server/cmsintegration/cmsintegrationv3/model.go b/server/cmsintegration/cmsintegrationv3/model.go index 79a7002a6..4c3be141f 100644 --- a/server/cmsintegration/cmsintegrationv3/model.go +++ b/server/cmsintegration/cmsintegrationv3/model.go @@ -2,7 +2,9 @@ package cmsintegrationv3 import ( "github.com/eukarya-inc/reearth-plateauview/server/cmsintegration/cmsintegrationcommon" + "github.com/oklog/ulid/v2" cms "github.com/reearth/reearth-cms-api/go" + "github.com/samber/lo" ) const modelPrefix = "plateau-" @@ -230,51 +232,51 @@ func (i *GenericItem) CMSItem() *cms.Item { } type RelatedItem struct { - ID string `json:"id,omitempty" cms:"id"` - City string `json:"city,omitempty" cms:"city,reference"` - Assets map[string][]string `json:"assets,omitempty" cms:"-"` - ConvertedAssets map[string][]string `json:"converted,omitempty" cms:"-"` - Merged string `json:"merged,omitempty" cms:"merged,asset"` + ID string `json:"id,omitempty" cms:"id"` + City string `json:"city,omitempty" cms:"city,reference"` + Items map[string]RelatedItemDatum `json:"items,omitempty" cms:"-"` + Merged string `json:"merged,omitempty" cms:"merged,asset"` // metadata - ConvertStatus *cms.Tag `json:"conv_status,omitempty" cms:"conv_status,tag,metadata"` - MergeStatus *cms.Tag `json:"merge_status,omitempty" cms:"merge_status,tag,metadata"` - Public bool `json:"public,omitempty" cms:"public,bool,metadata"` + ConvertStatus map[string]*cms.Tag `json:"conv_status,omitempty" cms:"-"` + MergeStatus *cms.Tag `json:"merge_status,omitempty" cms:"merge_status,tag,metadata"` +} + +type RelatedItemDatum struct { + ID string `json:"id,omitempty" cms:"id"` + Asset []string `json:"asset,omitempty" cms:"asset,asset"` + Converted []string `json:"converted,omitempty" cms:"converted,asset"` + Description string `json:"description,omitempty" cms:"description,textarea"` } func RelatedItemFrom(item *cms.Item) (i *RelatedItem) { i = &RelatedItem{} item.Unmarshal(i) - for _, t := range relatedDataTypes { - v := item.FieldByKey(t).GetValue() - cv := item.FieldByKey(t + "_conv").GetValue() - - var assets []string - if s := v.String(); s != nil { - assets = []string{*s} - } else if s := v.Strings(); s != nil { - assets = s - } + if i.Items == nil { + i.Items = map[string]RelatedItemDatum{} + } + if i.ConvertStatus == nil { + i.ConvertStatus = map[string]*cms.Tag{} + } - var conv []string - if s := cv.String(); s != nil { - conv = []string{*s} - } else if s := cv.Strings(); s != nil { - conv = s + for _, t := range relatedDataTypes { + g := item.FieldByKey(t).GetValue().String() + if g == nil { + continue } - if len(assets) > 0 { - if i.Assets == nil { - i.Assets = map[string][]string{} + if group := item.Group(*g); group != nil && len(group.Fields) > 0 { + i.Items[t] = RelatedItemDatum{ + ID: group.ID, + Asset: group.FieldByKey("asset").GetValue().Strings(), + Converted: group.FieldByKey("conv").GetValue().Strings(), + Description: lo.FromPtr(group.FieldByKey("description").GetValue().String()), } - i.Assets[t] = append(i.Assets[t], assets...) } - if len(conv) > 0 { - if i.ConvertedAssets == nil { - i.ConvertedAssets = map[string][]string{} - } - i.ConvertedAssets[t] = append(i.ConvertedAssets[t], conv...) + tag := item.MetadataFieldByKey(t + "_status").GetValue().Tag() + if tag != nil { + i.ConvertStatus[t] = tag } } @@ -286,29 +288,52 @@ func (i *RelatedItem) CMSItem() *cms.Item { cms.Marshal(i, item) for _, t := range relatedDataTypes { - if asset, ok := i.Assets[t]; ok { + if d, ok := i.Items[t]; ok { + if d.ID == "" { + d.ID = ulid.Make().String() + } + + if len(d.Asset) > 0 { + item.Fields = append(item.Fields, &cms.Field{ + Key: "asset", + Type: "asset", + Value: d.Asset, + Group: d.ID, + }) + } + + if len(d.Converted) > 0 { + item.Fields = append(item.Fields, &cms.Field{ + Key: "conv", + Type: "asset", + Value: d.Converted, + Group: d.ID, + }) + } + + if d.Description != "" { + item.Fields = append(item.Fields, &cms.Field{ + Key: "description", + Type: "textarea", + Value: d.Description, + Group: d.ID, + }) + } + item.Fields = append(item.Fields, &cms.Field{ Key: t, - Type: "asset", - Value: asset, + Type: "group", + Value: d.ID, }) } - if conv, ok := i.ConvertedAssets[t]; ok { - item.Fields = append(item.Fields, &cms.Field{ - Key: t + "_conv", - Type: "asset", - Value: conv, + if tag, ok := i.ConvertStatus[t]; ok { + item.MetadataFields = append(item.MetadataFields, &cms.Field{ + Key: t + "_status", + Type: "tag", + Value: tag.Name, }) } - - // if pub, ok := i.Public[t]; ok { - // item.MetadataFields = append(item.MetadataFields, &cms.Field{ - // Key: t + "_public", - // Type: "bool", - // Value: pub, - // }) - // } } return item diff --git a/server/cmsintegration/cmsintegrationv3/model_test.go b/server/cmsintegration/cmsintegrationv3/model_test.go index 8cb3b977f..91dafa248 100644 --- a/server/cmsintegration/cmsintegrationv3/model_test.go +++ b/server/cmsintegration/cmsintegrationv3/model_test.go @@ -127,80 +127,115 @@ func TestRelatedItemFrom(t *testing.T) { ID: "id", Fields: []*cms.Field{ { - Key: "park", + Key: "asset", Type: "asset", Value: []string{"PARK"}, + Group: "park", }, { - Key: "park_conv", + Key: "conv", Type: "asset", Value: []string{"PARK_CONV"}, + Group: "park", }, { - Key: "landmark", + Key: "park", + Type: "group", + Value: "park", + }, + { + Key: "asset", Type: "asset", Value: []string{"LANDMARK"}, + Group: "landmark", + }, + { + Key: "landmark", + Type: "group", + Value: "landmark", }, }, MetadataFields: []*cms.Field{ { - Key: "conv_status", + Key: "park_status", Type: "tag", Value: map[string]any{"id": "xxx", "name": string(ConvertionStatusSuccess)}, }, { - Key: "public", - Type: "bool", - Value: true, + Key: "merge_status", + Type: "tag", + Value: map[string]any{"id": "xxx", "name": string(ConvertionStatusSuccess)}, }, }, } expected := &RelatedItem{ ID: "id", - Assets: map[string][]string{ - "park": {"PARK"}, - "landmark": {"LANDMARK"}, + Items: map[string]RelatedItemDatum{ + "park": { + ID: "park", + Asset: []string{"PARK"}, + Converted: []string{"PARK_CONV"}, + }, + "landmark": { + ID: "landmark", + Asset: []string{"LANDMARK"}, + }, }, - ConvertedAssets: map[string][]string{ - "park": {"PARK_CONV"}, + ConvertStatus: map[string]*cms.Tag{ + "park": { + ID: "xxx", + Name: string(ConvertionStatusSuccess), + }, }, - ConvertStatus: &cms.Tag{ + MergeStatus: &cms.Tag{ ID: "xxx", Name: string(ConvertionStatusSuccess), }, - Public: true, } expected2 := &cms.Item{ ID: "id", Fields: []*cms.Field{ { - Key: "park", + Key: "asset", Type: "asset", Value: []string{"PARK"}, + Group: "park", }, { - Key: "park_conv", + Key: "conv", Type: "asset", Value: []string{"PARK_CONV"}, + Group: "park", }, { - Key: "landmark", + Key: "park", + Type: "group", + Value: "park", + }, + { + Key: "asset", Type: "asset", Value: []string{"LANDMARK"}, + Group: "landmark", + }, + { + Key: "landmark", + Type: "group", + Value: "landmark", }, }, MetadataFields: []*cms.Field{ { - Key: "conv_status", + Key: "merge_status", Type: "tag", Value: "xxx", }, { - Key: "public", - Type: "bool", - Value: true, + Key: "park_status", + Type: "tag", + Value: string(ConvertionStatusSuccess), }, }, } diff --git a/server/cmsintegration/cmsintegrationv3/service_related.go b/server/cmsintegration/cmsintegrationv3/service_related.go index 640de6e41..bfb3b40b5 100644 --- a/server/cmsintegration/cmsintegrationv3/service_related.go +++ b/server/cmsintegration/cmsintegrationv3/service_related.go @@ -5,12 +5,16 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "path" + "regexp" + "strconv" "strings" "github.com/eukarya-inc/reearth-plateauview/server/cmsintegration/dataconv" geojson "github.com/paulmach/go.geojson" + cms "github.com/reearth/reearth-cms-api/go" "github.com/reearth/reearth-cms-api/go/cmswebhook" "github.com/reearth/reearthx/log" "github.com/samber/lo" @@ -43,6 +47,7 @@ func handleRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Payloa } item := RelatedItemFrom(mainItem) + log.Debugfc(ctx, "cmsintegrationv3: related dataset: %#v", item) if err := convertRelatedDataset(ctx, s, w, item); err != nil { return err @@ -56,40 +61,69 @@ func handleRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Payloa } func convertRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Payload, item *RelatedItem) (err error) { - if tagIsNot(item.ConvertStatus, ConvertionStatusNotStarted) { - log.Debugfc(ctx, "cmsintegrationv3: already converted") - return nil + project := w.ProjectID() + convTargets := make([]string, 0, len(relatedDataConvertionTargets)) + newStatus := map[string]*cms.Tag{} + newItems := map[string]RelatedItemDatum{} + + for _, target := range relatedDataConvertionTargets { + if tagIsNot(item.ConvertStatus[target], ConvertionStatusNotStarted) { + log.Debugfc(ctx, "cmsintegrationv3: already converted") + continue + } + + if len(item.Items[target].Asset) == 0 { + continue + } + + convTargets = append(convTargets, target) + newStatus[target] = tagFrom(ConvertionStatusRunning) } - if !lo.SomeBy(relatedDataConvertionTargets, func(t string) bool { - return len(item.Assets[t]) > 0 - }) { - log.Debugfc(ctx, "cmsintegrationv3: no assets") + if len(convTargets) == 0 { + log.Debugfc(ctx, "cmsintegrationv3: no conv targets") + return nil } - log.Infofc(ctx, "cmsintegrationv3: convertRelatedDataset") + log.Infofc(ctx, "cmsintegrationv3: convertRelatedDataset: %v", convTargets) // update status if _, err := s.CMS.UpdateItem(ctx, item.ID, nil, (&RelatedItem{ - ConvertStatus: tagFrom(ConvertionStatusRunning), + ConvertStatus: newStatus, }).CMSItem().MetadataFields); err != nil { return fmt.Errorf("failed to update item: %w", err) } defer func() { - if err == nil { - return + for k, v := range newStatus { + if tagIs(v, ConvertionStatusRunning) { + newStatus[k] = tagFrom(ConvertionStatusNotStarted) + } } - if _, err := s.CMS.UpdateItem(ctx, item.ID, nil, (&RelatedItem{ - MergeStatus: tagFrom(ConvertionStatusError), - }).CMSItem().MetadataFields); err != nil { - log.Warnf("cmsintegrationv3: failed to update item: %w", err) + // update item + ritem := (&RelatedItem{ + Items: newItems, + ConvertStatus: newStatus, + }).CMSItem() + if _, err2 := s.CMS.UpdateItem(ctx, item.ID, ritem.Fields, ritem.MetadataFields); err2 != nil { + err = fmt.Errorf("failed to update item: %w", err2) } // comment to the item - if err := s.CMS.CommentToItem(ctx, item.ID, "G空間情報センター公開用zipファイルの作成に失敗しました。"); err != nil { - log.Warnf("cmsintegrationv3: failed to add comment: %w", err) + succeeded := lo.EveryBy(convTargets, func(t string) bool { + return tagIs(newStatus[t], ConvertionStatusSuccess) + }) + + var comment string + if succeeded { + comment = "変換が完了しました。" + } else { + comment = fmt.Sprintf("変換が完了しましたが、一部のデータの変換に失敗しました。\n%s", err) + } + + if err2 := s.CMS.CommentToItem(ctx, item.ID, comment); err2 != nil { + err = fmt.Errorf("failed to add comment: %w", err2) } }() @@ -98,74 +132,82 @@ func convertRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Paylo return fmt.Errorf("failed to add comment: %w", err) } - conv := map[string][]string{} - for _, target := range relatedDataConvertionTargets { - if len(item.Assets[target]) == 0 { - continue + for _, target := range convTargets { + log.Debugf("cmsintegrationv3: convert %s: %#v", target) + + d := item.Items[target] + var converror bool + var convassets []string + for _, a := range d.Asset { + newAsset, err2 := convRelatedType(ctx, project, target, a, s) + if err2 != nil { + err = errors.Join(err, fmt.Errorf("%s: %s", target, err2)) + converror = true + } else { + convassets = append(convassets, newAsset) + } } - for _, t := range item.Assets[target] { - asset, err := s.CMS.Asset(ctx, t) - if err != nil { - return fmt.Errorf("failed to get asset (%s): %w", target, err) + if converror { + newStatus[target] = tagFrom(ConvertionStatusError) + } else { + newStatus[target] = tagFrom(ConvertionStatusSuccess) + newItems[target] = RelatedItemDatum{ + ID: d.ID, + Converted: convassets, } + } + } - id := strings.TrimSuffix(path.Base(asset.URL), path.Ext(asset.URL)) - log.Debugf("cmsintegrationv3: convert %s (%s)", target, id) + log.Infofc(ctx, "cmsintegrationv3: convertRelatedDataset: converted: %v", convTargets) + return err +} - // download asset - data, err := s.GETAsBytes(ctx, asset.URL) - if err != nil { - return fmt.Errorf("failed to download asset: %w", err) - } +func convRelatedType(ctx context.Context, project, target, assetID string, s *Services) (_ string, err error) { + asset, err := s.CMS.Asset(ctx, assetID) + if err != nil { + return "", fmt.Errorf("failed to get asset (%s): %w", target, err) + } - fc, err := geojson.UnmarshalFeatureCollection(data) - if err != nil { - return fmt.Errorf("failed to unmarshal asset: %w", err) - } + id := strings.TrimSuffix(path.Base(asset.URL), path.Ext(asset.URL)) + log.Debugf("cmsintegrationv3: convert %s (%s)", target, id) - // conv - var res any - if target == "border" { - res, err = dataconv.ConvertBorder(fc, id) - } else if target == "landmark" || target == "station" { - res, err = dataconv.ConvertLandmark(fc, id) - } - - if err != nil || res == nil { - return fmt.Errorf("failed to convert: %w", err) - } + // download asset + data, err := s.GETAsBytes(ctx, asset.URL) + if err != nil { + return "", fmt.Errorf("failed to download asset: %w", err) + } - uploadBody, err := json.Marshal(res) - if err != nil { - return fmt.Errorf("failed to marshal: %w", err) - } + fc, err := geojson.UnmarshalFeatureCollection(data) + if err != nil { + return "", fmt.Errorf("failed to unmarshal asset: %w", err) + } - // upload - assetID, err := s.CMS.UploadAssetDirectly(ctx, w.ProjectID(), id+".czml", bytes.NewReader(uploadBody)) - if err != nil { - return fmt.Errorf("failed to upload asset: %w", err) - } + // conv + var res any + if target == "border" { + res, err = dataconv.ConvertBorder(fc, id) + } else if target == "landmark" || target == "station" { + res, err = dataconv.ConvertLandmark(fc, id) + } - conv[target] = append(conv[target], assetID) - } + if err != nil || res == nil { + return "", fmt.Errorf("failed to convert: %w", err) } - // update item - ritem := (&RelatedItem{ - ConvertedAssets: conv, - ConvertStatus: tagFrom(ConvertionStatusSuccess), - }).CMSItem() - if _, err := s.CMS.UpdateItem(ctx, item.ID, ritem.Fields, ritem.MetadataFields); err != nil { - return fmt.Errorf("failed to update item: %w", err) + uploadBody, err := json.Marshal(res) + if err != nil { + return "", fmt.Errorf("failed to marshal: %w", err) } - // comment to the item - if err := s.CMS.CommentToItem(ctx, item.ID, "変換に成功しました。"); err != nil { - return fmt.Errorf("failed to add comment: %w", err) + // upload + newAssetID, err := s.CMS.UploadAssetDirectly(ctx, project, id+".czml", bytes.NewReader(uploadBody)) + if err != nil { + return "", fmt.Errorf("failed to upload asset: %w", err) } - return nil + log.Debugf("cmsintegrationv3: converted %s (%s) to %s", target, id, newAssetID) + return newAssetID, nil } func packRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Payload, item *RelatedItem) (err error) { @@ -180,7 +222,7 @@ func packRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Payload, } if missingTypes := lo.Filter(relatedDataTypes, func(t string, _ int) bool { - return len(item.Assets[t]) == 0 + return len(item.Items[t].Asset) == 0 }); len(missingTypes) > 0 { log.Debugfc(ctx, "cmsintegrationv3: there are some missing assets: %v", missingTypes) return nil @@ -195,20 +237,34 @@ func packRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Payload, return fmt.Errorf("failed to update item: %w", err) } + var mergedAssetID string + defer func() { + var status ConvertionStatus if err == nil { - return + status = ConvertionStatusSuccess + } else { + status = ConvertionStatusError } - if _, err := s.CMS.UpdateItem(ctx, item.ID, nil, (&RelatedItem{ - MergeStatus: tagFrom(ConvertionStatusError), - }).CMSItem().MetadataFields); err != nil { - log.Warnf("cmsintegrationv3: failed to update item: %w", err) + newItem := (&RelatedItem{ + Merged: mergedAssetID, + MergeStatus: tagFrom(status), + }).CMSItem() + if _, err := s.CMS.UpdateItem(ctx, item.ID, newItem.Fields, newItem.MetadataFields); err != nil { + log.Errorfc(ctx, "cmsintegrationv3: failed to update item: %w", err) } // comment to the item - if err := s.CMS.CommentToItem(ctx, item.ID, fmt.Sprintf("G空間情報センター公開用zipファイルの作成に失敗しました。%s", err)); err != nil { - log.Warnf("cmsintegrationv3: failed to add comment: %w", err) + var comment string + if err == nil { + comment = "G空間情報センター公開用zipファイルの作成が完了しました。" + } else { + comment = fmt.Sprintf("G空間情報センター公開用zipファイルの作成に失敗しました。\n%s", err) + } + + if err := s.CMS.CommentToItem(ctx, item.ID, comment); err != nil { + log.Errorfc(ctx, "cmsintegrationv3: failed to add comment: %w", err) } }() @@ -224,104 +280,152 @@ func packRelatedDataset(ctx context.Context, s *Services, w *cmswebhook.Payload, } cityItem := CityItemFrom(cityItemRaw) - zipName := fmt.Sprintf("%s_%s_related.zip", cityItem.CityCode, cityItem.CityNameEn) zipbuf := bytes.NewBuffer(nil) zw := zip.NewWriter(zipbuf) - assetPreset := false + assetMerged := false + var year int for _, target := range relatedDataTypes { - name := fmt.Sprintf("%s_%s_%s", cityItem.CityCode, cityItem.CityNameEn, target) - var features []*geojson.Feature - noNeedToWriteAssets := len(item.Assets[target]) == 1 + d := item.Items[target] - for _, t := range item.Assets[target] { - // get asset - asset, err := s.CMS.Asset(ctx, t) - if err != nil { + if len(d.Asset) == 0 { + continue + } - return fmt.Errorf("failed to get asset (%s): %w", target, err) - } + y, err := packRelatedDatasetTarget(ctx, target, d.Asset, zw, cityItem.CityCode, cityItem.CityNameEn, s) + if err != nil { + return err + } - // download asset - data, err := s.GETAsBytes(ctx, asset.URL) - if err != nil { - return fmt.Errorf("failed to download asset (%s): %w", target, err) - } + assetMerged = true + year = y + } - // add to zip - if !noNeedToWriteAssets { - f, err := zw.Create(path.Base(asset.URL)) - if err != nil { - return fmt.Errorf("failed to create zip file (%s): %w", target, err) - } + if !assetMerged { + log.Debugfc(ctx, "cmsintegrationv3: no assets") + return nil + } - if _, err := f.Write(data); err != nil { - return fmt.Errorf("failed to write zip file (%s): %w", target, err) - } - } + if err := zw.Close(); err != nil { + return fmt.Errorf("failed to close zip: %w", err) + } - fc := geojson.NewFeatureCollection() - if err := json.Unmarshal(data, fc); err != nil { - return fmt.Errorf("failed to decode asset (%s): %w", target, err) - } + // upload zip + zipName := fmt.Sprintf("%s_%s_%d_related.zip", cityItem.CityCode, cityItem.CityNameEn, year) + mergedAssetID, err = s.CMS.UploadAssetDirectly(ctx, w.ProjectID(), zipName, zipbuf) + if err != nil { + return fmt.Errorf("failed to upload zip: %w", err) + } - features = append(features, fc.Features...) - assetPreset = true + log.Infofc(ctx, "cmsintegrationv3: packageRelatedDatasetForGeospatialjp: done") + return nil +} + +func packRelatedDatasetTarget(ctx context.Context, target string, assets []string, zw *zip.Writer, cityCode, cityNameEn string, s *Services) (int, error) { + var mergedFeatures []*geojson.Feature + noNeedToWriteAssets := len(assets) == 1 + + var assetName *relatedAssetName + + for i, asset := range assets { + asset, err := s.CMS.Asset(ctx, asset) + if err != nil { + return 0, fmt.Errorf("(%s/%d): アセットが見つかりません: %v", target, i+1, err) } - // merge multiple assets - if len(features) > 0 { - f, err := zw.Create(fmt.Sprintf("%s.geojson", target)) - if err != nil { - return fmt.Errorf("failed to create zip file (%s): %w", target, err) - } + an := parseRelatedAssetName(asset.URL) + if an == nil { + return 0, fmt.Errorf("(%s/%d/%s): ファイル名が命名規則に沿っていません。 \"[市区町村コード5桁]_[市区町村名英名]_[提供事業者名]_[整備年度4桁]_[landmark,shelterなど].geojson\" としてください。: %v", target, i+1, path.Base(asset.URL), err) + } - fc := map[string]any{ - "type": "FeatureCollection", - "name": name, - "features": features, - } - data, err := json.Marshal(fc) + if !noNeedToWriteAssets && an.CityCode == cityCode { + return 0, fmt.Errorf("(%s/%d/%s): アセット名の市区町村コードが全体の市区町村コードと同じです。区ごとに登録する場合はファイル名中のコードを各区のコードにしてください。", target, i+1, path.Base(asset.URL)) + } + + if assetName == nil { + assetName = an + } + + // download asset + data, err := s.GETAsBytes(ctx, asset.URL) + if err != nil { + return 0, fmt.Errorf("failed to download asset (%s): %w", target, err) + } + + // parse GeoJSON + fc := geojson.NewFeatureCollection() + if err := json.Unmarshal(data, fc); err != nil { + return 0, fmt.Errorf("(%s/%d/%s): GeoJSONとして読み込むことができませんでした。正しいGeoJSONかどうかファイルの内容を確認してください。: %v", target, i+1, path.Base(asset.URL), err) + } + + mergedFeatures = append(mergedFeatures, fc.Features...) + + // add to zip + if !noNeedToWriteAssets { + f, err := zw.Create(path.Base(asset.URL)) if err != nil { - return fmt.Errorf("failed to marshal (%s): %w", target, err) + return 0, fmt.Errorf("failed to create zip file (%s): %w", target, err) } if _, err := f.Write(data); err != nil { - return fmt.Errorf("failed to write zip file (%s): %w", target, err) + return 0, fmt.Errorf("failed to write zip file (%s): %w", target, err) } } } - if !assetPreset { - log.Debugfc(ctx, "cmsintegrationv3: no assets") - return nil - } + // merge multiple assets + if len(mergedFeatures) > 0 { + name := fmt.Sprintf("%s_%s_%s_%d_%s", cityCode, cityNameEn, assetName.Provider, assetName.Year, assetName.Type) + f, err := zw.Create(fmt.Sprintf("%s.geojson", name)) + if err != nil { + return 0, fmt.Errorf("failed to create zip file (%s): %w", target, err) + } - if err := zw.Close(); err != nil { - return fmt.Errorf("failed to close zip: %w", err) - } + fc := map[string]any{ + "type": "FeatureCollection", + "name": name, + "features": mergedFeatures, + } + data, err := json.Marshal(fc) + if err != nil { + return 0, fmt.Errorf("failed to marshal (%s): %w", target, err) + } - // upload zip - assetID, err := s.CMS.UploadAssetDirectly(ctx, w.ProjectID(), zipName, zipbuf) - if err != nil { - return fmt.Errorf("failed to upload zip: %w", err) + if _, err := f.Write(data); err != nil { + return 0, fmt.Errorf("failed to write zip file (%s): %w", target, err) + } } - // update item - ritem := (&RelatedItem{ - Merged: assetID, - MergeStatus: tagFrom(ConvertionStatusSuccess), - }).CMSItem() + return assetName.Year, nil +} - if _, err := s.CMS.UpdateItem(ctx, item.ID, ritem.Fields, ritem.MetadataFields); err != nil { - return fmt.Errorf("failed to update item: %w", err) - } +type relatedAssetName struct { + CityCode string + CityName string + Provider string + Year int + Type string + Ext string +} - // comment to the item - if err := s.CMS.CommentToItem(ctx, item.ID, "G空間情報センター公開用zipファイルの作成が完了しました。"); err != nil { - return fmt.Errorf("failed to add comment: %w", err) +var reRelatedAssetName = regexp.MustCompile(`^([0-9]{5})_([^_]+)_([^_]+)_([0-9]+)_(.+)\.(.+)$`) + +func parseRelatedAssetName(name string) *relatedAssetName { + name = path.Base(name) + m := reRelatedAssetName.FindStringSubmatch(name) + if m == nil { + return nil } - return nil + y, _ := strconv.Atoi(m[4]) + + return &relatedAssetName{ + CityCode: m[1], + CityName: m[2], + Provider: m[3], + Year: y, + Type: m[5], + Ext: m[6], + } } diff --git a/server/cmsintegration/cmsintegrationv3/service_related_test.go b/server/cmsintegration/cmsintegrationv3/service_related_test.go index cb4997377..0f918d810 100644 --- a/server/cmsintegration/cmsintegrationv3/service_related_test.go +++ b/server/cmsintegration/cmsintegrationv3/service_related_test.go @@ -50,8 +50,11 @@ func TestConvertRelatedDataset(t *testing.T) { } s := &Services{CMS: c, HTTP: http.DefaultClient} item := &RelatedItem{ - Assets: map[string][]string{ - "border": {"border"}, + Items: map[string]RelatedItemDatum{ + "border": { + ID: "border", + Asset: []string{"border"}, + }, }, } w := &cmswebhook.Payload{ @@ -75,22 +78,28 @@ func TestConvertRelatedDataset(t *testing.T) { nil, { { - Key: "border_conv", + Key: "conv", Type: "asset", Value: []string{"asset"}, + Group: "border", + }, + { + Key: "border", + Type: "group", + Value: "border", }, }, }, updatedFields) assert.Equal(t, [][]*cms.Field{ { - {Key: "conv_status", Type: "tag", Value: string(ConvertionStatusRunning)}, + {Key: "border_status", Type: "tag", Value: string(ConvertionStatusRunning)}, }, { - {Key: "conv_status", Type: "tag", Value: string(ConvertionStatusSuccess)}, + {Key: "border_status", Type: "tag", Value: string(ConvertionStatusSuccess)}, }, }, updatedMetadataFields) assert.Equal(t, []string{"hoge_border.czml"}, uploaded) - assert.Equal(t, []string{"変換を開始しました。", "変換に成功しました。"}, comments) + assert.Equal(t, []string{"変換を開始しました。", "変換が完了しました。"}, comments) }) } @@ -133,8 +142,8 @@ func TestPackRelatedDataset(t *testing.T) { c := &cmsMock{ getItem: func(ctx context.Context, id string, asset bool) (*cms.Item, error) { return (&CityItem{ - CityNameEn: "city", - CityCode: "code", + CityNameEn: "hoge", + CityCode: "00000", }).CMSItem(), nil }, asset: func(ctx context.Context, id string) (*cms.Asset, error) { @@ -162,14 +171,14 @@ func TestPackRelatedDataset(t *testing.T) { s := &Services{CMS: c, HTTP: http.DefaultClient} item := &RelatedItem{ City: "city", - Assets: map[string][]string{ - "shelter": {"shelter"}, - "landmark": {"landmark1", "landmark2"}, - "station": {"station"}, - "park": {"park"}, - "railway": {"railway"}, - "emergency_route": {"emergency_route"}, - "border": {"border"}, + Items: map[string]RelatedItemDatum{ + "shelter": {Asset: []string{"00000_hoge_city_2023_shelter"}}, + "landmark": {Asset: []string{"00001_hoge_city_2023_landmark", "00002_hoge_city_2023_landmark"}}, + "station": {Asset: []string{"00000_hoge_city_2023_station"}}, + "park": {Asset: []string{"00000_hoge_city_2023_park"}}, + "railway": {Asset: []string{"00000_hoge_city_2023_railway"}}, + "emergency_route": {Asset: []string{"00000_hoge_city_2023_emergency_route"}}, + "border": {Asset: []string{"00000_hoge_city_2023_border"}}, }, } w := &cmswebhook.Payload{ @@ -207,7 +216,7 @@ func TestPackRelatedDataset(t *testing.T) { {Key: "merge_status", Type: "tag", Value: string(ConvertionStatusSuccess)}, }, }, updatedMetadataFields) - assert.Equal(t, []string{"code_city_related.zip"}, uploaded) + assert.Equal(t, []string{"00000_hoge_2023_related.zip"}, uploaded) assert.Equal(t, []string{ "G空間情報センター公開用zipファイルの作成を開始しました。", "G空間情報センター公開用zipファイルの作成が完了しました。", @@ -215,44 +224,56 @@ func TestPackRelatedDataset(t *testing.T) { zr, _ := zip.NewReader(bytes.NewReader(uploadedData[0]), int64(len(uploadedData[0]))) assert.Equal(t, []string{ - "shelter.geojson", - "park.geojson", - "landmark1.geojson", - "landmark2.geojson", - "landmark.geojson", - "station.geojson", - "railway.geojson", - "emergency_route.geojson", - "border.geojson", + "00000_hoge_city_2023_shelter.geojson", + "00000_hoge_city_2023_park.geojson", + "00001_hoge_city_2023_landmark.geojson", + "00002_hoge_city_2023_landmark.geojson", + "00000_hoge_city_2023_landmark.geojson", + "00000_hoge_city_2023_station.geojson", + "00000_hoge_city_2023_railway.geojson", + "00000_hoge_city_2023_emergency_route.geojson", + "00000_hoge_city_2023_border.geojson", }, lo.Map(zr.File, func(f *zip.File, _ int) string { return f.Name })) - // assert landmark1.geojson - zf := lo.Must(zr.Open("landmark1.geojson")) + // assert 00001_hoge_city_2023_landmark.geojson + zf := lo.Must(zr.Open("00001_hoge_city_2023_landmark.geojson")) var ge map[string]any _ = json.NewDecoder(zf).Decode(&ge) - assert.Equal(t, mockGeoJSON("landmark1"), ge) + assert.Equal(t, mockGeoJSON("00001_hoge_city_2023_landmark"), ge) - // assert landmark.geojson - zf = lo.Must(zr.Open("landmark.geojson")) + // assert 00000_hoge_city_2023_landmark.geojson + zf = lo.Must(zr.Open("00000_hoge_city_2023_landmark.geojson")) ge = nil _ = json.NewDecoder(zf).Decode(&ge) assert.Equal(t, map[string]any{ "type": "FeatureCollection", - "name": "code_city_landmark", + "name": "00000_hoge_city_2023_landmark", "features": []any{ map[string]any{ "type": "Feature", - "properties": map[string]any{"name": "landmark1"}, + "properties": map[string]any{"name": "00001_hoge_city_2023_landmark"}, "geometry": map[string]any{"type": "Point", "coordinates": []any{0.0, 0.0}}, }, map[string]any{ "type": "Feature", - "properties": map[string]any{"name": "landmark2"}, + "properties": map[string]any{"name": "00002_hoge_city_2023_landmark"}, "geometry": map[string]any{"type": "Point", "coordinates": []any{0.0, 0.0}}, }, }, }, ge) }) } + +func TestParseRelatedAssetName(t *testing.T) { + assert.Equal(t, &relatedAssetName{ + CityCode: "00000", + CityName: "hoge", + Provider: "city", + Year: 2023, + Type: "landmark", + Ext: "geojson", + }, parseRelatedAssetName("https://example.com/00000_hoge_city_2023_landmark.geojson")) + assert.Nil(t, parseRelatedAssetName("https://example.com/0000_hoge_city_2023_landmark.geojson")) +} diff --git a/server/cmsintegration/cmsintegrationv3/util.go b/server/cmsintegration/cmsintegrationv3/util.go index 03eb2ae1a..4c4523bc7 100644 --- a/server/cmsintegration/cmsintegrationv3/util.go +++ b/server/cmsintegration/cmsintegrationv3/util.go @@ -7,9 +7,9 @@ import ( "github.com/reearth/reearth-cms-api/go/cmswebhook" ) -// func tagIs(t *cms.Tag, v fmt.Stringer) bool { -// return t != nil && t.Name == v.String() -// } +func tagIs(t *cms.Tag, v fmt.Stringer) bool { + return t != nil && t.Name == v.String() +} func tagIsNot(t *cms.Tag, v fmt.Stringer) bool { return t != nil && t.Name != v.String()