From 937242fdf30153b0a33ec315e20b0fb0a0341367 Mon Sep 17 00:00:00 2001 From: rot1024 Date: Thu, 8 Feb 2024 13:10:12 +0900 Subject: [PATCH] fix(server): veg datasets were wrongly delivered from datacatalogv3 --- .../{cms_test.go => model_test.go} | 0 server/datacatalog/datacatalogv3/cms_model.go | 2 + .../datacatalogv3/cms_model_test.go | 7 -- .../datacatalogv3/conv_asset_test.go | 21 ++++ .../datacatalog/datacatalogv3/conv_dataset.go | 10 +- .../datacatalogv3/conv_dataset_generic.go | 2 +- .../datacatalogv3/conv_dataset_plateau.go | 2 +- .../conv_dataset_plateau_seed.go | 107 ++++++++++-------- .../conv_dataset_plateau_test.go | 101 +++++++++++++++++ .../datacatalogv3/conv_dataset_related.go | 4 +- .../datacatalogv3/conv_dataset_test.go | 12 +- .../datacatalogv3/conv_internal.go | 17 +++ 12 files changed, 215 insertions(+), 70 deletions(-) rename server/cmsintegration/cmsintegrationv3/geospatialjpv3/{cms_test.go => model_test.go} (100%) diff --git a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/cms_test.go b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/model_test.go similarity index 100% rename from server/cmsintegration/cmsintegrationv3/geospatialjpv3/cms_test.go rename to server/cmsintegration/cmsintegrationv3/geospatialjpv3/model_test.go diff --git a/server/datacatalog/datacatalogv3/cms_model.go b/server/datacatalog/datacatalogv3/cms_model.go index 7f383327a..ca074eaf2 100644 --- a/server/datacatalog/datacatalogv3/cms_model.go +++ b/server/datacatalog/datacatalogv3/cms_model.go @@ -134,6 +134,8 @@ type PlateauFeatureItemDatum struct { Name string `json:"name,omitempty" cms:"name,text"` Desc string `json:"desc,omitempty" cms:"desc,textarea"` Key string `json:"key,omitempty" cms:"key,text"` + // Simple indicates that this item should not use subcode and subname + Simple bool `json:"simple,omitempty" cms:"-"` } type Dic map[string][]DicEntry // admin, fld. htd, tnm, urf, gen diff --git a/server/datacatalog/datacatalogv3/cms_model_test.go b/server/datacatalog/datacatalogv3/cms_model_test.go index 9e6ad82a5..3f88042df 100644 --- a/server/datacatalog/datacatalogv3/cms_model_test.go +++ b/server/datacatalog/datacatalogv3/cms_model_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "testing" - "github.com/k0kubun/pp/v3" cms "github.com/reearth/reearth-cms-api/go" "github.com/stretchr/testify/assert" ) @@ -195,9 +194,3 @@ func TestStringOrNumber(t *testing.T) { assert.NoError(t, json.Unmarshal([]byte(`123.456`), stringOrNumber)) assert.Equal(t, "123.456000", stringOrNumber.String()) } - -func TestUrf(t *testing.T) { - ppp := pp.New() - ppp.SetColoringEnabled(false) - t.Log(ppp.Sprint(UrfFeatureTypeMap)) -} diff --git a/server/datacatalog/datacatalogv3/conv_asset_test.go b/server/datacatalog/datacatalogv3/conv_asset_test.go index 18ab036e4..618982d8f 100644 --- a/server/datacatalog/datacatalogv3/conv_asset_test.go +++ b/server/datacatalog/datacatalogv3/conv_asset_test.go @@ -273,6 +273,27 @@ func TestParseAssetName(t *testing.T) { }, }, }, + { + name: "veg", + args: "11111_bar-shi_city_2023_citygml_1_op_veg_PlantCover_3dtiles_lod3", + want: &AssetName{ + CityCode: "11111", + CityName: "bar-shi", + Provider: "city", + Year: 2023, + Format: "citygml", + UpdateCount: 1, + Ex: AssetNameEx{ + Urf: &AssetNameExUrf{ + Type: "veg", + Format: "3dtiles", + LOD: 3, + Name: "PlantCover", + }, + Ex: "veg_PlantCover_3dtiles_lod3", + }, + }, + }, { name: "invalid ex", args: "26100_kyoto-shi_city_2023_citygml_1_op_exex", diff --git a/server/datacatalog/datacatalogv3/conv_dataset.go b/server/datacatalog/datacatalogv3/conv_dataset.go index e978c0489..c0e11bebf 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset.go +++ b/server/datacatalog/datacatalogv3/conv_dataset.go @@ -106,17 +106,17 @@ func detectDatasetFormatFromURL(url string) plateauapi.DatasetFormat { return "" } -func standardItemID(name string, area plateauapi.Area, ex string) string { +func standardItemID(name string, areaCode plateauapi.AreaCode, ex string) string { if ex != "" { ex = fmt.Sprintf("_%s", ex) } - return fmt.Sprintf("%s_%s%s", area.GetCode(), name, ex) + return fmt.Sprintf("%s_%s%s", areaCode, name, ex) } -func standardItemName(name, subname string, area plateauapi.Area) string { +func standardItemName(name, subname, areaName string) string { var suffix string - if area != nil { - suffix = fmt.Sprintf("（%s）", area.GetName()) + if areaName != "" { + suffix = fmt.Sprintf("（%s）", areaName) name = strings.TrimSuffix(name, suffix) } diff --git a/server/datacatalog/datacatalogv3/conv_dataset_generic.go b/server/datacatalog/datacatalogv3/conv_dataset_generic.go index caca44a19..b5f55240f 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset_generic.go +++ b/server/datacatalog/datacatalogv3/conv_dataset_generic.go @@ -55,7 +55,7 @@ func (i *GenericItem) toDatasets(area *areaContext, dts []plateauapi.DatasetType res := plateauapi.GenericDataset{ ID: id, - Name: standardItemName(i.Name, "", area.City), + Name: standardItemName(i.Name, "", area.Name()), Description: lo.EmptyableToPtr(i.Desc), Year: area.CityItem.YearInt(), PrefectureID: area.PrefID, diff --git a/server/datacatalog/datacatalogv3/conv_dataset_plateau.go b/server/datacatalog/datacatalogv3/conv_dataset_plateau.go index d6f4a825b..b6683accb 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset_plateau.go +++ b/server/datacatalog/datacatalogv3/conv_dataset_plateau.go @@ -90,7 +90,7 @@ func seedToDataset(seed plateauDatasetSeed) (res *plateauapi.PlateauDataset, war res = &plateauapi.PlateauDataset{ ID: id, - Name: standardItemName(seed.DatasetType.Name, seed.Subname, seed.TargetArea), + Name: standardItemName(seed.DatasetType.Name, seed.Subname, seed.TargetArea.GetName()), Subname: lo.EmptyableToPtr(seed.Subname), Subcode: lo.EmptyableToPtr(seed.Subcode), Suborder: seed.Suborder, diff --git a/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go b/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go index b40dc0b07..d0290f8d3 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go +++ b/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go @@ -32,7 +32,7 @@ type plateauDatasetSeed struct { } func (seed plateauDatasetSeed) GetID() string { - return standardItemID(seed.DatasetType.Code, seed.TargetArea, seed.Subcode) + return standardItemID(seed.DatasetType.Code, seed.TargetArea.GetCode(), seed.Subcode) } func plateauDatasetSeedsFrom(i *PlateauFeatureItem, dt *plateauapi.PlateauDatasetType, area *areaContext, spec *plateauapi.PlateauSpecMinor, layerNames LayerNames, cmsurl string) (res []plateauDatasetSeed, warning []string) { @@ -54,8 +54,9 @@ func plateauDatasetSeedsFrom(i *PlateauFeatureItem, dt *plateauapi.PlateauDatase if len(i.Data) > 0 { items = []PlateauFeatureItemDatum{ { - Data: i.Data, - Desc: i.Desc, + Data: i.Data, + Desc: i.Desc, + Simple: true, }, } } @@ -112,62 +113,66 @@ func plateauDatasetSeedsFromItem(item PlateauFeatureItemDatum, dt *plateauapi.Pl return a.B }) - assetName := assetNames[0] - key, dickey := assetName.Ex.DatasetKey(), assetName.Ex.DicKey() - var e *DicEntry - - if dickey != "" { - var found bool - e, found = dic.FindEntryOrDefault(dt.Code, dickey) - if !found { - warning = append(warning, fmt.Sprintf("plateau %s %s: unknown dic key: %s", cityCode, dt.Code, dickey)) - if e == nil { - return + res = plateauDatasetSeed{ + AssetURLs: assetUrls, + Assets: assetNames, + Desc: item.Desc, + } + + if !item.Simple { + assetName := res.Assets[0] + key, dickey := assetName.Ex.DatasetKey(), assetName.Ex.DicKey() + var e *DicEntry + + if dickey != "" { + var found bool + e, found = dic.FindEntryOrDefault(dt.Code, dickey) + if !found { + warning = append(warning, fmt.Sprintf("plateau %s %s: unknown dic key: %s", cityCode, dt.Code, dickey)) + if e == nil { + return + } } } - } - var river *plateauapi.River - if assetName.Ex.Fld != nil { - if a := riverAdminFrom(assetName.Ex.Fld.Admin); a != nil { - if e == nil || e.Description == "" { - warning = append(warning, fmt.Sprintf("plateau %s %s: dic entry has no description or entry not found: %s", cityCode, dt.Code, key)) - } else { - river = &plateauapi.River{ - Name: e.Description, - Admin: *a, + var river *plateauapi.River + if assetName.Ex.Fld != nil { + if a := riverAdminFrom(assetName.Ex.Fld.Admin); a != nil { + if e == nil || e.Description == "" { + warning = append(warning, fmt.Sprintf("plateau %s %s: dic entry has no description or entry not found: %s", cityCode, dt.Code, key)) + } else { + river = &plateauapi.River{ + Name: e.Description, + Admin: *a, + } } } } - } - subname := item.Name - if subname == "" && e != nil { - if river != nil { - // fld - subname = fmt.Sprintf("%s（%s管理区間）", e.Description, toRiverAdminName(river.Admin)) - } else { - subname = e.Description + subname := item.Name + if subname == "" && e != nil { + if river != nil { + // fld + subname = fmt.Sprintf("%s（%s管理区間）", e.Description, toRiverAdminName(river.Admin)) + } else { + subname = e.Description + } + } + if subname == "" && e != nil { + warning = append(warning, fmt.Sprintf("plateau %s %s: invalid dic entry: %s", cityCode, dt.Code, key)) + } + + var suborder *int + if e != nil { + suborder = e.Order } - } - if subname == "" && e != nil { - warning = append(warning, fmt.Sprintf("plateau %s %s: invalid dic entry: %s", cityCode, dt.Code, key)) - } - var suborder *int - if e != nil { - suborder = e.Order + res.Subcode = key + res.Suborder = suborder + res.Subname = subname + res.River = river } - res = plateauDatasetSeed{ - Subcode: key, - Suborder: suborder, - AssetURLs: assetUrls, - Assets: assetNames, - Subname: subname, - Desc: item.Desc, - River: river, - } return } @@ -236,6 +241,10 @@ type plateauDatasetItemSeed struct { } func (i plateauDatasetItemSeed) GetID(parentID string) string { + if i.ID != "" { + parentID = strings.TrimSuffix(parentID, "_"+i.ID) + } + ids := []string{parentID, i.ID} if i.LOD != nil { @@ -326,6 +335,7 @@ func plateauDatasetItemSeedFromNormal(url string, ex *AssetNameExNormal, layerNa } return &plateauDatasetItemSeed{ + ID: "", Name: "", // use default URL: assetURLFromFormat(url, format), Format: format, @@ -358,6 +368,7 @@ func plateauDatasetItemSeedFromUrf(url string, ex *AssetNameExUrf, dic Dic, laye } return &plateauDatasetItemSeed{ + ID: key, Name: entry.Description, URL: assetURLFromFormat(url, format), Format: format, diff --git a/server/datacatalog/datacatalogv3/conv_dataset_plateau_test.go b/server/datacatalog/datacatalogv3/conv_dataset_plateau_test.go index 4fdb55183..0f38bc0d2 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset_plateau_test.go +++ b/server/datacatalog/datacatalogv3/conv_dataset_plateau_test.go @@ -538,3 +538,104 @@ func TestPlateauDataset_ToDatasets_Fld(t *testing.T) { assert.Nil(t, warning) assert.Equal(t, expected, res) } + +func TestPlateauDataset_ToDatasets_Veg(t *testing.T) { + item := &PlateauFeatureItem{ + ID: "id", + Desc: "desc", + Data: []string{ + "https://example.com/11111_bar-shi_city_2023_citygml_1_op_veg_PlantCover_3dtiles_lod3.zip", + "https://example.com/11111_bar-shi_city_2023_citygml_1_op_veg_SolitaryVegetationObject_3dtiles_lod3.zip", + }, + Dic: `{ + "veg" : [ + { + "name" : "PlantCover", + "description" : "植被" + }, + { + "name" : "SolitaryVegetationObject", + "description" : "単独木" + } + ] + }`, + } + + expected := []plateauapi.Dataset{ + &plateauapi.PlateauDataset{ + ID: plateauapi.NewID("11111_veg", plateauapi.TypeDataset), + Name: "植生モデル（bar市）", + Description: lo.ToPtr("desc"), + Year: 2023, + PrefectureID: lo.ToPtr(plateauapi.NewID("11", plateauapi.TypeArea)), + PrefectureCode: lo.ToPtr(plateauapi.AreaCode("11")), + CityID: lo.ToPtr(plateauapi.NewID("11111", plateauapi.TypeArea)), + CityCode: lo.ToPtr(plateauapi.AreaCode("11111")), + TypeID: plateauapi.NewID("veg", plateauapi.TypeDatasetType), + TypeCode: "veg", + PlateauSpecMinorID: plateauapi.NewID("3.2", plateauapi.TypePlateauSpec), + Admin: map[string]any{ + "stage": string(stageBeta), + }, + Items: []*plateauapi.PlateauDatasetItem{ + { + ID: plateauapi.NewID("11111_veg_PlantCover_lod3", plateauapi.TypeDatasetItem), + Format: plateauapi.DatasetFormatCesium3dtiles, + Name: "植被 LOD3", + URL: "https://example.com/11111_bar-shi_city_2023_citygml_1_op_veg_PlantCover_3dtiles_lod3/tileset.json", + Texture: lo.ToPtr(plateauapi.TextureTexture), + Lod: lo.ToPtr(3), + ParentID: plateauapi.NewID("11111_veg", plateauapi.TypeDataset), + }, + { + ID: plateauapi.NewID("11111_veg_SolitaryVegetationObject_lod3", plateauapi.TypeDatasetItem), + Format: plateauapi.DatasetFormatCesium3dtiles, + Name: "単独木 LOD3", + URL: "https://example.com/11111_bar-shi_city_2023_citygml_1_op_veg_SolitaryVegetationObject_3dtiles_lod3/tileset.json", + Texture: lo.ToPtr(plateauapi.TextureTexture), + Lod: lo.ToPtr(3), + ParentID: plateauapi.NewID("11111_veg", plateauapi.TypeDataset), + }, + }, + }, + } + + area := &areaContext{ + Pref: &plateauapi.Prefecture{}, + City: &plateauapi.City{ + Name: "bar市", + Code: "11111", + }, + PrefID: lo.ToPtr(plateauapi.NewID("11", plateauapi.TypeArea)), + CityID: lo.ToPtr(plateauapi.NewID("11111", plateauapi.TypeArea)), + PrefCode: lo.ToPtr(plateauapi.AreaCode("11")), + CityCode: lo.ToPtr(plateauapi.AreaCode("11111")), + CityItem: &CityItem{ + Year: "2023年", + PlateauDataStatus: &cms.Tag{ + Name: string(ManagementStatusReady), + }, + }, + } + + dts := &plateauapi.PlateauDatasetType{ + ID: plateauapi.NewID("veg", plateauapi.TypeDatasetType), + Code: "veg", + Name: "植生モデル", + } + + spec := &plateauapi.PlateauSpecMinor{ + ID: plateauapi.NewID("3.2", plateauapi.TypePlateauSpec), + Name: "第3.2版", + MajorVersion: 3, + Version: "3.2", + Year: 2023, + ParentID: plateauapi.NewID("3", plateauapi.TypePlateauSpec), + } + + layerNames := LayerNames{} + + res, warning := item.toDatasets(area, dts, spec, layerNames, "") + assert.Nil(t, warning) + assert.Equal(t, expected, res) +} diff --git a/server/datacatalog/datacatalogv3/conv_dataset_related.go b/server/datacatalog/datacatalogv3/conv_dataset_related.go index e20c73a73..b038a5af8 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset_related.go +++ b/server/datacatalog/datacatalogv3/conv_dataset_related.go @@ -34,7 +34,7 @@ func (i *RelatedItem) toDatasets(area *areaContext, dts []plateauapi.DatasetType admin := newAdmin(area.CityItem.ID, relatedStage(i, area.CityItem), cmsurl) for _, seed := range seeds { - sid := standardItemID(ftcode, seed.Area, "") + sid := standardItemID(ftcode, seed.Area.GetCode(), "") id := plateauapi.NewID(sid, plateauapi.TypeDataset) var ou *string @@ -44,7 +44,7 @@ func (i *RelatedItem) toDatasets(area *areaContext, dts []plateauapi.DatasetType res = append(res, &plateauapi.RelatedDataset{ ID: id, - Name: standardItemName(ftname, "", seed.Area), + Name: standardItemName(ftname, "", seed.Area.GetName()), Description: lo.EmptyableToPtr(d.Description), Year: area.CityItem.YearInt(), PrefectureID: area.PrefID, diff --git a/server/datacatalog/datacatalogv3/conv_dataset_test.go b/server/datacatalog/datacatalogv3/conv_dataset_test.go index 31ff2dc19..af8578890 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset_test.go +++ b/server/datacatalog/datacatalogv3/conv_dataset_test.go @@ -24,10 +24,10 @@ func TestAssetRootPath(t *testing.T) { } func TestStandardItemName(t *testing.T) { - assert.Equal(t, "name（area）", standardItemName("name", "", &plateauapi.City{Name: "area"})) - assert.Equal(t, "name name2（area）", standardItemName("name", "name2", &plateauapi.City{Name: "area"})) - assert.Equal(t, "name", standardItemName("name", "", nil)) - assert.Equal(t, "name name2", standardItemName("name", "name2", nil)) - assert.Equal(t, "name（area）", standardItemName("name（area）", "", &plateauapi.City{Name: "area"})) - assert.Equal(t, "name name2（area）", standardItemName("name（area）", "name2", &plateauapi.City{Name: "area"})) + assert.Equal(t, "name（area）", standardItemName("name", "", "area")) + assert.Equal(t, "name name2（area）", standardItemName("name", "name2", "area")) + assert.Equal(t, "name", standardItemName("name", "", "")) + assert.Equal(t, "name name2", standardItemName("name", "name2", "")) + assert.Equal(t, "name（area）", standardItemName("name（area）", "", "area")) + assert.Equal(t, "name name2（area）", standardItemName("name（area）", "name2", "area")) } diff --git a/server/datacatalog/datacatalogv3/conv_internal.go b/server/datacatalog/datacatalogv3/conv_internal.go index cc1bab1f2..3a47d9514 100644 --- a/server/datacatalog/datacatalogv3/conv_internal.go +++ b/server/datacatalog/datacatalogv3/conv_internal.go @@ -105,6 +105,23 @@ func (c *areaContext) IsValid() bool { return c.Pref != nil && c.City != nil && c.CityItem != nil && c.PrefID != nil && c.CityID != nil && c.PrefCode != nil && c.CityCode != nil } +func (c *areaContext) Code() *plateauapi.AreaCode { + if c.CityCode != nil { + return c.CityCode + } + return c.PrefCode +} + +func (c *areaContext) Name() string { + if c.City != nil { + return c.City.Name + } + if c.Pref != nil { + return c.Pref.Name + } + return "" +} + func (c *internalContext) AreaContext(cityItemID string) *areaContext { var prefID, cityID *plateauapi.ID var prefCode, cityCode *plateauapi.AreaCode