diff --git a/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache.go b/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache.go index 7da45d6af..c76fd7c6b 100644 --- a/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache.go +++ b/server/datacatalog/datacatalogv2/datacatalogv2adapter/cache.go @@ -18,7 +18,7 @@ func New(cmsbase, project string) (*plateauapi.RepoWrapper, error) { } func From(fetcher datacatalogv2.Fetchable, project string) *plateauapi.RepoWrapper { - return plateauapi.NewRepoWrapper(nil, func(ctx context.Context, repo *plateauapi.Repo) error { + r := plateauapi.NewRepoWrapper(nil, func(ctx context.Context, repo *plateauapi.Repo) error { r, err := fetchAndCreateCache(ctx, project, fetcher, datacatalogv2.FetcherDoOptions{}) if err != nil { return err @@ -26,6 +26,8 @@ func From(fetcher datacatalogv2.Fetchable, project string) *plateauapi.RepoWrapp *repo = r return nil }) + r.SetName(fmt.Sprintf("%s(v2)", project)) + return r } func fetchAndCreateCache(ctx context.Context, project string, fetcher datacatalogv2.Fetchable, opts datacatalogv2.FetcherDoOptions) (*plateauapi.InMemoryRepo, error) { diff --git a/server/datacatalog/datacatalogv3/cms.go b/server/datacatalog/datacatalogv3/cms.go index 7e16b563e..80646869b 100644 --- a/server/datacatalog/datacatalogv3/cms.go +++ b/server/datacatalog/datacatalogv3/cms.go @@ -18,7 +18,9 @@ func NewCMS(cms cms.Interface) *CMS { } func (c *CMS) GetAll(ctx context.Context, project string) (*AllData, error) { - all := AllData{} + all := AllData{ + Name: project, + } // TODO: get CMSInfo diff --git a/server/datacatalog/datacatalogv3/conv.go b/server/datacatalog/datacatalogv3/conv.go index 9b5c6a31b..7aaf20909 100644 --- a/server/datacatalog/datacatalogv3/conv.go +++ b/server/datacatalog/datacatalogv3/conv.go @@ -9,6 +9,7 @@ import ( func (all *AllData) Into() (res *plateauapi.InMemoryRepoContext, warning []string) { res = &plateauapi.InMemoryRepoContext{ + Name: all.Name, Areas: plateauapi.Areas{}, Datasets: plateauapi.Datasets{}, } diff --git a/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go b/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go index d13bd02fc..ca7068b00 100644 --- a/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go +++ b/server/datacatalog/datacatalogv3/conv_dataset_plateau_seed.go @@ -41,12 +41,22 @@ func plateauDatasetSeedsFrom(i *PlateauFeatureItem, dt *plateauapi.PlateauDatase return } - if dt.Code == "bldg" && len(i.Items) == 0 { + items := i.Items + if len(i.Data) > 0 { + items = append(items, lo.Map(i.Data, func(url string, _ int) PlateauFeatureItemDatum { + return PlateauFeatureItemDatum{ + Data: []string{url}, + Desc: i.Desc, + } + })...) + } + + if dt.Code == "bldg" { seeds, w := plateauDatasetSeedsFromBldg(i, dt, cityCode, area.Wards) warning = append(warning, w...) res = append(res, seeds...) } else { - for _, item := range i.Items { + for _, item := range items { seeds, w := plateauDatasetSeedsFromItem(i, item, dt, dic, cityCode) warning = append(warning, w...) res = append(res, seeds) @@ -70,13 +80,13 @@ func plateauDatasetSeedsFrom(i *PlateauFeatureItem, dt *plateauapi.PlateauDatase } func plateauDatasetSeedsFromItem(i *PlateauFeatureItem, item PlateauFeatureItemDatum, dt *plateauapi.PlateauDatasetType, dic Dic, cityCode string) (res plateauDatasetSeed, warning []string) { - assets := lo.Map(item.Data, func(url string, _ int) *AssetName { + assets := lo.FilterMap(item.Data, func(url string, _ int) (*AssetName, bool) { n := nameWithoutExt(nameFromURL(url)) an := ParseAssetName(n) if an == nil || !an.Ex.IsValid() { warning = append(warning, fmt.Sprintf("plateau %s %s: invalid asset name: %s", cityCode, dt.Code, n)) } - return an + return an, an != nil }) if len(assets) == 0 { // warning = append(warning, fmt.Sprintf("plateau %s %s: no assets", cityCode, dt.Code)) @@ -86,7 +96,7 @@ func plateauDatasetSeedsFromItem(i *PlateauFeatureItem, item PlateauFeatureItemD assetName := assets[0] key, dickey := assetName.Ex.ItemKey(), assetName.Ex.DicKey() if key == "" || dickey == "" { - warning = append(warning, fmt.Sprintf("plateau %s %s: invalid asset name key: %s", cityCode, dt.Code, assets[0].Ex.Ex)) + warning = append(warning, fmt.Sprintf("plateau %s %s: invalid asset name key: %s", cityCode, dt.Code, assetName.Ex.Ex)) return } diff --git a/server/datacatalog/datacatalogv3/model.go b/server/datacatalog/datacatalogv3/model.go index b1dd2571a..7d7383ca9 100644 --- a/server/datacatalog/datacatalogv3/model.go +++ b/server/datacatalog/datacatalogv3/model.go @@ -3,6 +3,7 @@ package datacatalogv3 import "github.com/eukarya-inc/reearth-plateauview/server/datacatalog/plateauapi" type AllData struct { + Name string PlateauSpecs []plateauapi.PlateauSpecSimple FeatureTypes FeatureTypes City []*CityItem diff --git a/server/datacatalog/datacatalogv3/repo.go b/server/datacatalog/datacatalogv3/repo.go index 428570edf..3ce3b0ba6 100644 --- a/server/datacatalog/datacatalogv3/repo.go +++ b/server/datacatalog/datacatalogv3/repo.go @@ -79,7 +79,7 @@ func (r *Repos) Update(ctx context.Context, project string, rawcms cms.Interface r.cms[project] = cms } - log.Infofc(ctx, "datacatalogv3: updating project %s", project) + log.Infofc(ctx, "datacatalogv3: updating repo %s", project) data, err := cms.GetAll(ctx, project) if err != nil { return err @@ -97,6 +97,7 @@ func (r *Repos) Update(ctx context.Context, project string, rawcms cms.Interface adminRepoWrapper := r.adminRepos[project] if adminRepoWrapper == nil { adminRepoWrapper = plateauapi.NewRepoWrapper(adminRepo, nil) + adminRepoWrapper.SetName(fmt.Sprintf("%s(admin)", project)) r.adminRepos[project] = adminRepoWrapper } else { adminRepoWrapper.SetRepo(adminRepo) @@ -105,6 +106,7 @@ func (r *Repos) Update(ctx context.Context, project string, rawcms cms.Interface repoWrapper := r.repos[project] if repoWrapper == nil { repoWrapper = plateauapi.NewRepoWrapper(repo, nil) + repoWrapper.SetName(project) r.repos[project] = repoWrapper } else { repoWrapper.SetRepo(repo) @@ -116,7 +118,7 @@ func (r *Repos) Update(ctx context.Context, project string, rawcms cms.Interface r.updatedAt[project] = time.Now() } - log.Infofc(ctx, "datacatalogv3: updated project %s", project) + log.Infofc(ctx, "datacatalogv3: updated repo %s", project) return nil } diff --git a/server/datacatalog/plateauapi/generated.go b/server/datacatalog/plateauapi/generated.go index 397f0eb58..fa05d2d9f 100644 --- a/server/datacatalog/plateauapi/generated.go +++ b/server/datacatalog/plateauapi/generated.go @@ -285,6 +285,8 @@ type PlateauDatasetResolver interface { City(ctx context.Context, obj *PlateauDataset) (*City, error) Ward(ctx context.Context, obj *PlateauDataset) (*Ward, error) Type(ctx context.Context, obj *PlateauDataset) (*PlateauDatasetType, error) + + PlateauSpecMinor(ctx context.Context, obj *PlateauDataset) (*PlateauSpecMinor, error) } type PlateauDatasetItemResolver interface { Parent(ctx context.Context, obj *PlateauDatasetItem) (*PlateauDataset, error) @@ -4847,7 +4849,7 @@ func (ec *executionContext) _PlateauDataset_plateauSpecMinor(ctx context.Context }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.PlateauSpecMinor, nil + return ec.resolvers.PlateauDataset().PlateauSpecMinor(rctx, obj) }) if err != nil { ec.Error(ctx, err) @@ -4868,8 +4870,8 @@ func (ec *executionContext) fieldContext_PlateauDataset_plateauSpecMinor(ctx con fc = &graphql.FieldContext{ Object: "PlateauDataset", Field: field, - IsMethod: false, - IsResolver: false, + IsMethod: true, + IsResolver: true, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { case "id": @@ -12482,10 +12484,41 @@ func (ec *executionContext) _PlateauDataset(ctx context.Context, sel ast.Selecti atomic.AddUint32(&out.Invalids, 1) } case "plateauSpecMinor": - out.Values[i] = ec._PlateauDataset_plateauSpecMinor(ctx, field, obj) - if out.Values[i] == graphql.Null { - atomic.AddUint32(&out.Invalids, 1) + field := field + + innerFunc := func(ctx context.Context, fs *graphql.FieldSet) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._PlateauDataset_plateauSpecMinor(ctx, field, obj) + if res == graphql.Null { + atomic.AddUint32(&fs.Invalids, 1) + } + return res + } + + if field.Deferrable != nil { + dfs, ok := deferred[field.Deferrable.Label] + di := 0 + if ok { + dfs.AddField(field) + di = len(dfs.Values) - 1 + } else { + dfs = graphql.NewFieldSet([]graphql.CollectedField{field}) + deferred[field.Deferrable.Label] = dfs + } + dfs.Concurrently(di, func(ctx context.Context) graphql.Marshaler { + return innerFunc(ctx, dfs) + }) + + // don't run the out.Concurrently() call below + out.Values[i] = graphql.Null + continue } + + out.Concurrently(i, func(ctx context.Context) graphql.Marshaler { return innerFunc(ctx, out) }) case "river": out.Values[i] = ec._PlateauDataset_river(ctx, field, obj) default: @@ -15026,6 +15059,10 @@ func (ec *executionContext) marshalNPlateauSpec2ᚖgithubᚗcomᚋeukaryaᚑinc return ec._PlateauSpec(ctx, sel, v) } +func (ec *executionContext) marshalNPlateauSpecMinor2githubᚗcomᚋeukaryaᚑincᚋreearthᚑplateauviewᚋserverᚋdatacatalogᚋplateauapiᚐPlateauSpecMinor(ctx context.Context, sel ast.SelectionSet, v PlateauSpecMinor) graphql.Marshaler { + return ec._PlateauSpecMinor(ctx, sel, &v) +} + func (ec *executionContext) marshalNPlateauSpecMinor2ᚕᚖgithubᚗcomᚋeukaryaᚑincᚋreearthᚑplateauviewᚋserverᚋdatacatalogᚋplateauapiᚐPlateauSpecMinorᚄ(ctx context.Context, sel ast.SelectionSet, v []*PlateauSpecMinor) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup diff --git a/server/datacatalog/plateauapi/gqlgen.yml b/server/datacatalog/plateauapi/gqlgen.yml index cd66ff5ee..7c701e1a4 100644 --- a/server/datacatalog/plateauapi/gqlgen.yml +++ b/server/datacatalog/plateauapi/gqlgen.yml @@ -90,7 +90,7 @@ models: resolver: true type: resolver: true - plateauSpec: + plateauSpecMinor: resolver: true PlateauDatasetItem: fields: diff --git a/server/datacatalog/plateauapi/inmemory.go b/server/datacatalog/plateauapi/inmemory.go index 068720579..ed7494851 100644 --- a/server/datacatalog/plateauapi/inmemory.go +++ b/server/datacatalog/plateauapi/inmemory.go @@ -2,6 +2,7 @@ package plateauapi import ( "context" + "fmt" "slices" "github.com/reearth/reearthx/util" @@ -10,6 +11,7 @@ import ( ) type InMemoryRepoContext struct { + Name string Areas Areas DatasetTypes DatasetTypes Datasets Datasets @@ -34,6 +36,13 @@ func NewInMemoryRepo(ctx *InMemoryRepoContext) *InMemoryRepo { return r } +func (c *InMemoryRepo) Name() string { + if c.ctx == nil || c.ctx.Name == "" { + return "inmemory" + } + return fmt.Sprintf("inmemory(%s)", c.ctx.Name) +} + func (c *InMemoryRepo) SetContext(ctx *InMemoryRepoContext) { c.ctx = ctx c.areasForDataTypes = areasForDatasetTypes(ctx.Datasets.All()) diff --git a/server/datacatalog/plateauapi/inmemory_wrapper.go b/server/datacatalog/plateauapi/inmemory_wrapper.go index ad5e8b5a3..3178f5d9b 100644 --- a/server/datacatalog/plateauapi/inmemory_wrapper.go +++ b/server/datacatalog/plateauapi/inmemory_wrapper.go @@ -10,6 +10,7 @@ type RepoUpdater func(ctx context.Context, repo *Repo) error // RepoWrapper is a thread-safe wrapper of Repo. type RepoWrapper struct { repo Repo + name string lock sync.RWMutex updater RepoUpdater } @@ -36,6 +37,10 @@ func (a *RepoWrapper) SetRepo(repo Repo) { a.repo = repo } +func (a *RepoWrapper) SetName(name string) { + a.name = name +} + func (a *RepoWrapper) Update(ctx context.Context) error { if a.updater == nil { return nil @@ -59,6 +64,16 @@ func (a *RepoWrapper) use(f func(r Repo) error) error { var _ Repo = (*RepoWrapper)(nil) +func (a *RepoWrapper) Name() string { + if a.name != "" { + return a.name + } + if a.repo == nil { + return "wrapper" + } + return a.repo.Name() +} + func (a *RepoWrapper) Node(ctx context.Context, id ID) (res Node, err error) { err = a.use(func(r Repo) (err error) { res, err = r.Node(ctx, id) diff --git a/server/datacatalog/plateauapi/merge.go b/server/datacatalog/plateauapi/merge.go index 19a0af9b7..ad267a83f 100644 --- a/server/datacatalog/plateauapi/merge.go +++ b/server/datacatalog/plateauapi/merge.go @@ -2,6 +2,7 @@ package plateauapi import ( "context" + "fmt" "sort" "github.com/reearth/reearthx/util" @@ -19,6 +20,15 @@ func NewMerger(repos ...Repo) *Merger { var _ Repo = (*Merger)(nil) +func (m *Merger) Name() string { + return fmt.Sprintf("merger(%s)", lo.Map(m.repos, func(r Repo, _ int) string { + if r == nil { + return "nil" + } + return r.Name() + })) +} + func (m *Merger) Node(ctx context.Context, id ID) (Node, error) { nodes, err := getRepoResults(m.repos, func(r Repo) (Node, error) { return r.Node(ctx, id) @@ -118,7 +128,11 @@ func getRepoResults[T any](repos []Repo, f func(r Repo) (T, error)) ([]T, error) if r == nil { return } - return f(r) + res, err := f(r) + if err != nil { + return res, fmt.Errorf("repo %s: %w", r.Name(), err) + } + return res, nil }) } @@ -152,6 +166,9 @@ func sortNodes[T IDNode](nodes []T) { } func getLatestYearNode[T any](results []T) T { + results = lo.Filter(results, func(a T, _ int) bool { + return isNodePresent(a) + }) return lo.MaxBy(results, func(a, b T) bool { return getYear(a) > getYear(b) }) @@ -181,6 +198,13 @@ type YearNode interface { GetYear() int } +func isNodePresent(n any) bool { + if n, ok := n.(Node); ok { + return n != nil + } + return false +} + func getYear(n any) int { if yn, ok := n.(YearNode); ok { return yn.GetYear() diff --git a/server/datacatalog/plateauapi/resolver.go b/server/datacatalog/plateauapi/resolver.go index 5b82f92dc..3fd1c420a 100644 --- a/server/datacatalog/plateauapi/resolver.go +++ b/server/datacatalog/plateauapi/resolver.go @@ -35,6 +35,7 @@ var ErrDatacatalogUnavailable = errors.New("datacatalog is currently unavailable type Repo interface { QueryResolver + Name() string } type Resolver struct { diff --git a/server/datacatalog/plateauapi/schema.resolvers.go b/server/datacatalog/plateauapi/schema.resolvers.go index 42094280c..c3e0b9050 100644 --- a/server/datacatalog/plateauapi/schema.resolvers.go +++ b/server/datacatalog/plateauapi/schema.resolvers.go @@ -125,6 +125,11 @@ func (r *plateauDatasetResolver) Type(ctx context.Context, obj *PlateauDataset) return to[*PlateauDatasetType](r.Repo.Node(ctx, obj.TypeID)) } +// PlateauSpecMinor is the resolver for the plateauSpecMinor field. +func (r *plateauDatasetResolver) PlateauSpecMinor(ctx context.Context, obj *PlateauDataset) (*PlateauSpecMinor, error) { + return to[*PlateauSpecMinor](r.Repo.Node(ctx, obj.PlateauSpecMinorID)) +} + // Parent is the resolver for the parent field. func (r *plateauDatasetItemResolver) Parent(ctx context.Context, obj *PlateauDatasetItem) (*PlateauDataset, error) { return to[*PlateauDataset](r.Repo.Node(ctx, obj.ParentID)) @@ -397,16 +402,3 @@ type relatedDatasetResolver struct{ *Resolver } type relatedDatasetItemResolver struct{ *Resolver } type relatedDatasetTypeResolver struct{ *Resolver } type wardResolver struct{ *Resolver } - -// !!! WARNING !!! -// The code below was going to be deleted when updating resolvers. It has been copied here so you have -// one last chance to move it out of harms way if you want. There are two reasons this happens: -// - When renaming or deleting a resolver the old code will be put in here. You can safely delete -// it when you're done. -// - You have helper methods in this file. Move them out to keep these resolver files clean. -func (r *plateauDatasetResolver) PlateauSpecMinor(ctx context.Context, obj *PlateauDataset) (*PlateauSpecMinor, error) { - return to[*PlateauSpecMinor](r.Repo.Node(ctx, obj.PlateauSpecMinorID)) -} -func (r *plateauDatasetResolver) PlateauSpec(ctx context.Context, obj *PlateauDataset) (*PlateauSpecMinor, error) { - return to[*PlateauSpecMinor](r.Repo.Node(ctx, obj.PlateauSpecMinorID)) -} diff --git a/server/datacatalog/v3.go b/server/datacatalog/v3.go index 1c1ccf62e..5cd70a56e 100644 --- a/server/datacatalog/v3.go +++ b/server/datacatalog/v3.go @@ -39,19 +39,25 @@ func echov3(conf Config, g *echo.Group) (func(ctx context.Context) error, error) // GraphQL playground (all) plateauapig.GET("/graphql", gqlPlaygroundHandler(conf.PlaygroundEndpoint, false)) + // GraphQL playground (all, admin) + plateauapig.GET("/admin/graphql", gqlPlaygroundHandler(conf.PlaygroundEndpoint, true)) + // GraphQL playground (project) plateauapig.GET("/:pid/graphql", gqlPlaygroundHandler(conf.PlaygroundEndpoint, false)) - // GraphQL playground (admin) + // GraphQL playground (project, admin) plateauapig.GET("/:pid/admin/graphql", gqlPlaygroundHandler(conf.PlaygroundEndpoint, true)) // GraphQL API (all) plateauapig.POST("/graphql", h.Handler(false)) + // GraphQL API (all, admin) + plateauapig.POST("/admin/graphql", h.Handler(true)) + // GraphQL API (project) plateauapig.POST("/:pid/graphql", h.Handler(false)) - // GraphQL API (admin) + // GraphQL API (project, admin) plateauapig.POST("/:pid/admin/graphql", h.Handler(true)) // warning API @@ -209,7 +215,11 @@ func (h *reposHandler) Init(ctx context.Context) error { return m.DataCatalogSchemaVersion == cmsSchemaVersion }) - log.Infofc(ctx, "datacatalogv3: initializing repos for %d projects", len(target)) + targetLen := len(target) + if h.repov2 != nil { + targetLen++ + } + log.Infofc(ctx, "datacatalogv3: initializing repos for %d projects", targetLen) for _, md := range target { cms, err := md.CMS() @@ -223,6 +233,13 @@ func (h *reposHandler) Init(ctx context.Context) error { } } + // repov2 + log.Infofc(ctx, "datacatalogv3: updating v2 repo") + if err := h.repov2.Update(ctx); err != nil { + log.Errorfc(ctx, "datacatalogv3: failed to update v2 repo: %w", err) + } + + log.Infofc(ctx, "datacatalogv3: updated v2 repos") return nil } diff --git a/tools/src/args.rs b/tools/src/args.rs index ce2a1e3c5..a644a4a2c 100644 --- a/tools/src/args.rs +++ b/tools/src/args.rs @@ -23,6 +23,13 @@ pub enum Commands { #[clap(short, long)] output: Option, }, + /// PLATEAU VIEW 1.1 のS3内に格納されたユースケースデータのファイルを移行します。 + MigrateUC { + #[clap(short, long)] + list_path: PathBuf, + #[clap(short, long)] + output: Option, + }, } #[derive(Debug, Clone, ValueEnum)] diff --git a/tools/src/main.rs b/tools/src/main.rs index cf4ecfe76..06f9cb079 100644 --- a/tools/src/main.rs +++ b/tools/src/main.rs @@ -1,4 +1,5 @@ mod args; +mod migrateuc; mod prepare; use args::{Cli, Commands}; @@ -17,6 +18,9 @@ fn main() { output, format: format.into(), }), + Commands::MigrateUC { list_path, output } => { + migrateuc::migrateuc(migrateuc::Config { list_path, output }) + } } { eprintln!("{}", err); std::process::exit(1); diff --git a/tools/src/migrateuc.rs b/tools/src/migrateuc.rs new file mode 100644 index 000000000..890eb6dba --- /dev/null +++ b/tools/src/migrateuc.rs @@ -0,0 +1,13 @@ +use std::{fs::create_dir_all, path::PathBuf}; + +pub struct Config { + pub list_path: PathBuf, + pub output: Option, +} + +pub fn migrateuc(config: Config) -> anyhow::Result<()> { + let output = config.output.unwrap_or_else(|| PathBuf::from("uc")); + create_dir_all(output)?; + + todo!() +}