diff --git a/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go b/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go index 199682b26..c12206662 100644 --- a/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go +++ b/server/datacatalog/datacatalogv2/datacatalogv2adapter/type.go @@ -496,7 +496,7 @@ func cityFrom(d datacatalogv2.DataCatalogItem) *plateauapi.City { func citygmlFrom(d datacatalogv2.DataCatalogItem) *plateauapi.CityGMLDataset { id, code := cityIDFrom(d), cityCodeFrom(d) - if id == nil || code == nil || d.CityGMLURL == "" || d.Spec == "" || len(d.CityGMLFeatureTypes) == 0 { + if id == nil || code == nil || d.CityGMLURL == "" || d.MaxLODURL == "" || d.Spec == "" || len(d.CityGMLFeatureTypes) == 0 { return nil } @@ -514,7 +514,6 @@ func citygmlFrom(d datacatalogv2.DataCatalogItem) *plateauapi.CityGMLDataset { Admin: map[string]any{ "maxlod": d.MaxLODURL, }, - // Items: items, } } diff --git a/server/datacatalog/datacatalogv3/repo.go b/server/datacatalog/datacatalogv3/repo.go deleted file mode 100644 index 807dad795..000000000 --- a/server/datacatalog/datacatalogv3/repo.go +++ /dev/null @@ -1,162 +0,0 @@ -package datacatalogv3 - -import ( - "context" - "fmt" - "slices" - "sort" - "time" - - "github.com/eukarya-inc/reearth-plateauview/server/datacatalog/plateauapi" - cms "github.com/reearth/reearth-cms-api/go" - "github.com/reearth/reearthx/log" - "github.com/reearth/reearthx/util" - "github.com/samber/lo" -) - -const cacheUpdateDuration = 10 * time.Second - -var stagesForAdmin = []string{string(stageBeta)} - -type Repos struct { - locks util.LockMap[string] - cms map[string]*CMS - context map[string]*plateauapi.InMemoryRepoContext - repos map[string]*plateauapi.RepoWrapper - adminRepos map[string]*plateauapi.RepoWrapper - warnings map[string][]string - updatedAt map[string]time.Time - now func() time.Time -} - -func NewRepos() *Repos { - return &Repos{ - locks: util.LockMap[string]{}, - cms: map[string]*CMS{}, - context: map[string]*plateauapi.InMemoryRepoContext{}, - repos: map[string]*plateauapi.RepoWrapper{}, - adminRepos: map[string]*plateauapi.RepoWrapper{}, - warnings: map[string][]string{}, - updatedAt: map[string]time.Time{}, - } -} - -func (r *Repos) Prepare(ctx context.Context, project string, year int, cms cms.Interface) error { - if r.cms[project] != nil { - return nil - } - - r.setCMS(project, year, cms) - return r.Update(ctx, project) -} - -func (r *Repos) Repo(project string, admin bool) *plateauapi.RepoWrapper { - if admin { - return r.adminRepos[project] - } - return r.repos[project] -} - -func (r *Repos) Projects() []string { - keys := lo.Keys(r.repos) - sort.Strings(keys) - return keys -} - -func (r *Repos) UpdateAll(ctx context.Context) error { - projects := r.Projects() - for _, project := range projects { - if err := r.Update(ctx, project); err != nil { - return fmt.Errorf("failed to update project %s: %w", project, err) - } - } - return nil -} - -func (r *Repos) Update(ctx context.Context, project string) error { - r.locks.Lock(project) - defer r.locks.Unlock(project) - - updated := r.UpdatedAt(project) - var updatedStr string - if !updated.IsZero() { - updatedStr = updated.Format(time.RFC3339) - } - - // avoid too frequent updates - since := r.getNow().Sub(updated) - if !updated.IsZero() && since < cacheUpdateDuration { - log.Infofc(ctx, "datacatalogv3: skip updating repo %s: last_update=%s, since=%s", project, updatedStr, since) - return nil - } - - cms := r.cms[project] - if cms == nil { - return fmt.Errorf("cms is not initialized for %s", project) - } - - log.Infofc(ctx, "datacatalogv3: updating repo %s: last_update=%s", project, updatedStr) - data, err := cms.GetAll(ctx, project) - if err != nil { - return err - } - - c, warning := data.Into() - sort.Strings(warning) - r.warnings[project] = warning - r.context[project] = c - - repo := plateauapi.NewInMemoryRepo(c) - adminRepo := plateauapi.NewInMemoryRepo(c) - adminRepo.SetAdmin(true) - adminRepo.SetIncludedStages(stagesForAdmin...) - - adminRepoWrapper := r.adminRepos[project] - if adminRepoWrapper == nil { - adminRepoWrapper = plateauapi.NewRepoWrapper(adminRepo, nil) - adminRepoWrapper.SetName(fmt.Sprintf("%s(admin)", project)) - r.adminRepos[project] = adminRepoWrapper - } else { - adminRepoWrapper.SetRepo(adminRepo) - } - - repoWrapper := r.repos[project] - if repoWrapper == nil { - repoWrapper = plateauapi.NewRepoWrapper(repo, nil) - repoWrapper.SetName(project) - r.repos[project] = repoWrapper - } else { - repoWrapper.SetRepo(repo) - } - - r.updatedAt[project] = r.getNow() - - log.Infofc(ctx, "datacatalogv3: updated repo %s", project) - return nil -} - -func (r *Repos) Warnings(project string) []string { - if r.UpdatedAt(project).IsZero() { - return []string{"project is not initialized"} - } - return slices.Clone(r.warnings[project]) -} - -func (r *Repos) UpdatedAt(project string) time.Time { - return r.updatedAt[project] -} - -func (r *Repos) setCMS(project string, year int, cms cms.Interface) { - r.locks.Lock(project) - defer r.locks.Unlock(project) - - c := NewCMS(cms, year) - r.cms[project] = c -} - -func (r *Repos) getNow() time.Time { - if r.now != nil { - return r.now() - } - return time.Now() -} diff --git a/server/datacatalog/datacatalogv3/repos.go b/server/datacatalog/datacatalogv3/repos.go new file mode 100644 index 000000000..9873c8d05 --- /dev/null +++ b/server/datacatalog/datacatalogv3/repos.go @@ -0,0 +1,77 @@ +package datacatalogv3 + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/eukarya-inc/reearth-plateauview/server/datacatalog/plateauapi" + cms "github.com/reearth/reearth-cms-api/go" + "github.com/reearth/reearthx/log" + "github.com/reearth/reearthx/util" +) + +var stagesForAdmin = []string{string(stageBeta)} + +type Repos struct { + cms *util.SyncMap[string, *CMS] + *plateauapi.Repos +} + +func NewRepos() *Repos { + r := &Repos{ + cms: util.NewSyncMap[string, *CMS](), + } + r.Repos = plateauapi.NewRepos(r.update) + return r +} + +func (r *Repos) Prepare(ctx context.Context, project string, year int, cms cms.Interface) error { + if _, ok := r.cms.Load(project); ok { + return nil + } + + r.setCMS(project, year, cms) + _, err := r.Update(ctx, project) + return err +} + +func (r *Repos) update(ctx context.Context, project string) (*plateauapi.ReposUpdateResult, error) { + cms, ok := r.cms.Load(project) + if !ok { + return nil, fmt.Errorf("cms is not initialized for %s", project) + } + + updated := r.UpdatedAt(project) + var updatedStr string + if !updated.IsZero() { + updatedStr = updated.Format(time.RFC3339) + } + log.Infofc(ctx, "datacatalogv3: updating repo %s: last_update=%s", project, updatedStr) + + data, err := cms.GetAll(ctx, project) + if err != nil { + return nil, err + } + + c, warning := data.Into() + sort.Strings(warning) + + repo := plateauapi.NewInMemoryRepo(c) + + adminRepo := plateauapi.NewInMemoryRepo(c) + adminRepo.SetAdmin(true) + adminRepo.SetIncludedStages(stagesForAdmin...) + + return &plateauapi.ReposUpdateResult{ + Repo: repo, + AdminRepo: adminRepo, + Warnings: warning, + }, nil +} + +func (r *Repos) setCMS(project string, year int, cms cms.Interface) { + c := NewCMS(cms, year) + r.cms.Store(project, c) +} diff --git a/server/datacatalog/datacatalogv3/repo_test.go b/server/datacatalog/datacatalogv3/repos_test.go similarity index 100% rename from server/datacatalog/datacatalogv3/repo_test.go rename to server/datacatalog/datacatalogv3/repos_test.go diff --git a/server/datacatalog/plateauapi/inmemory_wrapper.go b/server/datacatalog/plateauapi/repo_wrapper.go similarity index 100% rename from server/datacatalog/plateauapi/inmemory_wrapper.go rename to server/datacatalog/plateauapi/repo_wrapper.go diff --git a/server/datacatalog/plateauapi/repos.go b/server/datacatalog/plateauapi/repos.go new file mode 100644 index 000000000..7dbb3490b --- /dev/null +++ b/server/datacatalog/plateauapi/repos.go @@ -0,0 +1,145 @@ +package plateauapi + +import ( + "context" + "fmt" + "slices" + "sort" + "time" + + cms "github.com/reearth/reearth-cms-api/go" + "github.com/reearth/reearthx/util" + "github.com/samber/lo" +) + +const cacheUpdateDuration = 10 * time.Second + +type ReposUpdater = func(ctx context.Context, project string) (*ReposUpdateResult, error) + +type ReposUpdateResult struct { + Repo Repo + AdminRepo Repo + Warnings []string +} + +type Repos struct { + updater ReposUpdater + locks util.LockMap[string] + repos map[string]*RepoWrapper + adminRepos map[string]*RepoWrapper + warnings map[string][]string + updatedAt map[string]time.Time + now func() time.Time +} + +func NewRepos(u ReposUpdater) *Repos { + return &Repos{ + updater: u, + locks: util.LockMap[string]{}, + repos: map[string]*RepoWrapper{}, + adminRepos: map[string]*RepoWrapper{}, + warnings: map[string][]string{}, + updatedAt: map[string]time.Time{}, + } +} + +func (r *Repos) Prepare(ctx context.Context, project string, year int, cms cms.Interface) error { + _, err := r.Update(ctx, project) + return err +} + +func (r *Repos) Repo(project string, admin bool) *RepoWrapper { + if admin { + return r.adminRepos[project] + } + return r.repos[project] +} + +func (r *Repos) Projects() []string { + keys := lo.Keys(r.repos) + sort.Strings(keys) + return keys +} + +func (r *Repos) UpdateAll(ctx context.Context) error { + projects := r.Projects() + for _, project := range projects { + if _, err := r.Update(ctx, project); err != nil { + return fmt.Errorf("failed to update project %s: %w", project, err) + } + } + return nil +} + +// Update updates the project's repo if it's not updated recently. If false is returned, it means the repo is not updated. +func (r *Repos) Update(ctx context.Context, project string) (bool, error) { + r.locks.Lock(project) + defer r.locks.Unlock(project) + + // avoid too frequent updates + updated := r.UpdatedAt(project) + since := r.getNow().Sub(updated) + if !updated.IsZero() && since < cacheUpdateDuration { + return false, nil + } + + // update + ur, err := r.updater(ctx, project) + if err != nil { + return false, fmt.Errorf("failed to update project %s: %w", project, err) + } + + u := false + if ur != nil { + if ur.AdminRepo != nil { + adminRepoWrapper := r.adminRepos[project] + if adminRepoWrapper == nil { + adminRepoWrapper = NewRepoWrapper(ur.AdminRepo, nil) + adminRepoWrapper.SetName(fmt.Sprintf("%s(admin)", project)) + r.adminRepos[project] = adminRepoWrapper + } else { + adminRepoWrapper.SetRepo(ur.AdminRepo) + } + + u = true + } + + if ur.Repo != nil { + repoWrapper := r.repos[project] + if repoWrapper == nil { + repoWrapper = NewRepoWrapper(ur.Repo, nil) + repoWrapper.SetName(project) + r.repos[project] = repoWrapper + } else { + repoWrapper.SetRepo(ur.Repo) + } + + u = true + } + + r.warnings[project] = ur.Warnings + } + + if u { + r.updatedAt[project] = r.getNow() + } + return u, nil +} + +func (r *Repos) Warnings(project string) []string { + if r.UpdatedAt(project).IsZero() { + return []string{"project is not initialized"} + } + return slices.Clone(r.warnings[project]) +} + +func (r *Repos) UpdatedAt(project string) time.Time { + return r.updatedAt[project] +} + +func (r *Repos) getNow() time.Time { + if r.now != nil { + return r.now() + } + return time.Now() +} diff --git a/server/datacatalog/repos.go b/server/datacatalog/repos.go index 171d4cfb1..82df7fa8d 100644 --- a/server/datacatalog/repos.go +++ b/server/datacatalog/repos.go @@ -311,7 +311,7 @@ func (h *reposHandler) updateV2(ctx context.Context, prj string) error { } func (h *reposHandler) updateV3(ctx context.Context, prj string) error { - if err := h.reposv3.Update(ctx, prj); err != nil { + if _, err := h.reposv3.Update(ctx, prj); err != nil { return fmt.Errorf("datacatalogv3: failed to update repo %s: %w", prj, err) } return nil diff --git a/server/go.mod b/server/go.mod index 79b2bbc30..b75b6a500 100644 --- a/server/go.mod +++ b/server/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/99designs/gqlgen v0.17.43 github.com/dustin/go-humanize v1.0.1 - github.com/eukarya-inc/jpareacode v1.0.1-0.20240313051948-672368075753 + github.com/eukarya-inc/jpareacode v1.0.1-0.20240314080116-ae89cfd85c6a github.com/go-playground/validator/v10 v10.16.0 github.com/hasura/go-graphql-client v0.12.1 github.com/jarcoal/httpmock v1.3.1 diff --git a/server/go.sum b/server/go.sum index 40412ba7a..7982a43eb 100644 --- a/server/go.sum +++ b/server/go.sum @@ -86,6 +86,10 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBF github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/eukarya-inc/jpareacode v1.0.1-0.20240313051948-672368075753 h1:5Ot2vk7/ekNWb1fpjM2Jkg3jPku7i7rtXFnW0g8/T0g= github.com/eukarya-inc/jpareacode v1.0.1-0.20240313051948-672368075753/go.mod h1:thkfAUkPUNwQq/0zySwRf9sQ/8hIFtUMLOfDsiZcI5I= +github.com/eukarya-inc/jpareacode v1.0.1-0.20240314074357-38fd8c41e046 h1:Lzz8cKc0ozP78T8Nc+MljkdAMEJ0tYA+beVc0k+abtI= +github.com/eukarya-inc/jpareacode v1.0.1-0.20240314074357-38fd8c41e046/go.mod h1:thkfAUkPUNwQq/0zySwRf9sQ/8hIFtUMLOfDsiZcI5I= +github.com/eukarya-inc/jpareacode v1.0.1-0.20240314080116-ae89cfd85c6a h1:ZEntf2oouZdXmtmcuUbA00RNzaiGzJPaK/vQFqPqZE0= +github.com/eukarya-inc/jpareacode v1.0.1-0.20240314080116-ae89cfd85c6a/go.mod h1:thkfAUkPUNwQq/0zySwRf9sQ/8hIFtUMLOfDsiZcI5I= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=