From 93d31a3674e54010a1feaa7618cba59b1b9d0d1b Mon Sep 17 00:00:00 2001 From: rot1024 Date: Tue, 20 Feb 2024 12:48:04 +0900 Subject: [PATCH] fix(server): support cloud build for preparation in geospatialjpv3 --- .../cmsintegrationcommon/config.go | 7 +- .../cmsintegrationv3/geospatialjpv3/common.go | 9 +- .../geospatialjpv3/prepare.go | 52 ++----- .../geospatialjpv3/prepare_cloudbuild.go | 98 ++++++++++++ .../geospatialjpv3/prepare_cloudrunjobs.go | 50 +++++++ .../geospatialjpv3/prepare_test.go | 20 --- .../geospatialjpv3/webhook.go | 2 +- server/cmsintegration/handler.go | 19 ++- server/config.go | 140 ++++++++++-------- 9 files changed, 264 insertions(+), 133 deletions(-) create mode 100644 server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudbuild.go create mode 100644 server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudrunjobs.go delete mode 100644 server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_test.go diff --git a/server/cmsintegration/cmsintegrationcommon/config.go b/server/cmsintegration/cmsintegrationcommon/config.go index f5b69de03..6517f4c5c 100644 --- a/server/cmsintegration/cmsintegrationcommon/config.go +++ b/server/cmsintegration/cmsintegrationcommon/config.go @@ -23,7 +23,12 @@ type Config struct { // FME v3 FMEURLV3 string // geospatial.jp v3 - GeospatialjpJobName string + GeospatialjpBuildType string + GeospatialjpCloudRunJobsJobName string + GeospatialjpCloudBuildImage string + GeospatialjpCloudBuildMachineType string + GeospatialjpCloudBuildProject string + GeospatialjpCloudBuildRegion string // compat // geospatial.jp v2 diff --git a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/common.go b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/common.go index dc221f6a5..ba1d4e43a 100644 --- a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/common.go +++ b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/common.go @@ -15,7 +15,14 @@ type Config struct { CMSBase string CMSToken string CMSIntegration string - JobName string + BuildType string + // cloud run jobs + CloudRunJobsJobName string + // cloud build image + CloudBuildImage string + CloudBuildMachineType string + CloudBuildProject string + CloudBuildRegion string } var ppp *pp.PrettyPrinter diff --git a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare.go b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare.go index beee2f70e..356d12ceb 100644 --- a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare.go +++ b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare.go @@ -2,45 +2,21 @@ package geospatialjpv3 import ( "context" - - run "cloud.google.com/go/run/apiv2" - - runpb "cloud.google.com/go/run/apiv2/runpb" - "github.com/reearth/reearthx/log" ) -// jobName (Cloud Run Jobs): "projects/" + gcpProjectID + "/locations/" + gcpLocation + "/jobs/plateauview-api-worker" - -func Prepare(ctx context.Context, itemID, projectID, jobName string) error { - log.Debugfc(ctx, "geospatialjp webhook: Prepare: %s", jobName) - - client, err := run.NewJobsClient(ctx) - if err != nil { - log.Debugfc(ctx, "geospatialjp webhook: failed to create run client: %v", err) - return err - } - defer client.Close() - - overrides := runpb.RunJobRequest_Overrides{ - ContainerOverrides: []*runpb.RunJobRequest_Overrides_ContainerOverride{ - {Args: []string{ - "prepare-gspatialjp", - "--city=" + itemID, - "--project=" + projectID, - "--wetrun", - }}, - }} - - req := &runpb.RunJobRequest{ - Name: jobName, - Overrides: &overrides, +func Prepare(ctx context.Context, itemID, projectID string, conf Config) error { + if conf.BuildType == "cloudbuild" { + return prepareOnCloudBuild(ctx, prepareOnCloudBuildConfig{ + City: itemID, + Project: projectID, + CMSURL: conf.CMSBase, + CMSToken: conf.CMSToken, + CloudBuildImage: conf.CloudBuildImage, + CloudBuildMachineType: conf.CloudBuildMachineType, + CloudBuildProject: conf.CloudBuildProject, + CloudBuildRegion: conf.CloudBuildRegion, + }) + } else { + return prepareWithCloudRunJobs(ctx, itemID, projectID, conf.CloudRunJobsJobName) } - - if _, err = client.RunJob(ctx, req); err != nil { - log.Debugfc(ctx, "geospatialjp webhook: failed to run job: %v", err) - return err - } - - log.Debugfc(ctx, "geospatialjp webhook: run job: %v", req) - return nil } diff --git a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudbuild.go b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudbuild.go new file mode 100644 index 000000000..4b1373306 --- /dev/null +++ b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudbuild.go @@ -0,0 +1,98 @@ +package geospatialjpv3 + +import ( + "context" + "path" + + "github.com/reearth/reearthx/log" + "github.com/reearth/reearthx/rerror" + "google.golang.org/api/cloudbuild/v1" +) + +type prepareOnCloudBuildConfig struct { + City string + Project string + CMSURL string + CMSToken string + CloudBuildImage string + CloudBuildMachineType string + CloudBuildProject string + CloudBuildRegion string +} + +const defaultDockerImage = "eukarya/plateauview2-sidecar-worker:latest" + +func prepareOnCloudBuild(ctx context.Context, conf prepareOnCloudBuildConfig) error { + if conf.CloudBuildImage == "" { + conf.CloudBuildImage = defaultDockerImage + } + + log.Debugfc(ctx, "geospatialjp webhook: prepare (cloud build): %s", ppp.Sprint(conf)) + + return runCloudBuild(ctx, CloudBuildConfig{ + Image: conf.CloudBuildImage, + Args: []string{ + "--city=" + conf.City, + "--project=" + conf.Project, + "--wetrun", + }, + Env: []string{ + "REEARTH_CMS_URL=" + conf.CMSURL, + "REEARTH_CMS_TOKEN=" + conf.CMSToken, + }, + MachineType: conf.CloudBuildMachineType, + Project: conf.CloudBuildProject, + Region: conf.CloudBuildRegion, + }) +} + +type CloudBuildConfig struct { + Image string + Args []string + Env []string + MachineType string + Region string + Project string +} + +func runCloudBuild(ctx context.Context, conf CloudBuildConfig) error { + cb, err := cloudbuild.NewService(ctx) + if err != nil { + return rerror.ErrInternalBy(err) + } + + machineType := "" + if v := conf.MachineType; v != "default" { + machineType = v + } + + build := &cloudbuild.Build{ + Timeout: "86400s", // 1 day + QueueTtl: "86400s", // 1 day + Steps: []*cloudbuild.BuildStep{ + { + Name: conf.Image, + Args: conf.Args, + Env: conf.Env, + }, + }, + Options: &cloudbuild.BuildOptions{ + MachineType: machineType, + }, + } + + if conf.Region != "" { + call := cb.Projects.Locations.Builds.Create( + path.Join("projects", conf.Project, "locations", conf.Region), + build, + ) + _, err = call.Do() + } else { + call := cb.Projects.Builds.Create(conf.Project, build) + _, err = call.Do() + } + if err != nil { + return rerror.ErrInternalBy(err) + } + return nil +} diff --git a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudrunjobs.go b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudrunjobs.go new file mode 100644 index 000000000..f6d2b0829 --- /dev/null +++ b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_cloudrunjobs.go @@ -0,0 +1,50 @@ +package geospatialjpv3 + +import ( + "context" + + run "cloud.google.com/go/run/apiv2" + runpb "cloud.google.com/go/run/apiv2/runpb" + "github.com/reearth/reearthx/log" +) + +// jobName (Cloud Run Jobs): "projects/" + gcpProjectID + "/locations/" + gcpLocation + "/jobs/plateauview-api-worker" + +func prepareWithCloudRunJobs(ctx context.Context, itemID, projectID, jobName string) error { + if jobName == "" { + log.Debugfc(ctx, "geospatialjp webhook: no job name") + return nil + } + + log.Debugfc(ctx, "geospatialjp webhook: prepare (cloud run jobs): %s", jobName) + + client, err := run.NewJobsClient(ctx) + if err != nil { + log.Debugfc(ctx, "geospatialjp webhook: failed to create run client: %v", err) + return err + } + defer client.Close() + + overrides := runpb.RunJobRequest_Overrides{ + ContainerOverrides: []*runpb.RunJobRequest_Overrides_ContainerOverride{ + {Args: []string{ + "prepare-gspatialjp", + "--city=" + itemID, + "--project=" + projectID, + "--wetrun", + }}, + }} + + req := &runpb.RunJobRequest{ + Name: jobName, + Overrides: &overrides, + } + + if _, err = client.RunJob(ctx, req); err != nil { + log.Debugfc(ctx, "geospatialjp webhook: failed to run job: %v", err) + return err + } + + log.Debugfc(ctx, "geospatialjp webhook: run job: %v", req) + return nil +} diff --git a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_test.go b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_test.go deleted file mode 100644 index 58f9c076a..000000000 --- a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/prepare_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package geospatialjpv3 - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" -) - -const jobName = "projects/xxxxx/locations/xxxxx/jobs/plateauview-api-worker" -const itemID = "" -const projectID = "" - -func TestPrepare_RequestZip(t *testing.T) { - t.Skip() - - ctx := context.Background() - err := Prepare(ctx, itemID, projectID, jobName) - assert.NoError(t, err) -} diff --git a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/webhook.go b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/webhook.go index 86e994be9..b9ee5266d 100644 --- a/server/cmsintegration/cmsintegrationv3/geospatialjpv3/webhook.go +++ b/server/cmsintegration/cmsintegrationv3/geospatialjpv3/webhook.go @@ -130,7 +130,7 @@ func (h *handler) Webhook(conf Config) (cmswebhook.Handler, error) { log.Debugfc(ctx, "geospatialjpv3 webhook: %s", ppp.Sprint(cityItem)) if b := getChangedBool(ctx, w, prepareFieldKey); b != nil && *b { - if err := Prepare(ctx, cityItem.ID, w.ProjectID(), conf.JobName); err != nil { + if err := Prepare(ctx, cityItem.ID, w.ProjectID(), conf); err != nil { log.Errorfc(ctx, "geospatialjpv3 webhook: failed to prepare: %v", err) } } else { diff --git a/server/cmsintegration/handler.go b/server/cmsintegration/handler.go index db222d9c8..7cea2fc5a 100644 --- a/server/cmsintegration/handler.go +++ b/server/cmsintegration/handler.go @@ -97,13 +97,18 @@ func geospatialjpv2Config(conf Config) geospatialjpv2.Config { func geospatialjpv3Config(conf Config) geospatialjpv3.Config { return geospatialjpv3.Config{ - CMSBase: conf.CMSBaseURL, - CMSToken: conf.CMSToken, - CMSIntegration: conf.CMSIntegration, - CkanBase: conf.CkanBaseURL, - CkanOrg: conf.CkanOrg, - CkanToken: conf.CkanToken, - JobName: conf.GeospatialjpJobName, + CMSBase: conf.CMSBaseURL, + CMSToken: conf.CMSToken, + CMSIntegration: conf.CMSIntegration, + CkanBase: conf.CkanBaseURL, + CkanOrg: conf.CkanOrg, + CkanToken: conf.CkanToken, + BuildType: conf.GeospatialjpBuildType, + CloudRunJobsJobName: conf.GeospatialjpCloudRunJobsJobName, + CloudBuildImage: conf.GeospatialjpCloudBuildImage, + CloudBuildMachineType: conf.GeospatialjpCloudBuildMachineType, + CloudBuildProject: conf.GeospatialjpCloudBuildProject, + CloudBuildRegion: conf.GeospatialjpCloudBuildRegion, } } diff --git a/server/config.go b/server/config.go index cd394460a..61127fe06 100644 --- a/server/config.go +++ b/server/config.go @@ -26,50 +26,55 @@ func init() { const configPrefix = "REEARTH_PLATEAUVIEW" type Config struct { - Port uint `default:"8080" envconfig:"PORT"` - Host string `default:"http://localhost:8080"` - Debug bool `pp:",omitempty"` - Origin []string `pp:",omitempty"` - Secret string `pp:",omitempty"` - Delegate_URL string `pp:",omitempty"` - CMS_Webhook_Secret string `pp:",omitempty"` - CMS_BaseURL string `pp:",omitempty"` - CMS_Token string `pp:",omitempty"` - CMS_IntegrationID string `pp:",omitempty"` - CMS_PlateauProject string `pp:",omitempty"` - CMS_SystemProject string `pp:",omitempty"` - CMS_TokenProject string `pp:",omitempty"` - FME_BaseURL string `pp:",omitempty"` - FME_BaseURL_V2 string `pp:",omitempty"` - FME_URL_V3 string `pp:",omitempty"` - FME_Mock bool `pp:",omitempty"` - FME_Token string `pp:",omitempty"` - FME_SkipQualityCheck bool `pp:",omitempty"` - Ckan_BaseURL string `pp:",omitempty"` - Ckan_Org string `pp:",omitempty"` - Ckan_Token string `pp:",omitempty"` - Ckan_Private bool `pp:",omitempty"` - SDK_Token string `pp:",omitempty"` - SendGrid_APIKey string `pp:",omitempty"` - Opinion_From string `pp:",omitempty"` - Opinion_FromName string `pp:",omitempty"` - Opinion_To string `pp:",omitempty"` - Opinion_ToName string `pp:",omitempty"` - Sidebar_Token string `pp:",omitempty"` - Share_Disable bool `pp:",omitempty"` - Geospatialjp_Publication_Disable bool `pp:",omitempty"` - Geospatialjp_CatalocCheck_Disable bool `pp:",omitempty"` - Geospatialjp_JobName string `pp:",omitempty"` - DataConv_Disable bool `pp:",omitempty"` - Indexer_Delegate bool `pp:",omitempty"` - DataCatalog_DisableCache bool `pp:",omitempty"` - DataCatalog_CacheUpdateKey string `pp:",omitempty"` - DataCatalog_PlaygroundEndpoint string `pp:",omitempty"` - DataCatalog_CacheTTL int `pp:",omitempty"` - DataCatalog_GQL_MaxComplexity int `pp:",omitempty"` - SDKAPI_DisableCache bool `pp:",omitempty"` - SDKAPI_CacheTTL int `pp:",omitempty"` - GCParcent int `pp:",omitempty"` + Port uint `default:"8080" envconfig:"PORT"` + Host string `default:"http://localhost:8080"` + Debug bool `pp:",omitempty"` + Origin []string `pp:",omitempty"` + Secret string `pp:",omitempty"` + Delegate_URL string `pp:",omitempty"` + CMS_Webhook_Secret string `pp:",omitempty"` + CMS_BaseURL string `pp:",omitempty"` + CMS_Token string `pp:",omitempty"` + CMS_IntegrationID string `pp:",omitempty"` + CMS_PlateauProject string `pp:",omitempty"` + CMS_SystemProject string `pp:",omitempty"` + CMS_TokenProject string `pp:",omitempty"` + FME_BaseURL string `pp:",omitempty"` + FME_BaseURL_V2 string `pp:",omitempty"` + FME_URL_V3 string `pp:",omitempty"` + FME_Mock bool `pp:",omitempty"` + FME_Token string `pp:",omitempty"` + FME_SkipQualityCheck bool `pp:",omitempty"` + Ckan_BaseURL string `pp:",omitempty"` + Ckan_Org string `pp:",omitempty"` + Ckan_Token string `pp:",omitempty"` + Ckan_Private bool `pp:",omitempty"` + SDK_Token string `pp:",omitempty"` + SendGrid_APIKey string `pp:",omitempty"` + Opinion_From string `pp:",omitempty"` + Opinion_FromName string `pp:",omitempty"` + Opinion_To string `pp:",omitempty"` + Opinion_ToName string `pp:",omitempty"` + Sidebar_Token string `pp:",omitempty"` + Share_Disable bool `pp:",omitempty"` + Geospatialjp_Publication_Disable bool `pp:",omitempty"` + Geospatialjp_CatalocCheck_Disable bool `pp:",omitempty"` + Geospatialjp_BuildType string `pp:",omitempty"` + Geospatialjp_JobName string `pp:",omitempty"` + Geospatialjp_CloudBuildImage string `pp:",omitempty"` + Geospatialjp_CloudBuildMachineType string `pp:",omitempty"` + Geospatialjp_CloudBuildProject string `pp:",omitempty"` + Geospatialjp_CloudBuildRegion string `pp:",omitempty"` + DataConv_Disable bool `pp:",omitempty"` + Indexer_Delegate bool `pp:",omitempty"` + DataCatalog_DisableCache bool `pp:",omitempty"` + DataCatalog_CacheUpdateKey string `pp:",omitempty"` + DataCatalog_PlaygroundEndpoint string `pp:",omitempty"` + DataCatalog_CacheTTL int `pp:",omitempty"` + DataCatalog_GQL_MaxComplexity int `pp:",omitempty"` + SDKAPI_DisableCache bool `pp:",omitempty"` + SDKAPI_CacheTTL int `pp:",omitempty"` + GCParcent int `pp:",omitempty"` } func NewConfig() (*Config, error) { @@ -91,27 +96,32 @@ func (c *Config) Print() string { func (c *Config) CMSIntegration() cmsintegration.Config { return cmsintegration.Config{ - Host: c.Host, - FMEMock: c.FME_Mock, - FMEBaseURL: c.FME_BaseURL, - FMEToken: c.FME_Token, - FMEBaseURLV2: c.FME_BaseURL_V2, - FMEURLV3: c.FME_URL_V3, - FMESkipQualityCheck: c.FME_SkipQualityCheck, - CMSBaseURL: c.CMS_BaseURL, - CMSToken: c.CMS_Token, - CMSIntegration: c.CMS_IntegrationID, - Secret: c.Secret, - Debug: c.Debug, - CkanBaseURL: c.Ckan_BaseURL, - CkanOrg: c.Ckan_Org, - CkanToken: c.Ckan_Token, - CkanPrivate: c.Ckan_Private, - DisableGeospatialjpPublication: c.Geospatialjp_Publication_Disable, - DisableGeospatialjpCatalogCheck: c.Geospatialjp_CatalocCheck_Disable, - DisableDataConv: c.DataConv_Disable, - APIToken: c.Sidebar_Token, - GeospatialjpJobName: c.Geospatialjp_JobName, + Host: c.Host, + FMEMock: c.FME_Mock, + FMEBaseURL: c.FME_BaseURL, + FMEToken: c.FME_Token, + FMEBaseURLV2: c.FME_BaseURL_V2, + FMEURLV3: c.FME_URL_V3, + FMESkipQualityCheck: c.FME_SkipQualityCheck, + CMSBaseURL: c.CMS_BaseURL, + CMSToken: c.CMS_Token, + CMSIntegration: c.CMS_IntegrationID, + Secret: c.Secret, + Debug: c.Debug, + CkanBaseURL: c.Ckan_BaseURL, + CkanOrg: c.Ckan_Org, + CkanToken: c.Ckan_Token, + CkanPrivate: c.Ckan_Private, + DisableGeospatialjpPublication: c.Geospatialjp_Publication_Disable, + DisableGeospatialjpCatalogCheck: c.Geospatialjp_CatalocCheck_Disable, + DisableDataConv: c.DataConv_Disable, + APIToken: c.Sidebar_Token, + GeospatialjpBuildType: c.Geospatialjp_BuildType, + GeospatialjpCloudRunJobsJobName: c.Geospatialjp_JobName, + GeospatialjpCloudBuildImage: c.Geospatialjp_CloudBuildImage, + GeospatialjpCloudBuildMachineType: c.Geospatialjp_CloudBuildMachineType, + GeospatialjpCloudBuildProject: c.Geospatialjp_CloudBuildProject, + GeospatialjpCloudBuildRegion: c.Geospatialjp_CloudBuildRegion, } }