Remote reporting of package build times and success

BUILD.yaml

@@ -10,7 +10,9 @@ packages:
   - name: app
     type: go
     srcs:
-      - "**/*.go"
+      - "*.go"
+      - "cmd/**/*.go"
+      - "pkg/**/*.go"
       - "**/*.html"
       - "**/*.js"
       - go.mod

cmd/build.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/gitpod-io/leeway/pkg/leeway"
+	"github.com/gitpod-io/leeway/pkg/remotereporter"
 	"github.com/gookit/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -173,6 +174,7 @@ func addBuildFlags(cmd *cobra.Command) {
 	cmd.Flags().String("coverage-output-path", "", "Output path where test coverage file will be copied after running tests")
 	cmd.Flags().StringToString("docker-build-options", nil, "Options passed to all 'docker build' commands")
 	cmd.Flags().String("report", "", "Generate a HTML report after the build has finished. (e.g. --report myreport.html)")
+	cmd.Flags().String("remote-report", os.Getenv("LEEWAY_REMOTE_REPORT"), "Report the build progress to a remote endoint. The LEEWAY_REMOTE_REPORT env var is used as default.")
 }
 
 func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemCache) {
@@ -236,23 +238,23 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemC
 		}
 	}
 
-	werftlog, err := cmd.Flags().GetBool("werft")
-	if err != nil {
+	var reporter leeway.CompositeReporter
+	reporter = append(reporter, leeway.NewConsoleReporter())
+
+	if werftlog, err := cmd.Flags().GetBool("werft"); err != nil {
 		log.Fatal(err)
+	} else if werftlog {
+		reporter = append(reporter, leeway.NewWerftReporter())
 	}
-	var reporter leeway.Reporter
-	if werftlog {
-		reporter = leeway.NewWerftReporter()
-	} else {
-		reporter = leeway.NewConsoleReporter()
-	}
-
-	report, err := cmd.Flags().GetString("report")
-	if err != nil {
+	if report, err := cmd.Flags().GetString("report"); err != nil {
 		log.Fatal(err)
+	} else if report != "" {
+		reporter = append(reporter, leeway.NewHTMLReporter(report))
 	}
-	if report != "" {
-		reporter = leeway.NewHTMLReporter(reporter, report)
+	if ep, err := cmd.Flags().GetString("remote-report"); err != nil {
+		log.Fatal(err)
+	} else if ep != "" {
+		reporter = append(reporter, remotereporter.NewReporter(ep, os.Getenv("LEEWAY_REMOTE_REPORT_TOKEN")))
 	}
 
 	dontTest, err := cmd.Flags().GetBool("dont-test")

go.mod

@@ -30,8 +30,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.47
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.18.0
+	github.com/bufbuild/connect-go v1.9.0
 	github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4
 	github.com/stretchr/testify v1.8.0
+	google.golang.org/protobuf v1.31.0
 )
 
 require (
@@ -55,6 +57,7 @@ require (
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect

go.sum

@@ -39,6 +39,8 @@ github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/bufbuild/connect-go v1.9.0 h1:JIgAeNuFpo+SUPfU19Yt5TcWlznsN5Bv10/gI/6Pjoc=
+github.com/bufbuild/connect-go v1.9.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
 github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
@@ -82,6 +84,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gookit/color v1.5.2 h1:uLnfXcaFjlrDnQDT+NCBcfhrXqYTx/rcCa6xn01Y8yI=
 github.com/gookit/color v1.5.2/go.mod h1:w8h4bGiHeeBpvQVePTutdbERIUf3oJE5lZ8HM0UgXyg=
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
@@ -189,6 +193,8 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

pkg/leeway/reporter.go

@@ -183,20 +183,16 @@ func getRunPrefix(p *Package) string {
 
 // NewWerftReporter craetes a new werft compatible reporter
 func NewWerftReporter() *WerftReporter {
-	return &WerftReporter{
-		ConsoleReporter: NewConsoleReporter(),
-	}
+	return &WerftReporter{}
 }
 
 // WerftReporter works like the console reporter but adds werft output
 type WerftReporter struct {
-	*ConsoleReporter
+	NoopReporter
 }
 
 // BuildStarted is called when the build of a package is started by the user.
 func (r *WerftReporter) BuildStarted(pkg *Package, status map[*Package]PackageBuildStatus) {
-	r.ConsoleReporter.BuildStarted(pkg, status)
-
 	for p, s := range status {
 		if s != PackageNotBuiltYet {
 			continue
@@ -211,8 +207,6 @@ func (r *WerftReporter) BuildStarted(pkg *Package, status map[*Package]PackageBu
 
 // PackageBuildFinished is called when the package build has finished.
 func (r *WerftReporter) PackageBuildFinished(pkg *Package, err error) {
-	r.ConsoleReporter.PackageBuildFinished(pkg, err)
-
 	if cfg, ok := pkg.Config.(DockerPkgConfig); ok && pkg.Type == DockerPackage {
 		for _, img := range cfg.Image {
 			fmt.Printf("[docker|RESULT] %s\n", img)
@@ -291,16 +285,14 @@ func (r *PackageReport) Error() string {
 }
 
 type HTMLReporter struct {
-	delegate    Reporter
 	filename    string
 	reports     map[string]*PackageReport
 	rootPackage *Package
 	mu          sync.RWMutex
 }
 
-func NewHTMLReporter(del Reporter, filename string) *HTMLReporter {
+func NewHTMLReporter(filename string) *HTMLReporter {
 	return &HTMLReporter{
-		delegate: del,
 		filename: filename,
 		reports:  make(map[string]*PackageReport),
 	}
@@ -331,16 +323,13 @@ func (r *HTMLReporter) getReport(pkg *Package) *PackageReport {
 
 func (r *HTMLReporter) BuildStarted(pkg *Package, status map[*Package]PackageBuildStatus) {
 	r.rootPackage = pkg
-	r.delegate.BuildStarted(pkg, status)
 }
 
 func (r *HTMLReporter) BuildFinished(pkg *Package, err error) {
-	r.delegate.BuildFinished(pkg, err)
 	r.Report()
 }
 
 func (r *HTMLReporter) PackageBuildStarted(pkg *Package) {
-	r.delegate.PackageBuildStarted(pkg)
 	rep := r.getReport(pkg)
 	rep.start = time.Now()
 	rep.status = PackageBuilding
@@ -349,11 +338,9 @@ func (r *HTMLReporter) PackageBuildStarted(pkg *Package) {
 func (r *HTMLReporter) PackageBuildLog(pkg *Package, isErr bool, buf []byte) {
 	report := r.getReport(pkg)
 	report.logs.Write(buf)
-	r.delegate.PackageBuildLog(pkg, isErr, buf)
 }
 
 func (r *HTMLReporter) PackageBuildFinished(pkg *Package, err error) {
-	r.delegate.PackageBuildFinished(pkg, err)
 	rep := r.getReport(pkg)
 	rep.duration = time.Since(rep.start)
 	rep.status = PackageBuilt
@@ -432,3 +419,61 @@ func (r *HTMLReporter) Report() {
 		log.WithError(err).Fatal("Can't render template")
 	}
 }
+
+type CompositeReporter []Reporter
+
+// BuildFinished implements Reporter
+func (cr CompositeReporter) BuildFinished(pkg *Package, err error) {
+	for _, r := range cr {
+		r.BuildFinished(pkg, err)
+	}
+}
+
+// BuildStarted implements Reporter
+func (cr CompositeReporter) BuildStarted(pkg *Package, status map[*Package]PackageBuildStatus) {
+	for _, r := range cr {
+		r.BuildStarted(pkg, status)
+	}
+}
+
+// PackageBuildFinished implements Reporter
+func (cr CompositeReporter) PackageBuildFinished(pkg *Package, err error) {
+	for _, r := range cr {
+		r.PackageBuildFinished(pkg, err)
+	}
+}
+
+// PackageBuildLog implements Reporter
+func (cr CompositeReporter) PackageBuildLog(pkg *Package, isErr bool, buf []byte) {
+	for _, r := range cr {
+		r.PackageBuildLog(pkg, isErr, buf)
+	}
+}
+
+// PackageBuildStarted implements Reporter
+func (cr CompositeReporter) PackageBuildStarted(pkg *Package) {
+	for _, r := range cr {
+		r.PackageBuildStarted(pkg)
+	}
+}
+
+var _ Reporter = CompositeReporter{}
+
+type NoopReporter struct{}
+
+// BuildFinished implements Reporter
+func (*NoopReporter) BuildFinished(pkg *Package, err error) {}
+
+// BuildStarted implements Reporter
+func (*NoopReporter) BuildStarted(pkg *Package, status map[*Package]PackageBuildStatus) {}
+
+// PackageBuildFinished implements Reporter
+func (*NoopReporter) PackageBuildFinished(pkg *Package, err error) {}
+
+// PackageBuildLog implements Reporter
+func (*NoopReporter) PackageBuildLog(pkg *Package, isErr bool, buf []byte) {}
+
+// PackageBuildStarted implements Reporter
+func (*NoopReporter) PackageBuildStarted(pkg *Package) {}
+
+var _ Reporter = ((*NoopReporter)(nil))

pkg/remotereporter/api/BUILD.yaml

@@ -0,0 +1,9 @@
+scripts:
+  - name: generate
+    description: generates the Go API for the remote reporter gRPC interface
+    script: |
+      go install github.com/bufbuild/buf/cmd/buf@latest
+      go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+      go install github.com/bufbuild/connect-go/cmd/protoc-gen-connect-go@latest
+      go install github.com/golang/mock/mockgen@latest
+      buf generate

pkg/remotereporter/api/buf.gen.yaml

@@ -0,0 +1,10 @@
+version: v1
+managed:
+  enabled: true
+plugins:
+  - name: go
+    out: gen
+    opt: paths=source_relative
+  - name: connect-go
+    out: gen
+    opt: paths=source_relative

pkg/remotereporter/api/buf.yaml

@@ -0,0 +1,7 @@
+version: v1
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - DEFAULT