From 14585f320df630a1ef4faa91746844849973acde Mon Sep 17 00:00:00 2001 From: Caleb Brown Date: Mon, 28 Nov 2022 17:02:09 +1100 Subject: [PATCH] Add the Git commit ID to the data produced by the worker. (#261) * Add the Git commit ID to the data produced by the worker. Signed-off-by: Caleb Brown * Small refactor to cache the commitID once per execution. Signed-off-by: Caleb Brown Signed-off-by: Caleb Brown --- cmd/collect_signals/Dockerfile | 2 +- cmd/collect_signals/main.go | 5 +++++ cmd/collect_signals/vcs.go | 40 +++++++++++++++++++++++++++++++++ cmd/collect_signals/worker.go | 17 +++++++++++++- cmd/criticality_score/README.md | 6 ++--- 5 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 cmd/collect_signals/vcs.go diff --git a/cmd/collect_signals/Dockerfile b/cmd/collect_signals/Dockerfile index 16beec710..4d82b9426 100644 --- a/cmd/collect_signals/Dockerfile +++ b/cmd/collect_signals/Dockerfile @@ -22,7 +22,7 @@ COPY . ./ FROM base AS collect_signals ARG TARGETOS ARG TARGETARCH -RUN CGO_ENABLED=0 go build ./cmd/collect_signals +RUN CGO_ENABLED=0 go build -buildvcs ./cmd/collect_signals RUN chmod -R 0775 /src/config/scorer/* FROM gcr.io/distroless/base:nonroot@sha256:533c15ef2acb1d3b1cd4e58d8aa2740900cae8f579243a53c53a6e28bcac0684 diff --git a/cmd/collect_signals/main.go b/cmd/collect_signals/main.go index db8d3d48b..c616c6da0 100644 --- a/cmd/collect_signals/main.go +++ b/cmd/collect_signals/main.go @@ -91,6 +91,11 @@ func main() { } defer logger.Sync() + // Embed the commitID with all log messages. + if commitID != "" { + logger = logger.With(zap.String("commit_id", commitID)) + } + // Extract the GCP project ID. gcpProjectID, err := config.GetProjectID() if err != nil { diff --git a/cmd/collect_signals/vcs.go b/cmd/collect_signals/vcs.go new file mode 100644 index 000000000..800b607c6 --- /dev/null +++ b/cmd/collect_signals/vcs.go @@ -0,0 +1,40 @@ +// Copyright 2022 Criticality Score Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "runtime/debug" +) + +const commitIDKey = "vcs.revision" + +var commitID = getCommitID() + +// getCommitID returns the vcs commit ID embedded in the binary when the +// -buildvcs flag is set while building. +func getCommitID() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "" + } + + for _, setting := range info.Settings { + if setting.Key == commitIDKey { + return setting.Value + } + } + + return "" +} diff --git a/cmd/collect_signals/worker.go b/cmd/collect_signals/worker.go index 1f938eeb8..590af168d 100644 --- a/cmd/collect_signals/worker.go +++ b/cmd/collect_signals/worker.go @@ -21,7 +21,10 @@ import ( "github.com/ossf/criticality_score/internal/signalio" ) -const collectionDateColumnName = "collection_date" +const ( + collectionDateColumnName = "collection_date" + commitIDColumnName = "worker_commit_id" +) type collectWorker struct { logger *zap.Logger @@ -51,6 +54,9 @@ func (w *collectWorker) Process(ctx context.Context, req *data.ScorecardBatchReq extras = append(extras, w.scoreColumnName) } extras = append(extras, collectionDateColumnName) + if commitID != "" { + extras = append(extras, commitIDColumnName) + } var jsonOutput bytes.Buffer jsonOut := signalio.JSONWriter(&jsonOutput) @@ -102,6 +108,15 @@ func (w *collectWorker) Process(ctx context.Context, req *data.ScorecardBatchReq Value: jobTime, }) + // Ensure the commit ID is included with each record for helping + // identify which Git commit is associated with this record. + if commitID != "" { + extras = append(extras, signalio.Field{ + Key: commitIDColumnName, + Value: commitID, + }) + } + // Write the signals to storage. if err := jsonOut.WriteSignals(ss, extras...); err != nil { return fmt.Errorf("failed writing signals: %w", err) diff --git a/cmd/criticality_score/README.md b/cmd/criticality_score/README.md index bc42405cc..ffe8cdca6 100644 --- a/cmd/criticality_score/README.md +++ b/cmd/criticality_score/README.md @@ -64,10 +64,10 @@ $ export GITHUB_TOKEN=ghp_abc,ghp_123 BigQuery access requires the "BigQuery User" (`roles/bigquery.user`) role added to the account used, or be an "Owner". -##### Option 1: `gcloud login` +##### Option 1: `gcloud auth login` -This option is useful during development. Run `gcloud login --update-adc` to -login to GCP and prepare application default credentials. +This option is useful during development. Run `gcloud auth login --update-adc` +to login to GCP and prepare application default credentials. ##### Option 2: GCE Service Worker