🧪 experimental: Store resources support (#1206)

* 🧪 experimental: Store resources support

Aligns with the changes outlined in mondoohq/cnquery#3668

Signed-off-by: Dominik Richter <[email protected]>

* 🧹 update go mod
* 🧹 handle error when setting the recording

---------

Signed-off-by: Dominik Richter <[email protected]>
Co-authored-by: Christoph Hartmann <[email protected]>

apps/cnspec/cmd/scan.go

@@ -261,6 +261,16 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes *
 		OutputTarget:   viper.GetString("output-target"),
 	}
 
+	// FIXME: DEPRECATED, remove in v12.0 and make this the default for all
+	// use-cases where we have upstream recording enabled vv
+	// Instead of depending on the feature-flag, we look at the config
+	if conf.Features.IsActive(cnquery.StoreResourcesData) {
+		if err = runtime.EnableResourcesRecording(); err != nil {
+			log.Fatal().Err(err).Msg("failed to enable resources recording")
+		}
+	}
+	// ^^
+
 	// if users want to get more information on available output options,
 	// print them before executing the scan
 	output, _ := cmd.Flags().GetString("output")

apps/cnspec/cmd/serve_api.go

@@ -19,7 +19,7 @@ import (
 	"github.com/spf13/viper"
 	"go.mondoo.com/cnquery/v10/cli/config"
 	"go.mondoo.com/cnquery/v10/logger"
-	"go.mondoo.com/cnquery/v10/providers"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/recording"
 	"go.mondoo.com/cnquery/v10/providers-sdk/v1/upstream"
 	cnspec_config "go.mondoo.com/cnspec/v10/apps/cnspec/cmd/config"
 	"go.mondoo.com/cnspec/v10/policy/scan"
@@ -73,7 +73,7 @@ var serveApiCmd = &cobra.Command{
 		scanner := scan.NewLocalScanner(
 			scan.WithUpstream(&upstreamConfig),
 			scan.DisableProgressBar(),
-			scan.WithRecording(providers.NullRecording{}),
+			scan.WithRecording(recording.Null{}),
 		)
 		if err := scanner.EnableQueue(); err != nil {
 			log.Fatal().Err(err).Msg("could not enable scan queue")

cli/reporter/print_compact.go

@@ -193,6 +193,10 @@ func (r *defaultReporter) printSummary(orderedAssets []assetMrnName) {
 
 			r.out("See more scan results and asset relationships on the Mondoo Console: ")
 			r.out(url + NewLineCharacter)
+
+			if len(orderedAssets) == 1 && orderedAssets[0].Mrn != "" {
+				r.out("Asset MRN: " + orderedAssets[0].Mrn + NewLineCharacter)
+			}
 		}
 	}
 }

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/spf13/pflag v1.0.6-0.20201009195203-85dd5c8bc61c
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.9.0
-	go.mondoo.com/cnquery/v10 v10.9.1
+	go.mondoo.com/cnquery/v10 v10.9.2-0.20240328085036-f4c664d5adb1
 	go.mondoo.com/mondoo-go v0.0.0-20240312054001-2ab23a39b500
 	go.mondoo.com/ranger-rpc v0.6.0
 	go.opentelemetry.io/otel v1.24.0
@@ -143,7 +143,7 @@ require (
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
 	github.com/containerd/console v1.0.4 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/curioswitch/go-reassign v0.2.0 // indirect
@@ -249,7 +249,7 @@ require (
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/kisielk/errcheck v1.7.0 // indirect
 	github.com/kkHAIKE/contextcheck v1.1.5 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect
 	github.com/knqyf263/go-rpmdb v0.1.0 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -362,7 +362,7 @@ require (
 	github.com/yeya24/promlinter v0.2.0 // indirect
 	github.com/ykadowak/zerologlint v0.1.5 // indirect
 	gitlab.com/bosi/decorder v0.4.1 // indirect
-	go-simpler.org/musttag v0.9.0 // indirect
+	go-simpler.org/musttag v0.10.0 // indirect
 	go-simpler.org/sloglint v0.5.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
@@ -395,9 +395,9 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	honnef.co/go/tools v0.4.7 // indirect
 	howett.net/plist v1.0.1 // indirect
-	k8s.io/api v0.29.0 // indirect
-	k8s.io/apimachinery v0.29.0 // indirect
-	k8s.io/client-go v0.29.0 // indirect
+	k8s.io/api v0.29.3 // indirect
+	k8s.io/apimachinery v0.29.3 // indirect
+	k8s.io/client-go v0.29.3 // indirect
 	k8s.io/component-base v0.29.0 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kubelet v0.29.0 // indirect

go.sum

@@ -318,8 +318,8 @@ github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn
 github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
-github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
+github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
+github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -756,8 +756,8 @@ github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkHAIKE/contextcheck v1.1.5 h1:CdnJh63tcDe53vG+RebdpdXJTc9atMgGqdx8LXxiilg=
 github.com/kkHAIKE/contextcheck v1.1.5/go.mod h1:O930cpht4xb1YQpK+1+AgoM3mFsvxr7uyFptcnWTYUA=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f h1:GvCU5GXhHq+7LeOzx/haG7HSIZokl3/0GkoUFzsRJjg=
 github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f/go.mod h1:q59u9px8b7UTj0nIjEjvmTWekazka6xIt6Uogz5Dm+8=
 github.com/knqyf263/go-rpmdb v0.1.0 h1:pOgjtOGtW0B+ibY905hP3ETrYFmLZsHiReKsplcs+to=
@@ -1179,15 +1179,15 @@ gitlab.com/bosi/decorder v0.4.1 h1:VdsdfxhstabyhZovHafFw+9eJ6eU0d2CkFNJcZz/NU4=
 gitlab.com/bosi/decorder v0.4.1/go.mod h1:jecSqWUew6Yle1pCr2eLWTensJMmsxHsBwt+PVbkAqA=
 go-simpler.org/assert v0.7.0 h1:OzWWZqfNxt8cLS+MlUp6Tgk1HjPkmgdKBq9qvy8lZsA=
 go-simpler.org/assert v0.7.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28=
-go-simpler.org/musttag v0.9.0 h1:Dzt6/tyP9ONr5g9h9P3cnYWCxeBFRkd0uJL/w+1Mxos=
-go-simpler.org/musttag v0.9.0/go.mod h1:gA9nThnalvNSKpEoyp3Ko4/vCX2xTpqKoUtNqXOnVR4=
+go-simpler.org/musttag v0.10.0 h1:7XBqvciYztgXU1BgLuUXbfrdB3letG8s2HzJtyMS+iw=
+go-simpler.org/musttag v0.10.0/go.mod h1:46HKu04A3Am9Lne5kKP0ssgwY3AeIlqsDzz3UxKROpY=
 go-simpler.org/sloglint v0.5.0 h1:2YCcd+YMuYpuqthCgubcF5lBSjb6berc5VMOYUHKrpY=
 go-simpler.org/sloglint v0.5.0/go.mod h1:EUknX5s8iXqf18KQxKnaBHUPVriiPnOrPjjJcsaTcSQ=
 go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs=
-go.mondoo.com/cnquery/v10 v10.9.1 h1:E0CHShKmq2ijX0AFSaFzrHqusm/6XmYc3EtwVp9Gi48=
-go.mondoo.com/cnquery/v10 v10.9.1/go.mod h1:aAcCZ6modg4DYeDTdeaeGtiQGbEjYExfjm6frUD3efY=
+go.mondoo.com/cnquery/v10 v10.9.2-0.20240328085036-f4c664d5adb1 h1:uzftWnMiYmjLWlnC2eCHFlPd6ohoqc5OOhdsKCECAXE=
+go.mondoo.com/cnquery/v10 v10.9.2-0.20240328085036-f4c664d5adb1/go.mod h1:EOqOiHse5uv2OP4mvV25J7V6Yd6sK/cgwyNm25zZDAY=
 go.mondoo.com/mondoo-go v0.0.0-20240312054001-2ab23a39b500 h1:HjtfupqfqKnDaK+XUXZC/3JyIwK09C3TFNggwReuDlY=
 go.mondoo.com/mondoo-go v0.0.0-20240312054001-2ab23a39b500/go.mod h1:pFmkGNtz+oa1iE/0Ssjo5KCTQ2VFeZgyIaD8c7nqcoc=
 go.mondoo.com/ranger-rpc v0.6.0 h1:u8FY0NGhR1B/LZk9+DdmqksrWc007argv+nMGJGa3OA=
@@ -1814,12 +1814,12 @@ honnef.co/go/tools v0.4.7 h1:9MDAWxMoSnB6QoSqiVr7P5mtkT9pOc1kSxchzPCnqJs=
 honnef.co/go/tools v0.4.7/go.mod h1:+rnGS1THNh8zMwnd2oVOTL9QF6vmfyG6ZXBULae2uc0=
 howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
 howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
-k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
-k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
-k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
-k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
-k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
-k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
+k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw=
+k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80=
+k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU=
+k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU=
+k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg=
+k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0=
 k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s=
 k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=

internal/datalakes/inmemory/policyresolver.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/v10/explorer"
+	"go.mondoo.com/cnquery/v10/explorer/resources"
 	"go.mondoo.com/cnquery/v10/llx"
 	"go.mondoo.com/cnquery/v10/types"
 	"go.mondoo.com/cnspec/v10/policy"
@@ -458,7 +459,7 @@ func (db *Db) GetReport(ctx context.Context, assetMrn string, qrID string) (*pol
 		return nil, err
 	}
 
-	risks, err := db.GetRisks(ctx, assetMrn)
+	risks, err := db.GetScoredRisks(ctx, assetMrn)
 	if err != nil {
 		log.Error().
 			Err(err).
@@ -508,8 +509,8 @@ func (db *Db) GetScores(ctx context.Context, assetMrn string, qrIDs []string) (m
 	return res, nil
 }
 
-// GetRisks retrieves risk scores for an asset
-func (db *Db) GetRisks(ctx context.Context, assetMrn string) (*policy.ScoredRiskFactors, error) {
+// GetScoredRisks retrieves risk scores for an asset
+func (db *Db) GetScoredRisks(ctx context.Context, assetMrn string) (*policy.ScoredRiskFactors, error) {
 	raw, ok := db.cache.Get(dbIDAssetRisk + assetMrn)
 	if !ok {
 		return nil, nil
@@ -903,6 +904,28 @@ func (db *Db) UpdateRisks(ctx context.Context, assetMrn string, data []*policy.S
 	return updates, nil
 }
 
+// GetResources retrieves previously stored resources about an asset
+func (db *Db) GetResources(ctx context.Context, assetMrn string, req []*resources.ResourceDataReq) ([]*llx.ResourceRecording, error) {
+	res := make([]*llx.ResourceRecording, len(req))
+	for i := range req {
+		rr := req[i]
+		raw, ok := db.cache.Get(dbIDData + assetMrn + "\x00" + rr.Resource + "\x00" + rr.Id)
+		if !ok {
+			return nil, errors.New("cannot find resource " + rr.Resource + " id=" + rr.Id + " on " + assetMrn)
+		}
+		res[i] = raw.(*llx.ResourceRecording)
+	}
+	return res, nil
+}
+
+// UpdateResources stores resources recording data for a given asset
+func (db *Db) UpdateResources(ctx context.Context, assetMrn string, resourcesRecording map[string]*llx.ResourceRecording) error {
+	for _, rr := range resourcesRecording {
+		db.cache.Set(dbIDData+assetMrn+"\x00"+rr.Resource+"\x00"+rr.Id, rr.Fields, 1)
+	}
+	return nil
+}
+
 // SetProps will override properties for a given entity (asset, space, org)
 func (db *Db) SetProps(ctx context.Context, req *explorer.PropsReq) error {
 	policyw, err := db.ensurePolicy(ctx, req.EntityMrn, false)