diff --git a/README.md b/README.md index 99c5c19..a90c28c 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,35 @@ the `--nobest` option can be supplied. With this option all packages are considered. Newest packages will have the higest weight but it may not always be able to choose them and older packages may be pulled in instead. +### Lock files + +bazeldnf can use lock files as the source of RPMs in lieu of using the WORKSPACE file. These +can be hand written or generated with `bazeldnf rpmtree` similarly to how WORKSPACE files work. +Lock files can *only* be used when working in bzlmod mode, not in workspace mode. + +To generate lock files you can run the following: + +```bash +bazeldnf rpmtree --lockfile rpms.json --configname myrpms --name libvirttree libvirt +``` + +The lock file JSON format is as follows: +``` +{ + "name": "bazeldnf-rpms", + "rpms": [ + { + "name": "libvirt-libs", + "sha256": "aac272a2ace134b5ef60a41e6624deb24331e79c76699ef6cef0dca22d94ac7e", + "urls": [ + "https://kojipkgs.fedoraproject.org//packages/libvirt/11.0.0/1.fc42/x86_64/libvirt-libs-11.0.0-1.fc42.x86_64.rpm" + ] + } + ] +} + +``` + ### Dependency resolution limitations ##### Missing features diff --git a/cmd/rpmtree.go b/cmd/rpmtree.go index 6fbcd5a..ba9282c 100644 --- a/cmd/rpmtree.go +++ b/cmd/rpmtree.go @@ -5,6 +5,8 @@ import ( "github.com/bazelbuild/buildtools/build" "github.com/rmohr/bazeldnf/cmd/template" + "github.com/rmohr/bazeldnf/pkg/api" + "github.com/rmohr/bazeldnf/pkg/api/bazeldnf" "github.com/rmohr/bazeldnf/pkg/bazel" "github.com/rmohr/bazeldnf/pkg/reducer" "github.com/rmohr/bazeldnf/pkg/repo" @@ -21,6 +23,8 @@ type rpmtreeOpts struct { workspace string toMacro string buildfile string + configname string + lockfile string name string public bool forceIgnoreRegex []string @@ -28,6 +32,101 @@ type rpmtreeOpts struct { var rpmtreeopts = rpmtreeOpts{} +type Handler interface { + Process(pkgs []*api.Package, arch string, buildfile *build.File) error + Write() error +} + +type MacroHandler struct { + bzl, defName string + bzlfile *build.File +} + +func NewMacroHandler(toMacro string) (Handler, error) { + bzl, defName, err := bazel.ParseMacro(rpmtreeopts.toMacro) + + if err != nil { + return nil, err + } + + bzlfile, err := bazel.LoadBzl(bzl) + if err != nil { + return nil, err + } + + return &MacroHandler{ + bzl: bzl, + bzlfile: bzlfile, + defName: defName, + }, nil +} + +func (h *MacroHandler) Process(pkgs []*api.Package, arch string, buildfile *build.File) error { + if err := bazel.AddBzlfileRPMs(h.bzlfile, h.defName, pkgs, arch); err != nil { + return err + } + + bazel.PruneBzlfileRPMs(buildfile, h.bzlfile, h.defName) + return nil +} + +func (h *MacroHandler) Write() error { + return bazel.WriteBzl(false, h.bzlfile, h.bzl) +} + +type WorkspaceHandler struct { + workspace string + workspacefile *build.File +} + +func NewWorkspaceHandler(workspace string) (Handler, error) { + workspacefile, err := bazel.LoadWorkspace(workspace) + if err != nil { + return nil, err + } + + return &WorkspaceHandler{ + workspace: workspace, + workspacefile: workspacefile, + }, nil +} + +func (h *WorkspaceHandler) Process(pkgs []*api.Package, arch string, buildfile *build.File) error { + if err := bazel.AddWorkspaceRPMs(h.workspacefile, pkgs, arch); err != nil { + return err + } + + bazel.PruneWorkspaceRPMs(buildfile, h.workspacefile) + return nil +} + +func (h *WorkspaceHandler) Write() error { + return bazel.WriteWorkspace(false, h.workspacefile, h.workspace) +} + +type LockFileHandler struct { + filename string + config *bazeldnf.Config +} + +func NewLockFileHandler(configname, filename string) (Handler, error) { + return &LockFileHandler{ + filename: filename, + config: &bazeldnf.Config{ + Name: configname, + RPMs: []bazeldnf.RPM{}, + }, + }, nil +} + +func (h *LockFileHandler) Process(pkgs []*api.Package, arch string, buildfile *build.File) error { + return bazel.AddConfigRPMs(h.config, pkgs, arch) +} + +func (h *LockFileHandler) Write() error { + return bazel.WriteLockFile(h.config, h.filename) +} + func NewRpmTreeCmd() *cobra.Command { rpmtreeCmd := &cobra.Command{ @@ -35,8 +134,6 @@ func NewRpmTreeCmd() *cobra.Command { Short: "Writes a rpmtree rule and its rpmdependencies to bazel files", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, required []string) error { - writeToMacro := rpmtreeopts.toMacro != "" - repos, err := repo.LoadRepoFiles(rpmtreeopts.repofiles) if err != nil { return err @@ -68,54 +165,42 @@ func NewRpmTreeCmd() *cobra.Command { if err != nil { return err } - workspace, err := bazel.LoadWorkspace(rpmtreeopts.workspace) + + var handler Handler + var configname string + + if rpmtreeopts.toMacro != "" { + handler, err = NewMacroHandler(rpmtreeopts.toMacro) + } else if rpmtreeopts.lockfile != "" { + configname = rpmtreeopts.configname + handler, err = NewLockFileHandler( + rpmtreeopts.configname, + rpmtreeopts.lockfile, + ) + } else { + handler, err = NewWorkspaceHandler(rpmtreeopts.workspace) + } + if err != nil { return err } - var bzlfile *build.File - var bzl, defName string - if writeToMacro { - bzl, defName, err = bazel.ParseMacro(rpmtreeopts.toMacro) - if err != nil { - return err - } - bzlfile, err = bazel.LoadBzl(bzl) - if err != nil { - return err - } - } + build, err := bazel.LoadBuild(rpmtreeopts.buildfile) if err != nil { return err } - if writeToMacro { - err = bazel.AddBzlfileRPMs(bzlfile, defName, install, rpmtreeopts.arch) - if err != nil { - return err - } - } else { - err = bazel.AddWorkspaceRPMs(workspace, install, rpmtreeopts.arch) - if err != nil { - return err - } - } - bazel.AddTree(rpmtreeopts.name, build, install, rpmtreeopts.arch, rpmtreeopts.public) - if writeToMacro { - bazel.PruneBzlfileRPMs(build, bzlfile, defName) - } else { - bazel.PruneWorkspaceRPMs(build, workspace) + bazel.AddTree(rpmtreeopts.name, configname, build, install, rpmtreeopts.arch, rpmtreeopts.public) + + if err := handler.Process(install, rpmtreeopts.arch, build); err != nil { + return err } + logrus.Info("Writing bazel files.") - err = bazel.WriteWorkspace(false, workspace, rpmtreeopts.workspace) + err = handler.Write() if err != nil { return err } - if writeToMacro { - err = bazel.WriteBzl(false, bzlfile, bzl) - if err != nil { - return err - } - } + err = bazel.WriteBuild(false, build, rpmtreeopts.buildfile) if err != nil { return err @@ -136,6 +221,8 @@ func NewRpmTreeCmd() *cobra.Command { rpmtreeCmd.Flags().StringVarP(&rpmtreeopts.workspace, "workspace", "w", "WORKSPACE", "Bazel workspace file") rpmtreeCmd.Flags().StringVarP(&rpmtreeopts.toMacro, "to-macro", "", "", "Tells bazeldnf to write the RPMs to a macro in the given bzl file instead of the WORKSPACE file. The expected format is: macroFile%defName") rpmtreeCmd.Flags().StringVarP(&rpmtreeopts.buildfile, "buildfile", "b", "rpm/BUILD.bazel", "Build file for RPMs") + rpmtreeCmd.Flags().StringVar(&rpmtreeopts.configname, "configname", "rpms", "config name to use in lockfile") + rpmtreeCmd.Flags().StringVar(&rpmtreeopts.lockfile, "lockfile", "", "lockfile for RPMs") rpmtreeCmd.Flags().StringVar(&rpmtreeopts.name, "name", "", "rpmtree rule name") rpmtreeCmd.Flags().StringArrayVar(&rpmtreeopts.forceIgnoreRegex, "force-ignore-with-dependencies", []string{}, "Packages matching these regex patterns will not be installed. Allows force-removing unwanted dependencies. Be careful, this can lead to hidden missing dependencies.") rpmtreeCmd.MarkFlagRequired("name") diff --git a/pkg/api/bazeldnf/BUILD.bazel b/pkg/api/bazeldnf/BUILD.bazel index 3176fe7..9b44a1f 100644 --- a/pkg/api/bazeldnf/BUILD.bazel +++ b/pkg/api/bazeldnf/BUILD.bazel @@ -2,7 +2,10 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "bazeldnf", - srcs = ["repo.go"], + srcs = [ + "config.go", + "repo.go", + ], importpath = "github.com/rmohr/bazeldnf/pkg/api/bazeldnf", visibility = ["//visibility:public"], ) diff --git a/pkg/api/bazeldnf/config.go b/pkg/api/bazeldnf/config.go new file mode 100644 index 0000000..3372613 --- /dev/null +++ b/pkg/api/bazeldnf/config.go @@ -0,0 +1,12 @@ +package bazeldnf + +type RPM struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + URLs []string `json:"urls"` +} + +type Config struct { + Name string `json:"name"` + RPMs []RPM `json:"rpms"` +} diff --git a/pkg/bazel/BUILD.bazel b/pkg/bazel/BUILD.bazel index be333c8..c43e65a 100644 --- a/pkg/bazel/BUILD.bazel +++ b/pkg/bazel/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/api", + "//pkg/api/bazeldnf", "@com_github_bazelbuild_buildtools//build:go_default_library", "@com_github_bazelbuild_buildtools//edit:go_default_library", ], diff --git a/pkg/bazel/bazel.go b/pkg/bazel/bazel.go index 64580e8..dbd8973 100644 --- a/pkg/bazel/bazel.go +++ b/pkg/bazel/bazel.go @@ -1,9 +1,10 @@ package bazel import ( + "encoding/json" "fmt" - "io/ioutil" "net/url" + "os" "path/filepath" "sort" "strings" @@ -11,6 +12,7 @@ import ( "github.com/bazelbuild/buildtools/build" "github.com/bazelbuild/buildtools/edit" "github.com/rmohr/bazeldnf/pkg/api" + "github.com/rmohr/bazeldnf/pkg/api/bazeldnf" ) type Artifact struct { @@ -18,7 +20,7 @@ type Artifact struct { } func LoadWorkspace(path string) (*build.File, error) { - workspaceData, err := ioutil.ReadFile(path) + workspaceData, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to parse WORSPACE orig: %v", err) } @@ -30,7 +32,7 @@ func LoadWorkspace(path string) (*build.File, error) { } func LoadBuild(path string) (*build.File, error) { - buildfileData, err := ioutil.ReadFile(path) + buildfileData, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to parse BUILD.bazel orig: %v", err) } @@ -42,7 +44,7 @@ func LoadBuild(path string) (*build.File, error) { } func LoadBzl(path string) (*build.File, error) { - bzlData, err := ioutil.ReadFile(path) + bzlData, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to parse bzl orig: %v", err) } @@ -58,7 +60,7 @@ func WriteBuild(dryRun bool, buildfile *build.File, path string) error { fmt.Println(build.FormatString(buildfile)) return nil } - return ioutil.WriteFile(path, build.Format(buildfile), 0666) + return os.WriteFile(path, build.Format(buildfile), 0644) } func WriteWorkspace(dryRun bool, workspace *build.File, path string) error { @@ -66,7 +68,7 @@ func WriteWorkspace(dryRun bool, workspace *build.File, path string) error { fmt.Println(build.FormatString(workspace)) return nil } - return ioutil.WriteFile(path, build.Format(workspace), 0666) + return os.WriteFile(path, build.Format(workspace), 0644) } func WriteBzl(dryRun bool, bzl *build.File, path string) error { @@ -74,7 +76,15 @@ func WriteBzl(dryRun bool, bzl *build.File, path string) error { fmt.Println(build.FormatString(bzl)) return nil } - return ioutil.WriteFile(path, build.Format(bzl), 0666) + return os.WriteFile(path, build.Format(bzl), 0644) +} + +func WriteLockFile(config *bazeldnf.Config, path string) error { + configJson, err := json.Marshal(config) + if err != nil { + return err + } + return os.WriteFile(path, configJson, 0644) } // ParseMacro parses a macro expression of the form macroFile%defName and returns the bzl file and the def name. @@ -258,7 +268,16 @@ func AddTar2Files(name string, rpmtree string, buildfile *build.File, files []st } } -func AddTree(name string, buildfile *build.File, pkgs []*api.Package, arch string, public bool) { +func AddTree(name, configname string, buildfile *build.File, pkgs []*api.Package, arch string, public bool) { + transform := func(n string) string { + return "@"+n+"//rpm" + } + if configname != "" { + transform = func(n string) string { + return "@" + configname + "//" + n + } + } + rpmtrees := map[string]*rpmTree{} for _, rule := range buildfile.Rules("rpmtree") { @@ -269,7 +288,7 @@ func AddTree(name string, buildfile *build.File, pkgs []*api.Package, arch strin rpms := []string{} for _, pkg := range pkgs { pkgName := sanitize(pkg.String() + "." + arch) - rpms = append(rpms, "@"+pkgName+"//rpm") + rpms = append(rpms, transform(pkgName)) } sort.SliceStable(rpms, func(i, j int) bool { return rpms[i] < rpms[j] @@ -486,6 +505,32 @@ func (r *tar2Files) SetFiles(dirs []string, fileMap map[string][]string) { r.Rule.SetAttr("files", filesMapExpr) } +func AddConfigRPMs(config *bazeldnf.Config, pkgs []*api.Package, arch string) error { + for _, pkg := range pkgs { + URLs := []string{} + + for _, mirror := range pkg.Repository.Mirrors { + u, err := url.Parse(mirror) + if err != nil { + return err + } + u = u.JoinPath(pkg.Location.Href) + URLs = append(URLs, u.String()) + } + + config.RPMs = append( + config.RPMs, + bazeldnf.RPM{ + Name: sanitize(pkg.String() + "." + arch), + SHA256: pkg.Checksum.Text, + URLs: URLs, + }, + ) + } + + return nil +} + func sanitize(name string) string { name = strings.ReplaceAll(name, ":", "__") name = strings.ReplaceAll(name, "+", "__plus__") diff --git a/pkg/bazel/bazel_test.go b/pkg/bazel/bazel_test.go index edf6a9a..bca03da 100644 --- a/pkg/bazel/bazel_test.go +++ b/pkg/bazel/bazel_test.go @@ -115,7 +115,7 @@ func TestBuildfileWithRPMs(t *testing.T) { defer os.Remove(tmpFile.Name()) file, err := LoadBuild(tt.orig) g.Expect(err).ToNot(HaveOccurred()) - AddTree("mytree", file, tt.pkgs, "myarch", false) + AddTree("mytree", "", file, tt.pkgs, "myarch", false) err = WriteBuild(false, file, tmpFile.Name()) g.Expect(err).ToNot(HaveOccurred())