From 71646cde347ef816a34e5939b52036fd478db629 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Thu, 31 Oct 2024 13:46:46 -0400 Subject: [PATCH 01/13] Split MiqRpmPackages based on backing database --- lib/metadata/linux/LinuxPackages.rb | 2 +- lib/metadata/linux/MiqRpmPackages.rb | 155 ++------------------ lib/metadata/linux/MiqRpmPackages/Bdb.rb | 151 +++++++++++++++++++ lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 4 + spec/metadata/linux/LinuxPackages_spec.rb | 1 + 5 files changed, 173 insertions(+), 140 deletions(-) create mode 100644 lib/metadata/linux/MiqRpmPackages/Bdb.rb create mode 100644 lib/metadata/linux/MiqRpmPackages/Sqlite.rb diff --git a/lib/metadata/linux/LinuxPackages.rb b/lib/metadata/linux/LinuxPackages.rb index 63c3bbd..ed12132 100644 --- a/lib/metadata/linux/LinuxPackages.rb +++ b/lib/metadata/linux/LinuxPackages.rb @@ -155,7 +155,7 @@ def procPortage(pd) # def procRPM(dbDir) $log.debug "Processing RPM package database" - rpmp = MiqRpmPackages.new(@fs, File.join(dbDir, "Packages")) + rpmp = MiqRpmPackages.new(@fs, dbDir) rpmp.each { |p| @packages << p } rpmp.close end diff --git a/lib/metadata/linux/MiqRpmPackages.rb b/lib/metadata/linux/MiqRpmPackages.rb index 6e899cf..83dfa7e 100644 --- a/lib/metadata/linux/MiqRpmPackages.rb +++ b/lib/metadata/linux/MiqRpmPackages.rb @@ -1,152 +1,29 @@ -require 'binary_struct' -require 'manageiq/gems/pending' -require 'util/miq-hash_struct' -require 'db/MiqBdb/MiqBdb' -require 'miq_unicode' -# RPM Specification located at: http://jrpm.sourceforge.net/rpmspec/index.html - -class MiqRpmPackages - using ManageIQ::UnicodeString - - # - # The data types we support. - # - RPM_INT32_TYPE = 4 - RPM_STRING_TYPE = 6 - RPM_STRING_ARRAY_TYPE = 8 - RPM_I18NSTRING_TYPE = 9 - # - # The things we care about. - # - NAME = 1000 - VERSION = 1001 - RELEASE = 1002 - SUMMARY = 1004 - DESCRIPTION = 1005 - BUILDTIME = 1006 - INSTALLTIME = 1008 - VENDOR = 1011 - GROUP = 1016 - URL = 1020 - ARCH = 1022 - REQUIRES = 1049 - - TAGIDS = { - 1000 => "name", - 1001 => "version", - 1002 => "release", - 1004 => "summary", - 1005 => "description", - 1006 => "buildtime", - 1008 => "installtime", - 1011 => "vendor", - 1016 => "category", # group - 1020 => "url", - 1022 => "arch", - 1049 => "depends", # requires - } +# RPM Specification located at: http://jrpm.sourceforge.net/rpmspec/index.html - # - # Nubbers on disk are in network byte order. - # - RPML_HEADER = BinaryStruct.new([ - 'N', 'num_index', - "N", 'num_data' - ]) - RPML_HEADER_LEN = RPML_HEADER.size +require_relative "MiqRpmPackages/Bdb" +require_relative "MiqRpmPackages/Sqlite" - ENTRY_INFO = BinaryStruct.new([ - 'N', 'tag', - 'N', 'ttype', - 'N', 'offset', - 'N', 'count' - ]) - ENTRY_INFO_LEN = ENTRY_INFO.size +class MiqRpmPackages + class << self + private - def initialize(fs, dbFile) - @pkgDb = MiqBerkeleyDB::MiqBdb.new(dbFile, fs) - # Pre-read all pages into the bdb cache, as we will be processing all of them anyway. - @pkgDb.readAllPages + alias orig_new new end - def each - @pkgDb.each_value do |v| - next if v.length <= RPML_HEADER_LEN - - hdr = RPML_HEADER.decode(v) - - offset = RPML_HEADER_LEN + (ENTRY_INFO_LEN * hdr['num_index']) - if v.length != offset + hdr['num_data'] - $log.debug "record length = #{v.length}" - $log.debug "num_index = #{hdr['num_index']}" - $log.debug "num_data = #{hdr['num_data']}" - $log.error "Invalid or corrupt RPM database record" - next - end - - data = v[offset, hdr['num_data']] - pkg = {} - - eis = ENTRY_INFO.decode(v[RPML_HEADER_LEN..-1], hdr['num_index']) - eis.each do |ei| - tag = TAGIDS[ei['tag']] - next if tag.nil? - pkg[tag] = getVal(data, ei) - pkg[tag] = convert(tag, pkg[tag]) + def self.new(fs, dbDir) + if self == MiqRpmPackages + if fs.fileExists?(File.join(dbDir, "Packages")) + MiqRpmPackages::Bdb.new(fs, File.join(dbDir, "Packages")) + elsif fs.fileExists?(File.join(dbDir, "rpmdb.sqlite")) + MiqRpmPackages::Sqlite.new(fs, File.join(dbDir, "rpmdb.sqlite")) + else + raise ArgumentError, "Invalid RPM database" end - pkg['installed'] = true unless pkg.empty? - yield(MiqHashStruct.new(pkg)) - end - end # def each - - def close - @pkgDb.close - end - - private - - def time_tag?(tag) - ['installtime', 'buildtime'].include?(tag) - end - - def convert(tag, val) - time_tag?(tag) ? Time.at(val).utc : val - end - - def getVal(data, ei) - case ei['ttype'] - when RPM_INT32_TYPE then return(getInt32Val(data, ei['offset'])) - when RPM_STRING_TYPE then return(getStringVal(data, ei['offset'])) - when RPM_STRING_ARRAY_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n")) - when RPM_I18NSTRING_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n").AsciiToUtf8) else - $log.warn "MiqRpmPackages.getVal: unsupported data type: #{ei['ttype']}" - return("") - end - end - - def getInt32Val(data, offset) - (data[offset, 4].unpack("N").first) - end - - def getStringVal(data, offset) - eos = data.index(0.chr, offset) - offset - (data[offset, eos]) - end - - def getStringArray(data, offset, count) - ra = [] - cpos = offset - - count.times do - s = getStringVal(data, cpos) - ra << s - cpos += (s.length + 1) + orig_new(fs, dbDir) end - ra.uniq! - (ra) end end # class MiqRPM diff --git a/lib/metadata/linux/MiqRpmPackages/Bdb.rb b/lib/metadata/linux/MiqRpmPackages/Bdb.rb new file mode 100644 index 0000000..315420a --- /dev/null +++ b/lib/metadata/linux/MiqRpmPackages/Bdb.rb @@ -0,0 +1,151 @@ +require 'binary_struct' +require 'manageiq/gems/pending' +require 'util/miq-hash_struct' +require 'db/MiqBdb/MiqBdb' +require 'miq_unicode' + +class MiqRpmPackages + class Bdb < MiqRpmPackages + using ManageIQ::UnicodeString + + # + # The data types we support. + # + RPM_INT32_TYPE = 4 + RPM_STRING_TYPE = 6 + RPM_STRING_ARRAY_TYPE = 8 + RPM_I18NSTRING_TYPE = 9 + + # + # The things we care about. + # + NAME = 1000 + VERSION = 1001 + RELEASE = 1002 + SUMMARY = 1004 + DESCRIPTION = 1005 + BUILDTIME = 1006 + INSTALLTIME = 1008 + VENDOR = 1011 + GROUP = 1016 + URL = 1020 + ARCH = 1022 + REQUIRES = 1049 + + TAGIDS = { + 1000 => "name", + 1001 => "version", + 1002 => "release", + 1004 => "summary", + 1005 => "description", + 1006 => "buildtime", + 1008 => "installtime", + 1011 => "vendor", + 1016 => "category", # group + 1020 => "url", + 1022 => "arch", + 1049 => "depends", # requires + } + + # + # Nubbers on disk are in network byte order. + # + RPML_HEADER = BinaryStruct.new([ + 'N', 'num_index', + "N", 'num_data' + ]) + RPML_HEADER_LEN = RPML_HEADER.size + + ENTRY_INFO = BinaryStruct.new([ + 'N', 'tag', + 'N', 'ttype', + 'N', 'offset', + 'N', 'count' + ]) + ENTRY_INFO_LEN = ENTRY_INFO.size + + def initialize(fs, dbFile) + @pkgDb = MiqBerkeleyDB::MiqBdb.new(dbFile, fs) + # Pre-read all pages into the bdb cache, as we will be processing all of them anyway. + @pkgDb.readAllPages + end + + def each + @pkgDb.each_value do |v| + next if v.length <= RPML_HEADER_LEN + + hdr = RPML_HEADER.decode(v) + + offset = RPML_HEADER_LEN + (ENTRY_INFO_LEN * hdr['num_index']) + if v.length != offset + hdr['num_data'] + $log.debug "record length = #{v.length}" + $log.debug "num_index = #{hdr['num_index']}" + $log.debug "num_data = #{hdr['num_data']}" + $log.error "Invalid or corrupt RPM database record" + next + end + + data = v[offset, hdr['num_data']] + pkg = {} + + eis = ENTRY_INFO.decode(v[RPML_HEADER_LEN..-1], hdr['num_index']) + eis.each do |ei| + tag = TAGIDS[ei['tag']] + next if tag.nil? + pkg[tag] = getVal(data, ei) + pkg[tag] = convert(tag, pkg[tag]) + end + pkg['installed'] = true unless pkg.empty? + yield(MiqHashStruct.new(pkg)) + end + end # def each + + def close + @pkgDb.close + end + + private + + def time_tag?(tag) + ['installtime', 'buildtime'].include?(tag) + end + + def convert(tag, val) + time_tag?(tag) ? Time.at(val).utc : val + end + + def getVal(data, ei) + case ei['ttype'] + when RPM_INT32_TYPE then return(getInt32Val(data, ei['offset'])) + when RPM_STRING_TYPE then return(getStringVal(data, ei['offset'])) + when RPM_STRING_ARRAY_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n")) + when RPM_I18NSTRING_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n").AsciiToUtf8) + else + $log.warn "MiqRpmPackages.getVal: unsupported data type: #{ei['ttype']}" + return("") + end + end + + def getInt32Val(data, offset) + (data[offset, 4].unpack("N").first) + end + + def getStringVal(data, offset) + eos = data.index(0.chr, offset) - offset + (data[offset, eos]) + end + + def getStringArray(data, offset, count) + ra = [] + cpos = offset + + count.times do + s = getStringVal(data, cpos) + ra << s + cpos += (s.length + 1) + end + ra.uniq! + (ra) + end + end +end diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb new file mode 100644 index 0000000..830164b --- /dev/null +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -0,0 +1,4 @@ +class MiqRpmPackages + class Sqlite < MiqRpmPackages + end +end diff --git a/spec/metadata/linux/LinuxPackages_spec.rb b/spec/metadata/linux/LinuxPackages_spec.rb index 5365abd..dbfebf4 100644 --- a/spec/metadata/linux/LinuxPackages_spec.rb +++ b/spec/metadata/linux/LinuxPackages_spec.rb @@ -42,6 +42,7 @@ context "with an RPM directory" do before do expect(fs).to receive(:fileDirectory?).with(MiqLinux::Packages::RPM_DB).and_return(true) + expect(fs).to receive(:fileExists?).with(File.join(MiqLinux::Packages::RPM_DB, "Packages")).and_return(true) end context "with a Packages Berkeley DB file" do From 4ea9a51814befd0a779cd619c35d0f28e1cf0f2c Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Thu, 9 Jan 2025 15:47:44 -0500 Subject: [PATCH 02/13] Extract rpmdb.sqlite from MiqFs --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index 830164b..91948e7 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -1,4 +1,40 @@ class MiqRpmPackages class Sqlite < MiqRpmPackages + def initialize(fs, dbFile) + @fs = fs + @db_file_path = dbFile + + @rpmdb_tempfile = nil + @rpmdb_path = nil + + if fs.present? + rpmdb_file = fs.fileOpen(@db_file_path, "r") + @rpmdb_tempfile = Tempfile.new("rpmdb", :binmode => true) + + loop do + chunk = rpmdb_file.read(1024) + break if chunk.nil? + + @rpmdb_tempfile.write(chunk) + end + + @rpmdb_tempfile.close + rpmdb_file.close + + @rpmdb_path = @rpmdb_tempfile.path + else + @rpmdb_path = @db_file_path + end + end + + def each + end + + def close + if @rpmdb_tempfile + @rpmdb_tempfile.close + @rpmdb_tempfile.unlink + end + end end end From 1debc412f721019e0519b003e21494d7ae806cce Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Mon, 13 Jan 2025 15:51:09 -0500 Subject: [PATCH 03/13] Use RPM FFI gem to parse rpmdb.sqlite --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 31 +++++++++++++-------- manageiq-smartstate.gemspec | 1 + 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index 91948e7..df1db66 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -4,37 +4,44 @@ def initialize(fs, dbFile) @fs = fs @db_file_path = dbFile - @rpmdb_tempfile = nil - @rpmdb_path = nil + @rpmdb_tempdir = nil + @rpmdb_path = nil if fs.present? - rpmdb_file = fs.fileOpen(@db_file_path, "r") - @rpmdb_tempfile = Tempfile.new("rpmdb", :binmode => true) + @rpmdb_tempdir = Dir.mktmpdir("rpmdb-") + FileUtils.mkdir_p("#{@rpmdb_tempdir}/var/lib/rpm") + + rpmdb_tempfile = File.open("#{@rpmdb_tempdir}/var/lib/rpm/rpmdb.sqlite", "wb") + rpmdb_file = fs.fileOpen(@db_file_path, "r") loop do - chunk = rpmdb_file.read(1024) + chunk = rpmdb_file.read(4_096) break if chunk.nil? - @rpmdb_tempfile.write(chunk) + rpmdb_tempfile.write(chunk) end - @rpmdb_tempfile.close + rpmdb_tempfile.close rpmdb_file.close - @rpmdb_path = @rpmdb_tempfile.path + @rpmdb_path = rpmdb_tempfile.path else @rpmdb_path = @db_file_path end end def each + require 'rpm' + + RPM.transaction(@rpmdb_tempdir) do |ts| + ts.each do |pkg| + yield pkg + end + end end def close - if @rpmdb_tempfile - @rpmdb_tempfile.close - @rpmdb_tempfile.unlink - end + FileUtils.rm_rf(@rpmdb_tempdir) if @rpmdb_tempdir end end end diff --git a/manageiq-smartstate.gemspec b/manageiq-smartstate.gemspec index e88c1e5..ef82d6c 100644 --- a/manageiq-smartstate.gemspec +++ b/manageiq-smartstate.gemspec @@ -28,6 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "linux_block_device", "~>0.2.1" spec.add_dependency "manageiq-password", "< 2" spec.add_dependency "memory_buffer", ">=0.1.0" + spec.add_dependency "rpm", "~>0.0.5" spec.add_dependency "rufus-lru", "~>1.0.3" spec.add_dependency "sys-uname", "~>1.2.1" spec.add_dependency "uuidtools", "~>2.1" From 1b07c5d0a22febe592071f22a1bad7de8cf28ed6 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Tue, 14 Jan 2025 15:25:27 -0500 Subject: [PATCH 04/13] Yield a compatible hash struct from Sqlite/Bdb --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index df1db66..942a65f 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -35,7 +35,15 @@ def each RPM.transaction(@rpmdb_tempdir) do |ts| ts.each do |pkg| - yield pkg + tagids = %w[name version release summary description buildtime vendor arch installtime] + + result = tagids.each_with_object({}) { |tag, obj| obj[tag] = pkg[tag.to_sym] } + # These have different tag names for the tagid + result["category"] = pkg[:group] + result["depends"] = pkg[:requirename] + result["installed"] = true unless result.empty? + + yield MiqHashStruct.new(result) end end end From 5061e88e3fd59a0fb642d3bd373c2737b2034fea Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Thu, 16 Jan 2025 13:02:06 -0500 Subject: [PATCH 05/13] Ensure that the rpmdb is closed --- lib/metadata/linux/LinuxPackages.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/metadata/linux/LinuxPackages.rb b/lib/metadata/linux/LinuxPackages.rb index ed12132..b9f4262 100644 --- a/lib/metadata/linux/LinuxPackages.rb +++ b/lib/metadata/linux/LinuxPackages.rb @@ -155,9 +155,13 @@ def procPortage(pd) # def procRPM(dbDir) $log.debug "Processing RPM package database" + rpmp = MiqRpmPackages.new(@fs, dbDir) - rpmp.each { |p| @packages << p } - rpmp.close + begin + rpmp.each { |p| @packages << p } + ensure + rpmp.close + end end # From 2900afa5df0688c443f4b0e79ac395c2c9a9ade5 Mon Sep 17 00:00:00 2001 From: Jason Frey Date: Thu, 16 Jan 2025 13:56:00 -0500 Subject: [PATCH 06/13] Add support for sqlite smartstate from a macOS host --- Gemfile | 1 + lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 17 +++++++++++++++-- spec/metadata/linux/LinuxPackages_spec.rb | 20 +++++++++++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index e8d2063..2f779f2 100644 --- a/Gemfile +++ b/Gemfile @@ -7,3 +7,4 @@ gem "manageiq-gems-pending", :git => "https://github.com/ManageIQ/manageiq-gems- # Modified gems for vmware_web_service. Setting sources here since they are git references gem "handsoap", "=0.2.5.5", :require => false, :source => "https://rubygems.manageiq.org" +gem "rpm", "~>0.0.5", :require => false, :git => "https://github.com/dmacvicar/ruby-rpm-ffi.git" diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index 942a65f..c4f5b3a 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -1,3 +1,6 @@ +require "metadata/linux/LinuxPackages" +require "tmpdir" + class MiqRpmPackages class Sqlite < MiqRpmPackages def initialize(fs, dbFile) @@ -9,9 +12,10 @@ def initialize(fs, dbFile) if fs.present? @rpmdb_tempdir = Dir.mktmpdir("rpmdb-") - FileUtils.mkdir_p("#{@rpmdb_tempdir}/var/lib/rpm") + rpmdb_dir = File.join(@rpmdb_tempdir, rpm_db_relative) + FileUtils.mkdir_p(rpmdb_dir) - rpmdb_tempfile = File.open("#{@rpmdb_tempdir}/var/lib/rpm/rpmdb.sqlite", "wb") + rpmdb_tempfile = File.open(File.join(rpmdb_dir, "rpmdb.sqlite"), "wb") rpmdb_file = fs.fileOpen(@db_file_path, "r") loop do @@ -51,5 +55,14 @@ def each def close FileUtils.rm_rf(@rpmdb_tempdir) if @rpmdb_tempdir end + + private + + def rpm_db_relative + @rpm_db_relative ||= begin + parts = [RbConfig::CONFIG["host_os"] =~ /darwin/ ? "opt/homebrew" : nil, MiqLinux::Packages::RPM_DB].compact + File.join(*parts) + end + end end end diff --git a/spec/metadata/linux/LinuxPackages_spec.rb b/spec/metadata/linux/LinuxPackages_spec.rb index dbfebf4..ab35b64 100644 --- a/spec/metadata/linux/LinuxPackages_spec.rb +++ b/spec/metadata/linux/LinuxPackages_spec.rb @@ -42,11 +42,11 @@ context "with an RPM directory" do before do expect(fs).to receive(:fileDirectory?).with(MiqLinux::Packages::RPM_DB).and_return(true) - expect(fs).to receive(:fileExists?).with(File.join(MiqLinux::Packages::RPM_DB, "Packages")).and_return(true) end context "with a Packages Berkeley DB file" do before do + expect(fs).to receive(:fileExists?).with(File.join(MiqLinux::Packages::RPM_DB, "Packages")).and_return(true) expect(fs) .to receive(:fileOpen) .with(File.join(MiqLinux::Packages::RPM_DB, "Packages"), "r") @@ -76,6 +76,24 @@ ) end end + + context "with a rpmdb.sqlite DB file" do + before do + expect(fs).to receive(:fileExists?).with(File.join(MiqLinux::Packages::RPM_DB, "Packages")).and_return(false) + expect(fs).to receive(:fileExists?).with(File.join(MiqLinux::Packages::RPM_DB, "rpmdb.sqlite")).and_return(true) + expect(fs) + .to receive(:fileOpen) + .with(File.join(MiqLinux::Packages::RPM_DB, "rpmdb.sqlite"), "r") + .and_return(File.open(File.expand_path('../../db/MiqSqlite/rpmdb-empty.sqlite', __dir__), "r")) + expect(fs).to receive(:present?).and_return(true) + end + + it "returns a list of rpm packages" do + result = described_class.new(fs) + + expect(result.packages.count).to eq(0) + end + end end context "with a conarydb file" do From 5d537777eda324833c5e538fe3acfafba69f38c1 Mon Sep 17 00:00:00 2001 From: Jason Frey Date: Thu, 16 Jan 2025 14:00:49 -0500 Subject: [PATCH 07/13] Fix issue where on Linux we were using an absolute path --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index c4f5b3a..e7ca092 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -60,7 +60,7 @@ def close def rpm_db_relative @rpm_db_relative ||= begin - parts = [RbConfig::CONFIG["host_os"] =~ /darwin/ ? "opt/homebrew" : nil, MiqLinux::Packages::RPM_DB].compact + parts = [RbConfig::CONFIG["host_os"] =~ /darwin/ ? "opt/homebrew" : nil, "var/lib/rpm"].compact File.join(*parts) end end From 39d9484b43cb8513344f04883eb17b01b47207b2 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Thu, 16 Jan 2025 14:01:23 -0500 Subject: [PATCH 08/13] Install librpm-dev for RPM specs --- .github/workflows/ci.yaml | 2 ++ bin/before_install | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100755 bin/before_install diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e12b10c..8085133 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,6 +17,8 @@ jobs: CC_TEST_REPORTER_ID: "${{ secrets.CC_TEST_REPORTER_ID }}" steps: - uses: actions/checkout@v4 + - name: Set up system + run: bin/before_install - name: Set up Ruby uses: ruby/setup-ruby@v1 with: diff --git a/bin/before_install b/bin/before_install new file mode 100755 index 0000000..351d56a --- /dev/null +++ b/bin/before_install @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ -n "$CI" ]; then + echo "== Installing system packages ==" + sudo apt-get update + sudo apt-get install -y librpm-dev + echo +fi From 081da05107aec0296b5d22cb6d1f9ea207e8f50f Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Thu, 16 Jan 2025 14:08:07 -0500 Subject: [PATCH 09/13] Replace fs.present? with fs.nil? --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 6 +++--- spec/metadata/linux/LinuxPackages_spec.rb | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index e7ca092..4cccb20 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -10,7 +10,9 @@ def initialize(fs, dbFile) @rpmdb_tempdir = nil @rpmdb_path = nil - if fs.present? + if fs.nil? + @rpmdb_path = @db_file_path + else @rpmdb_tempdir = Dir.mktmpdir("rpmdb-") rpmdb_dir = File.join(@rpmdb_tempdir, rpm_db_relative) FileUtils.mkdir_p(rpmdb_dir) @@ -29,8 +31,6 @@ def initialize(fs, dbFile) rpmdb_file.close @rpmdb_path = rpmdb_tempfile.path - else - @rpmdb_path = @db_file_path end end diff --git a/spec/metadata/linux/LinuxPackages_spec.rb b/spec/metadata/linux/LinuxPackages_spec.rb index ab35b64..e3ddc10 100644 --- a/spec/metadata/linux/LinuxPackages_spec.rb +++ b/spec/metadata/linux/LinuxPackages_spec.rb @@ -85,7 +85,6 @@ .to receive(:fileOpen) .with(File.join(MiqLinux::Packages::RPM_DB, "rpmdb.sqlite"), "r") .and_return(File.open(File.expand_path('../../db/MiqSqlite/rpmdb-empty.sqlite', __dir__), "r")) - expect(fs).to receive(:present?).and_return(true) end it "returns a list of rpm packages" do From ffa1e418516bd9067c83feee5791444a0e7497b6 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Mon, 3 Feb 2025 14:38:37 -0500 Subject: [PATCH 10/13] Switch to rpm2 gem --- Gemfile | 1 - manageiq-smartstate.gemspec | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 2f779f2..e8d2063 100644 --- a/Gemfile +++ b/Gemfile @@ -7,4 +7,3 @@ gem "manageiq-gems-pending", :git => "https://github.com/ManageIQ/manageiq-gems- # Modified gems for vmware_web_service. Setting sources here since they are git references gem "handsoap", "=0.2.5.5", :require => false, :source => "https://rubygems.manageiq.org" -gem "rpm", "~>0.0.5", :require => false, :git => "https://github.com/dmacvicar/ruby-rpm-ffi.git" diff --git a/manageiq-smartstate.gemspec b/manageiq-smartstate.gemspec index ef82d6c..bc1e20b 100644 --- a/manageiq-smartstate.gemspec +++ b/manageiq-smartstate.gemspec @@ -28,7 +28,7 @@ Gem::Specification.new do |spec| spec.add_dependency "linux_block_device", "~>0.2.1" spec.add_dependency "manageiq-password", "< 2" spec.add_dependency "memory_buffer", ">=0.1.0" - spec.add_dependency "rpm", "~>0.0.5" + spec.add_dependency "rpm2", "~>0.1.0" spec.add_dependency "rufus-lru", "~>1.0.3" spec.add_dependency "sys-uname", "~>1.2.1" spec.add_dependency "uuidtools", "~>2.1" From 990468021c698e43a884624ad5f367e951d917ac Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Mon, 3 Feb 2025 12:34:15 -0500 Subject: [PATCH 11/13] Use rpmdb.sqlite with a single sample package Improve the specs by testing with an rpmdb.sqlite that has a package installed --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 2 +- spec/metadata/linux/LinuxPackages_spec.rb | 17 +++++++++++++++-- spec/metadata/linux/data/rpm/rpmdb.sqlite | Bin 0 -> 217088 bytes 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 spec/metadata/linux/data/rpm/rpmdb.sqlite diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index 4cccb20..69c71c4 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -44,7 +44,7 @@ def each result = tagids.each_with_object({}) { |tag, obj| obj[tag] = pkg[tag.to_sym] } # These have different tag names for the tagid result["category"] = pkg[:group] - result["depends"] = pkg[:requirename] + result["depends"] = pkg[:requirename]&.join("\n") result["installed"] = true unless result.empty? yield MiqHashStruct.new(result) diff --git a/spec/metadata/linux/LinuxPackages_spec.rb b/spec/metadata/linux/LinuxPackages_spec.rb index e3ddc10..3584363 100644 --- a/spec/metadata/linux/LinuxPackages_spec.rb +++ b/spec/metadata/linux/LinuxPackages_spec.rb @@ -84,13 +84,26 @@ expect(fs) .to receive(:fileOpen) .with(File.join(MiqLinux::Packages::RPM_DB, "rpmdb.sqlite"), "r") - .and_return(File.open(File.expand_path('../../db/MiqSqlite/rpmdb-empty.sqlite', __dir__), "r")) + .and_return(File.open(File.expand_path('data/rpm/rpmdb.sqlite', __dir__), "r")) end it "returns a list of rpm packages" do result = described_class.new(fs) - expect(result.packages.count).to eq(0) + expect(result.packages.count).to eq(1) + + package = result.packages.first + expect(package.to_h).to include( + "name" => "simple", + "version" => "1.0", + "release" => "0", + "summary" => "Simple dummy package", + "vendor" => nil, + "category" => "Development", + "arch" => "i586", + "depends" => "rpmlib(CompressedFileNames)\nrpmlib(PayloadFilesHavePrefix)\nrpmlib(PayloadIsLzma)", + "installed" => true + ) end end end diff --git a/spec/metadata/linux/data/rpm/rpmdb.sqlite b/spec/metadata/linux/data/rpm/rpmdb.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..77ab3e7aaa0758ae225a740feeacab8e65100e8d GIT binary patch literal 217088 zcmeI*Z)_XqeZcWMk`hJBG9AatMR8=E>#7uOQ4%FfmYRBt|In@)S<)nBh_j~Zi9AWP z`9~os$1YZ&5(+dgf(?7sy;y@_XtBK*&;dh+t!RJ(eY0XHFm%~46nT+i9gr1U|ExhW zpy|5jxjRzN!%#*sU}7zN36ak||L(c_{Eif#yR*FfJ9GJ(Yp<6pMW<#@7{`rJ$avbe z4Z{eD=cIVnA8D}=sb@rLD$DyVhK!MC{-{GXj{icoIUE1Qv8A4`^<<9z;ONucKkdGB zNoEj-`Q_~M)^+abksU? z#@rpOIco)XdDBU6x=zlm>@5vkUCLZu$=EBGugqoa#9l?h9=hY+w-YOw=T;K-{KASo z|Lok{u)SH@F50v6E14UaC8ao@yVs)h`odCX_Qt&ZOy-t7Bzq?9rOfrrQfB^YX4y_G zI@vqUhFeYeb;+GG{ZXs0&)n_zw>Q60%uOzPucw=~(<`iR%qrQ4^Zo=bN+9->b9t5SP9dRi2My-p3 z=1z}iv#?ez7u=dF5BUbITKJgGqgFNA`=L;)lvvz3DdH~1^W&~IS`B>Fvfc*_r-ei9 zq{L(5(Wo^zXznJxc&Lrk<&%0u)T*|GjAyEq-)s57urjV#rYDgYY zL!vgSheRzq@qpp9kf@#1kQ`S-qBc^OAJIdiR<$D}x8v%k)ncW5H=onrBud$FT}Y{E ziic98gk$5F`e~)KsYgR8itj&^77L||5{j`N_0vjeqbwfN1EJKkB@Rc`I4rp@Zs#j{ z9F(#i9Y?8Yii1+4#-Uq{gVLrR2c_uf{zGYTP`apbIHJZuX`?K5>v2$O+7XBOSf`b+ z%%b;8<3^>ty;Vo?GDma{t%3jB!7EY-PIpBu%QAOQc^}kXt@jesciZpQhVR*H(WsRk zFn3IEkFPjY@2K1RR@f-%(s#X4-gL+{%9R7|Go9jl=Rokiv(caq+bD?cH-dJgHJU5P z=Q^U+z<{}X)dcqTgtjk*|4_7O_f6> zu}{OIXL8H*CILGW-YV-;#(EmkRoZ}|CvZ-{kYE`9BTv{}bf@+|Nhky5;jP#ZzowMD`7f=idZ*xo(;NW{^K3QrZ4nLB3n0(&z7j zymSzGQ?`@qlV96JdsCLnd_1T>8RVstY=1h)%W|;|=8#B*N2FJzvj5W}mHjP|%J!cO z@{fyD)_+!{Vml*!B9--pApb;=e(+{uzTkLv!91@&(S^&3I{he3Wb$iE}=R}HsnRP)8Hf@`El#*8sz*~{3u?PBr1 zz2#rYG8P>%)e+YY{g!slwTs(swN`PAYc1O;4Gd%P$N&6ag?wrIUh4kmZaX7YF?SL} z8{Ib+=ZtIaUAIu)D!Qebv01BaeSCEErI%hBk)s#yk7Ub5!|Q3}C#R=mC-D##M#s)q zcfKk!a^3PdHnV=w9UEUykI#&)x#Q_{&bc^t(HWnbT2D_-tW6lx+35=tO1WG!nqK@JPYI&b*hsaK9jR;;W%FMJkATomhVD&I zr6$IcqFw~_Nc}0g6!L3BSIfn%id(I^x$F6YH^;3eji7GPxnC$dUPbkj&Ruu0;;!fK zHCNA8=RQ|-lE%cy*vN%c+PEOLO1@qgnex(^k@U#8Va)&a-EYmmKKS&t?NZh$**BeR z>UZ;5r(!RT*q_K1o$Otid%C(^bw_gUZ;4R;dhJs7_53}%S}wY^&3tLYXv*7eq3Q+L zFn%d^{g!l++ttcwbcL_RKYE6)HW}T*{N%*;+pNs z?Dbo#%L~sgUCmq)-Pgq>&g`AkR;65Xv$b+Xlx&w?66Z2f{w!bQo$CEkcC%70mA9)Y zt#_@y9mFy;xs#J*W7zGBPULS%5m;D_E=l+esBKj zS0DU?C>8JfX#T0yuUvoSOE0c353i3T-+k)aGZSOZ%=m@%nKft9S)Z86u4UK9)~BZ~ zPNk>EXQrp8W^!v;XXc_@>&9!}f8Jla`I<<>GA}$8SanO?{cMs1FvL^^2fB*srAbiDgVKy z|M&lXX2j+Hh|2>31Q0*~0R#|0009ILKmY**4u`<8@Y$F$|CiVQjpK)7*VqdL5I_I{ z1Q0*~0R#|0009ILXe}VG|1L&m~v2b5-PQai4zi-4ph`;}^?jYNZ00IagfB*srAbo#$LLmYOAbZWY{7dHvsv|Hu$OJP<$t0R#|0009ILKmY**5J2Fe7w8SA zWBbkjKQQ7SJoG!r4j_O40tg_000IagfB*srAb>y%fp9n$^Y8zE&xpU*q5(w+Ab=l=hPIGAiY0tg_000IagfB*srAb&wIIBguE4`nEs+|DF;5p3;;>1Q0*~0R#|0009ILKmY** z5cmxj7&ap>#JXlC#+;e)3+pp$&ZM(GF_T@(u8*xxPhFf!Pmj+`PfyL{*0RpbMgQ;r zy=lbX{0-kNHV**=5I_I{1Q0*~0R#|0009Ivf%D-PV!!VEUta(33%_i{{~-Q}W3L`t z>iJqv=I9TOKHdG(?n_7BJhB;kJvP_%cGtP+8_}DcU+z5D@r{mXhkbeZYH}xaI%=IcW9|;toV9|x*L81cAXxj#{`B3a*A$sRmMUdX7z$6 z^@FTl+-TVt#0-u~1+YY&<=ue@>iCiiE!D0jJXK@%@L=?3FIcr^nc1QS0nkb2s7fD{YiTOAmrl(~dZldZX6GL35|avsqZH zmJ4ppm4|$TRxNx?=TWN~?fp=wRZ1-GoD^{vrRcc5cQA46Os)s}^ zJMn9`#aZtLbaX6yJL208bcI$CaYT6Np`BGbjUTzl>_cGo#K1vK=8e@(V!07D2VPif_9`enk&fXI-=IVfVq3t zLvFNaEOhGFjq-LJVV{b0T7$i2-aEqP<=2s5_vCViPM}u%hnFl-cKlR0YW4P-yM5k{ z%S!o-=)3FnX~Utl80xgnpEqxL+g5$CkgvI`Rq@H@ma=YBHmq%NQ{_-e?9;I5ncOnH zNx;s8x61mIv7UysmF&$Z{pn_7O1ii-d-L+rEpH-v`Pr3)*?H0LW@dh6c&$)gv*iS` zF_}%u`M+^|Z^F(J0tg_000IagfB*srAb#j?T!<@LS={(CeW&^KJ8^#@FL7AN!|cg=0p~U$w=enu=O8Npt5B z@9fg@_SROxExIND8cT4VNiQ7>PMXwjDQMZnKlh|}RL(-p3`ebG(%emXr)BhJdTvxd zMWa`?<80jR^Xhr`#d&wREkpj52c+A zRk|nv8B40?-IX@V;&5{)_BOc<=L#!B;=oPR58Nd;TP}*@HYd;gHBK;VrRQ}Zt*+6- zkA_yK#G{-Lmm|fP{uN@aRXrkF;kgG5sYONWro?6HV^J%SFn7;;ZxpSSmPzWd(dydq zF4;LY7`1LpnmZ})b7iICl&b4_aXr1|-3=eE9Ma_;*x!#?t0qe5cDg?rwK8IZlKXE^ z5RsN^`Zcyk3uw!g?A=@S%vtd%GHLG4cyVpjxK-K5^eDG#(vDz%?jup_(s^^I&!fG* zPnes^2X)b=+NL*TQ>_x$m(E13r^ML3EyoT*($xK<4<1^JV^dez9&iPA#o2Pii0GYRyn>tuQQZhaDC-^VX8L)9U;nQ`&UI#cS|Jz2-zz86K00IagfB*srAb2N009ILKmY**5I_I{1Q2MO0Q3L0(K9dt2q1s}0tg_000IagfB*sqCcypw z2S&|62q1s}0tg_000IagfB*srv`v8P|81jZU<43A009ILKmY**5I_I{1P)Aq`Tv1Y zGY|p@Ab Date: Tue, 4 Feb 2025 12:09:04 -0500 Subject: [PATCH 12/13] Use RPM _dbpath constant --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index 69c71c4..7f6b6e5 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -4,6 +4,8 @@ class MiqRpmPackages class Sqlite < MiqRpmPackages def initialize(fs, dbFile) + require "rpm" + @fs = fs @db_file_path = dbFile @@ -14,7 +16,7 @@ def initialize(fs, dbFile) @rpmdb_path = @db_file_path else @rpmdb_tempdir = Dir.mktmpdir("rpmdb-") - rpmdb_dir = File.join(@rpmdb_tempdir, rpm_db_relative) + rpmdb_dir = File.join(@rpmdb_tempdir, RPM['_dbpath']) FileUtils.mkdir_p(rpmdb_dir) rpmdb_tempfile = File.open(File.join(rpmdb_dir, "rpmdb.sqlite"), "wb") @@ -35,8 +37,6 @@ def initialize(fs, dbFile) end def each - require 'rpm' - RPM.transaction(@rpmdb_tempdir) do |ts| ts.each do |pkg| tagids = %w[name version release summary description buildtime vendor arch installtime] @@ -55,14 +55,5 @@ def each def close FileUtils.rm_rf(@rpmdb_tempdir) if @rpmdb_tempdir end - - private - - def rpm_db_relative - @rpm_db_relative ||= begin - parts = [RbConfig::CONFIG["host_os"] =~ /darwin/ ? "opt/homebrew" : nil, "var/lib/rpm"].compact - File.join(*parts) - end - end end end From e26193341172f7d7756c9b6e063c917060713b82 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Tue, 4 Feb 2025 12:55:18 -0500 Subject: [PATCH 13/13] Fix circular require in MiqRpmPackages::Sqlite --- lib/metadata/linux/MiqRpmPackages/Sqlite.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb index 7f6b6e5..8b0a9eb 100644 --- a/lib/metadata/linux/MiqRpmPackages/Sqlite.rb +++ b/lib/metadata/linux/MiqRpmPackages/Sqlite.rb @@ -1,4 +1,3 @@ -require "metadata/linux/LinuxPackages" require "tmpdir" class MiqRpmPackages