Skip to content

Commit

Permalink
Split MiqRpmPackages based on backing database
Browse files Browse the repository at this point in the history
  • Loading branch information
@agrare
agrare committed Oct 31, 2024
1 parent 34bd6f9 commit 43f8f90
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 140 deletions.
2 changes: 1 addition & 1 deletion lib/metadata/linux/LinuxPackages.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def procPortage(pd)
#
def procRPM(dbDir)
$log.debug "Processing RPM package database"
rpmp = MiqRpmPackages.new(@fs, File.join(dbDir, "Packages"))
rpmp = MiqRpmPackages.new(@fs, dbDir)
rpmp.each { |p| @packages << p }
rpmp.close
end
Expand Down
155 changes: 16 additions & 139 deletions lib/metadata/linux/MiqRpmPackages.rb
View file
Original file line number Diff line number Diff line change
@@ -1,152 +1,29 @@
require 'binary_struct'
require 'manageiq/gems/pending'
require 'util/miq-hash_struct'
require 'db/MiqBdb/MiqBdb'
require 'miq_unicode'

# RPM Specification located at: http://jrpm.sourceforge.net/rpmspec/index.html

class MiqRpmPackages
using ManageIQ::UnicodeString

#
# The data types we support.
#
RPM_INT32_TYPE = 4
RPM_STRING_TYPE = 6
RPM_STRING_ARRAY_TYPE = 8
RPM_I18NSTRING_TYPE = 9

#
# The things we care about.
#
NAME = 1000
VERSION = 1001
RELEASE = 1002
SUMMARY = 1004
DESCRIPTION = 1005
BUILDTIME = 1006
INSTALLTIME = 1008
VENDOR = 1011
GROUP = 1016
URL = 1020
ARCH = 1022
REQUIRES = 1049

TAGIDS = {
1000 => "name",
1001 => "version",
1002 => "release",
1004 => "summary",
1005 => "description",
1006 => "buildtime",
1008 => "installtime",
1011 => "vendor",
1016 => "category", # group
1020 => "url",
1022 => "arch",
1049 => "depends", # requires
}
# RPM Specification located at: http://jrpm.sourceforge.net/rpmspec/index.html

#
# Nubbers on disk are in network byte order.
#
RPML_HEADER = BinaryStruct.new([
'N', 'num_index',
"N", 'num_data'
])
RPML_HEADER_LEN = RPML_HEADER.size
require_relative "MiqRpmPackages/Bdb"
require_relative "MiqRpmPackages/Sqlite"

ENTRY_INFO = BinaryStruct.new([
'N', 'tag',
'N', 'ttype',
'N', 'offset',
'N', 'count'
])
ENTRY_INFO_LEN = ENTRY_INFO.size
class MiqRpmPackages
class << self
private

def initialize(fs, dbFile)
@pkgDb = MiqBerkeleyDB::MiqBdb.new(dbFile, fs)
# Pre-read all pages into the bdb cache, as we will be processing all of them anyway.
@pkgDb.readAllPages
alias orig_new new
end

def each
@pkgDb.each_value do |v|
next if v.length <= RPML_HEADER_LEN

hdr = RPML_HEADER.decode(v)

offset = RPML_HEADER_LEN + (ENTRY_INFO_LEN * hdr['num_index'])
if v.length != offset + hdr['num_data']
$log.debug "record length = #{v.length}"
$log.debug "num_index = #{hdr['num_index']}"
$log.debug "num_data = #{hdr['num_data']}"
$log.error "Invalid or corrupt RPM database record"
next
end

data = v[offset, hdr['num_data']]
pkg = {}

eis = ENTRY_INFO.decode(v[RPML_HEADER_LEN..-1], hdr['num_index'])
eis.each do |ei|
tag = TAGIDS[ei['tag']]
next if tag.nil?
pkg[tag] = getVal(data, ei)
pkg[tag] = convert(tag, pkg[tag])
def self.new(fs, dbDir)
if self == MiqRpmPackages
if fs.fileExists?(File.join(dbDir, "Packages"))
MiqRpmPackages::Bdb.new(fs, File.join(dbDir, "Packages"))
elsif fs.fileExists?(File.join(dbDir, "rpmdb.sqlite"))
MiqRpmPackages::Sqlite.new(fs, File.join(dbDir, "rpmdb.sqlite"))
else
raise ArgumentError, "Invalid RPM database"
end
pkg['installed'] = true unless pkg.empty?
yield(MiqHashStruct.new(pkg))
end
end # def each

def close
@pkgDb.close
end

private

def time_tag?(tag)
['installtime', 'buildtime'].include?(tag)
end

def convert(tag, val)
time_tag?(tag) ? Time.at(val).utc : val
end

def getVal(data, ei)
case ei['ttype']
when RPM_INT32_TYPE then return(getInt32Val(data, ei['offset']))
when RPM_STRING_TYPE then return(getStringVal(data, ei['offset']))
when RPM_STRING_ARRAY_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n"))
when RPM_I18NSTRING_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n").AsciiToUtf8)
else
$log.warn "MiqRpmPackages.getVal: unsupported data type: #{ei['ttype']}"
return("")
end
end

def getInt32Val(data, offset)
(data[offset, 4].unpack("N").first)
end

def getStringVal(data, offset)
eos = data.index(0.chr, offset) - offset
(data[offset, eos])
end

def getStringArray(data, offset, count)
ra = []
cpos = offset

count.times do
s = getStringVal(data, cpos)
ra << s
cpos += (s.length + 1)
orig_new(fs, dbDir)
end
ra.uniq!
(ra)
end
end # class MiqRPM

Expand Down
151 changes: 151 additions & 0 deletions lib/metadata/linux/MiqRpmPackages/Bdb.rb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
require 'binary_struct'
require 'manageiq/gems/pending'
require 'util/miq-hash_struct'
require 'db/MiqBdb/MiqBdb'
require 'miq_unicode'

class MiqRpmPackages
class Bdb < MiqRpmPackages
using ManageIQ::UnicodeString

#
# The data types we support.
#
RPM_INT32_TYPE = 4
RPM_STRING_TYPE = 6
RPM_STRING_ARRAY_TYPE = 8
RPM_I18NSTRING_TYPE = 9

#
# The things we care about.
#
NAME = 1000
VERSION = 1001
RELEASE = 1002
SUMMARY = 1004
DESCRIPTION = 1005
BUILDTIME = 1006
INSTALLTIME = 1008
VENDOR = 1011
GROUP = 1016
URL = 1020
ARCH = 1022
REQUIRES = 1049

TAGIDS = {
1000 => "name",
1001 => "version",
1002 => "release",
1004 => "summary",
1005 => "description",
1006 => "buildtime",
1008 => "installtime",
1011 => "vendor",
1016 => "category", # group
1020 => "url",
1022 => "arch",
1049 => "depends", # requires
}

#
# Nubbers on disk are in network byte order.
#
RPML_HEADER = BinaryStruct.new([
'N', 'num_index',
"N", 'num_data'
])
RPML_HEADER_LEN = RPML_HEADER.size

ENTRY_INFO = BinaryStruct.new([
'N', 'tag',
'N', 'ttype',
'N', 'offset',
'N', 'count'
])
ENTRY_INFO_LEN = ENTRY_INFO.size

def initialize(fs, dbFile)
@pkgDb = MiqBerkeleyDB::MiqBdb.new(dbFile, fs)
# Pre-read all pages into the bdb cache, as we will be processing all of them anyway.
@pkgDb.readAllPages
end

def each
@pkgDb.each_value do |v|
next if v.length <= RPML_HEADER_LEN

hdr = RPML_HEADER.decode(v)

offset = RPML_HEADER_LEN + (ENTRY_INFO_LEN * hdr['num_index'])
if v.length != offset + hdr['num_data']
$log.debug "record length = #{v.length}"
$log.debug "num_index = #{hdr['num_index']}"
$log.debug "num_data = #{hdr['num_data']}"
$log.error "Invalid or corrupt RPM database record"
next
end

data = v[offset, hdr['num_data']]
pkg = {}

eis = ENTRY_INFO.decode(v[RPML_HEADER_LEN..-1], hdr['num_index'])
eis.each do |ei|
tag = TAGIDS[ei['tag']]
next if tag.nil?
pkg[tag] = getVal(data, ei)
pkg[tag] = convert(tag, pkg[tag])
end
pkg['installed'] = true unless pkg.empty?
yield(MiqHashStruct.new(pkg))
end
end # def each

def close
@pkgDb.close
end

private

def time_tag?(tag)
['installtime', 'buildtime'].include?(tag)
end

def convert(tag, val)
time_tag?(tag) ? Time.at(val).utc : val
end

def getVal(data, ei)
case ei['ttype']
when RPM_INT32_TYPE then return(getInt32Val(data, ei['offset']))
when RPM_STRING_TYPE then return(getStringVal(data, ei['offset']))
when RPM_STRING_ARRAY_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n"))
when RPM_I18NSTRING_TYPE then return(getStringArray(data, ei['offset'], ei['count']).join("\n").AsciiToUtf8)
else
$log.warn "MiqRpmPackages.getVal: unsupported data type: #{ei['ttype']}"
return("")
end
end

def getInt32Val(data, offset)
(data[offset, 4].unpack("N").first)
end

def getStringVal(data, offset)
eos = data.index(0.chr, offset) - offset
(data[offset, eos])
end

def getStringArray(data, offset, count)
ra = []
cpos = offset

count.times do
s = getStringVal(data, cpos)
ra << s
cpos += (s.length + 1)
end
ra.uniq!
(ra)
end
end
end
4 changes: 4 additions & 0 deletions lib/metadata/linux/MiqRpmPackages/Sqlite.rb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
class MiqRpmPackages
class Sqlite < MiqRpmPackages
end
end
1 change: 1 addition & 0 deletions spec/metadata/linux/LinuxPackages_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
context "with an RPM directory" do
before do
expect(fs).to receive(:fileDirectory?).with(MiqLinux::Packages::RPM_DB).and_return(true)
expect(fs).to receive(:fileExists?).with(File.join(MiqLinux::Packages::RPM_DB, "Packages")).and_return(true)
end

context "with a Packages Berkeley DB file" do
Expand Down

0 comments on commit 43f8f90

Please sign in to comment.