EL9 RPM Sqlite3 DB support

.github/workflows/ci.yaml

@@ -17,6 +17,8 @@ jobs:
       CC_TEST_REPORTER_ID: "${{ secrets.CC_TEST_REPORTER_ID }}"
     steps:
     - uses: actions/checkout@v4
+    - name: Set up system
+      run: bin/before_install
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

bin/before_install

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -n "$CI" ]; then
+  echo "== Installing system packages =="
+  sudo apt-get update
+  sudo apt-get install -y librpm-dev
+  echo
+fi

lib/metadata/linux/LinuxPackages.rb

@@ -155,9 +155,13 @@ def procPortage(pd)
     #
     def procRPM(dbDir)
       $log.debug "Processing RPM package database"
-      rpmp = MiqRpmPackages.new(@fs, File.join(dbDir, "Packages"))
-      rpmp.each { |p| @packages << p }
-      rpmp.close
+
+      rpmp = MiqRpmPackages.new(@fs, dbDir)
+      begin
+        rpmp.each { |p| @packages << p }
+      ensure
+        rpmp.close
+      end
     end
 
     #

lib/metadata/linux/MiqRpmPackages.rb

@@ -1,152 +1,29 @@
-require 'binary_struct'
-require 'manageiq/gems/pending'
-require 'util/miq-hash_struct'
-require 'db/MiqBdb/MiqBdb'
-require 'miq_unicode'
 
-# RPM Specification located at: http://jrpm.sourceforge.net/rpmspec/index.html
-
-class MiqRpmPackages
-  using ManageIQ::UnicodeString
-
-  #
-  # The data types we support.
-  #
-  RPM_INT32_TYPE          =  4
-  RPM_STRING_TYPE         =  6
-  RPM_STRING_ARRAY_TYPE   =  8
-  RPM_I18NSTRING_TYPE     =  9
 
-  #
-  # The things we care about.
-  #
-  NAME        = 1000
-  VERSION     = 1001
-  RELEASE     = 1002
-  SUMMARY     = 1004
-  DESCRIPTION = 1005
-  BUILDTIME   = 1006
-  INSTALLTIME = 1008
-  VENDOR      = 1011
-  GROUP       = 1016
-  URL         = 1020
-  ARCH        = 1022
-  REQUIRES    = 1049
-
-  TAGIDS = {
-    1000 => "name",
-    1001 => "version",
-    1002 => "release",
-    1004 => "summary",
-    1005 => "description",
-    1006 => "buildtime",
-    1008 => "installtime",
-    1011 => "vendor",
-    1016 => "category",  # group
-    1020 => "url",
-    1022 => "arch",
-    1049 => "depends",   # requires
-  }
+# RPM Specification located at: http://jrpm.sourceforge.net/rpmspec/index.html
 
-  #
-  # Nubbers on disk are in network byte order.
-  #
-  RPML_HEADER = BinaryStruct.new([
-    'N',        'num_index',
-    "N",        'num_data'
-  ])
-  RPML_HEADER_LEN = RPML_HEADER.size
+require_relative "MiqRpmPackages/Bdb"
+require_relative "MiqRpmPackages/Sqlite"
 
-  ENTRY_INFO = BinaryStruct.new([
-    'N',        'tag',
-    'N',        'ttype',
-    'N',        'offset',
-    'N',        'count'
-  ])
-  ENTRY_INFO_LEN = ENTRY_INFO.size
+class MiqRpmPackages
+  class << self
+    private
 
-  def initialize(fs, dbFile)
-    @pkgDb = MiqBerkeleyDB::MiqBdb.new(dbFile, fs)
-    # Pre-read all pages into the bdb cache, as we will be processing all of them anyway.
-    @pkgDb.readAllPages
+    alias orig_new new
   end
 
-  def each
-    @pkgDb.each_value do |v|
-      next if v.length <= RPML_HEADER_LEN
-
-      hdr = RPML_HEADER.decode(v)
-
-      offset = RPML_HEADER_LEN + (ENTRY_INFO_LEN * hdr['num_index'])
-      if v.length != offset + hdr['num_data']
-        $log.debug "record length = #{v.length}"
-        $log.debug "num_index = #{hdr['num_index']}"
-        $log.debug "num_data = #{hdr['num_data']}"
-        $log.error "Invalid or corrupt RPM database record"
-        next
-      end
-
-      data = v[offset, hdr['num_data']]
-      pkg = {}
-
-      eis = ENTRY_INFO.decode(v[RPML_HEADER_LEN..-1], hdr['num_index'])
-      eis.each do |ei|
-        tag = TAGIDS[ei['tag']]
-        next if tag.nil?
-        pkg[tag] = getVal(data, ei)
-        pkg[tag] = convert(tag, pkg[tag])
+  def self.new(fs, dbDir)
+    if self == MiqRpmPackages
+      if fs.fileExists?(File.join(dbDir, "Packages"))
+        MiqRpmPackages::Bdb.new(fs, File.join(dbDir, "Packages"))
+      elsif fs.fileExists?(File.join(dbDir, "rpmdb.sqlite"))
+        MiqRpmPackages::Sqlite.new(fs, File.join(dbDir, "rpmdb.sqlite"))
+      else
+        raise ArgumentError, "Invalid RPM database"
       end
-      pkg['installed'] = true unless pkg.empty?
-      yield(MiqHashStruct.new(pkg))
-    end
-  end # def each
-
-  def close
-    @pkgDb.close
-  end
-
-  private
-
-  def time_tag?(tag)
-    ['installtime', 'buildtime'].include?(tag)
-  end
-
-  def convert(tag, val)
-    time_tag?(tag) ? Time.at(val).utc : val
-  end
-
-  def getVal(data, ei)
-    case ei['ttype']
-    when RPM_INT32_TYPE         then return(getInt32Val(data, ei['offset']))
-    when RPM_STRING_TYPE        then return(getStringVal(data, ei['offset']))
-    when RPM_STRING_ARRAY_TYPE  then return(getStringArray(data, ei['offset'], ei['count']).join("\n"))
-    when RPM_I18NSTRING_TYPE    then return(getStringArray(data, ei['offset'], ei['count']).join("\n").AsciiToUtf8)
     else
-      $log.warn "MiqRpmPackages.getVal: unsupported data type: #{ei['ttype']}"
-      return("")
-    end
-  end
-
-  def getInt32Val(data, offset)
-    (data[offset, 4].unpack("N").first)
-  end
-
-  def getStringVal(data, offset)
-    eos = data.index(0.chr, offset) - offset
-    (data[offset, eos])
-  end
-
-  def getStringArray(data, offset, count)
-    ra = []
-    cpos = offset
-
-    count.times do
-      s = getStringVal(data, cpos)
-      ra << s
-      cpos += (s.length + 1)
+      orig_new(fs, dbDir)
     end
-    ra.uniq!
-    (ra)
   end
 end # class MiqRPM
 

lib/metadata/linux/MiqRpmPackages/Bdb.rb

@@ -0,0 +1,151 @@
+require 'binary_struct'
+require 'manageiq/gems/pending'
+require 'util/miq-hash_struct'
+require 'db/MiqBdb/MiqBdb'
+require 'miq_unicode'
+
+class MiqRpmPackages
+  class Bdb < MiqRpmPackages
+    using ManageIQ::UnicodeString
+
+    #
+    # The data types we support.
+    #
+    RPM_INT32_TYPE          =  4
+    RPM_STRING_TYPE         =  6
+    RPM_STRING_ARRAY_TYPE   =  8
+    RPM_I18NSTRING_TYPE     =  9
+
+    #
+    # The things we care about.
+    #
+    NAME        = 1000
+    VERSION     = 1001
+    RELEASE     = 1002
+    SUMMARY     = 1004
+    DESCRIPTION = 1005
+    BUILDTIME   = 1006
+    INSTALLTIME = 1008
+    VENDOR      = 1011
+    GROUP       = 1016
+    URL         = 1020
+    ARCH        = 1022
+    REQUIRES    = 1049
+
+    TAGIDS = {
+      1000 => "name",
+      1001 => "version",
+      1002 => "release",
+      1004 => "summary",
+      1005 => "description",
+      1006 => "buildtime",
+      1008 => "installtime",
+      1011 => "vendor",
+      1016 => "category",  # group
+      1020 => "url",
+      1022 => "arch",
+      1049 => "depends",   # requires
+    }
+
+    #
+    # Nubbers on disk are in network byte order.
+    #
+    RPML_HEADER = BinaryStruct.new([
+      'N',        'num_index',
+      "N",        'num_data'
+    ])
+    RPML_HEADER_LEN = RPML_HEADER.size
+
+    ENTRY_INFO = BinaryStruct.new([
+      'N',        'tag',
+      'N',        'ttype',
+      'N',        'offset',
+      'N',        'count'
+    ])
+    ENTRY_INFO_LEN = ENTRY_INFO.size
+
+    def initialize(fs, dbFile)
+      @pkgDb = MiqBerkeleyDB::MiqBdb.new(dbFile, fs)
+      # Pre-read all pages into the bdb cache, as we will be processing all of them anyway.
+      @pkgDb.readAllPages
+    end
+
+    def each
+      @pkgDb.each_value do |v|
+        next if v.length <= RPML_HEADER_LEN
+
+        hdr = RPML_HEADER.decode(v)
+
+        offset = RPML_HEADER_LEN + (ENTRY_INFO_LEN * hdr['num_index'])
+        if v.length != offset + hdr['num_data']
+          $log.debug "record length = #{v.length}"
+          $log.debug "num_index = #{hdr['num_index']}"
+          $log.debug "num_data = #{hdr['num_data']}"
+          $log.error "Invalid or corrupt RPM database record"
+          next
+        end
+
+        data = v[offset, hdr['num_data']]
+        pkg = {}
+
+        eis = ENTRY_INFO.decode(v[RPML_HEADER_LEN..-1], hdr['num_index'])
+        eis.each do |ei|
+          tag = TAGIDS[ei['tag']]
+          next if tag.nil?
+          pkg[tag] = getVal(data, ei)
+          pkg[tag] = convert(tag, pkg[tag])
+        end
+        pkg['installed'] = true unless pkg.empty?
+        yield(MiqHashStruct.new(pkg))
+      end
+    end # def each
+
+    def close
+      @pkgDb.close
+    end
+
+    private
+
+    def time_tag?(tag)
+      ['installtime', 'buildtime'].include?(tag)
+    end
+
+    def convert(tag, val)
+      time_tag?(tag) ? Time.at(val).utc : val
+    end
+
+    def getVal(data, ei)
+      case ei['ttype']
+      when RPM_INT32_TYPE         then return(getInt32Val(data, ei['offset']))
+      when RPM_STRING_TYPE        then return(getStringVal(data, ei['offset']))
+      when RPM_STRING_ARRAY_TYPE  then return(getStringArray(data, ei['offset'], ei['count']).join("\n"))
+      when RPM_I18NSTRING_TYPE    then return(getStringArray(data, ei['offset'], ei['count']).join("\n").AsciiToUtf8)
+      else
+        $log.warn "MiqRpmPackages.getVal: unsupported data type: #{ei['ttype']}"
+        return("")
+      end
+    end
+
+    def getInt32Val(data, offset)
+      (data[offset, 4].unpack("N").first)
+    end
+
+    def getStringVal(data, offset)
+      eos = data.index(0.chr, offset) - offset
+      (data[offset, eos])
+    end
+
+    def getStringArray(data, offset, count)
+      ra = []
+      cpos = offset
+
+      count.times do
+        s = getStringVal(data, cpos)
+        ra << s
+        cpos += (s.length + 1)
+      end
+      ra.uniq!
+      (ra)
+    end
+  end
+end