Merge pull request #4 from smitterl/add_2_wise

Add 2 wise
smitterl · Mar 9, 2021 · 34d6863 · 34d6863
2 parents 3bf8439 + fa31bbb
commit 34d6863
Show file tree

Hide file tree

Showing 4 changed files with 144 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -1,14 +1,15 @@
 https://github.com/avocado-framework/avocado-vt/ uses cartesian_config to create list of test cases over the largest possible set of test parameter combinations, i.e. variants.
 
 The `minimal` function takes a list of test case names (p1.p2.p3, e.g. virsh.domstats.argument) and filters out test cases whose test parameter values, represented by the variant name, have already been covered in another test case earlier in the list.
+The `pairwise` function takes a list of test case names (p1.p2.p3, e.g. virsh.domstats.argument) and filters out test cases whose pairs of test parameter values, represented by the variant names, have already been covered in another test case earlier in the list.
 
-An example of what will be filtered can be seen in the unit test.
+An example of what will be filtered can be seen in the unit tests.
 
 Example:
 
 A filtered output could look like:
 ```bash
-# python3 minimal_avocado_list.py -s virsh.boot,boot_integration
+# python3 avocado_list.py -s -m "--vt-type libvirt --vt-machine-type s390-virtio" virsh.boot,boot_integration
 virsh.boot.loadparm
 virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_dev.os_loader.valid_loader_type.valid_readonly
 virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_dev.os_loader.valid_loader_type.no_readonly
@@ -39,9 +40,57 @@ boot_integration.by_qemu_on_s390.boot_dev.boot_non_default
 The number of test cases was reduced from 52 to 26.
 ```
 
+As `pairwise` also targets the interaction of two test parameters, naturally it filters out less test cases:
+```bash
+# python3 avocado_list.py -s -p "--vt-type libvirt --vt-machine-type s390-virtio" virsh.boot,boot_integration
+virsh.boot.loadparm
+virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_dev.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_dev.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_dev.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_dev.os_loader.no_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_order.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_order.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.file_disk.boot_order.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.block_disk.boot_dev.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.block_disk.boot_dev.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.block_disk.boot_dev.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.block_disk.boot_order.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.ceph.boot_dev.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.ceph.boot_dev.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.ceph.boot_dev.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.ceph.boot_dev.os_loader.no_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.ceph.boot_order.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.ceph.boot_order.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.ceph.boot_order.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.glusterfs.boot_dev.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.glusterfs.boot_dev.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.glusterfs.boot_dev.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.hd.network_disk.glusterfs.boot_order.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.cdrom.boot_dev.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.cdrom.boot_dev.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.cdrom.boot_dev.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.cdrom.boot_dev.os_loader.no_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.cdrom.boot_order.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.boot.cdrom.boot_order.os_loader.valid_loader_type.no_readonly
+virsh.boot.by_seabios.positive_test.options.boot.cdrom.boot_order.os_loader.no_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.options.no_boot.os_loader.valid_loader_type.valid_readonly
+virsh.boot.by_seabios.positive_test.boot_order_big_integer
+virsh.boot.by_seabios.positive_test.two_same_boot_dev
+virsh.boot.by_seabios.negative_test.not_existing_loader
+virsh.boot.by_seabios.negative_test.not_existing_loader_type
+virsh.boot.by_seabios.negative_test.not_existing_boot_dev
+virsh.boot.by_seabios.negative_test.special_boot_order.negative
+virsh.boot.by_seabios.negative_test.special_boot_order.character
+virsh.boot.by_seabios.negative_test.special_boot_order.zero
+boot_integration.by_qemu_on_s390.boot_dev.check_menu
+boot_integration.by_qemu_on_s390.boot_dev.boot_non_default
+
+The number of test cases was reduced from 52 to 41.
+```
+
 The filter can be double checked by
 ```bash
-# python3 minimal_avocado_list.py -t virsh.boot,boot_integration > original.list
-# python3 minimal_avocado_list.py virsh.boot,boot_integration > filtered.list
+# python3 avocado_list.py -t -m "--vt-type libvirt --vt-machine-type s390-virtio" virsh.boot,boot_integration > original.list
+# python3 avocado_list.py -m "--vt-type libvirt --vt-machine-type s390-virtio" virsh.boot,boot_integration > filtered.list
 diff -y filtered.list original.list
 ```
diff --git a/minimal_avocado_list.py → avocado_list.py b/minimal_avocado_list.py → avocado_list.py
@@ -1,43 +1,63 @@
 import re
 import subprocess
 import sys
-from minimal_filter import minimal
+from filters import minimal, pairwise
 
 # holds the vt only filter
 ONLY = []
 PRINT_ORIGINAL = False
 SHOW_STATS = False
+APPLY_FILTER = None
+OPTIONS = None
 
 def print_help(exit_code):
-    print(sys.argv[0] + " [-t] [-s] <--vt-only-filter argument>")
+    print(sys.argv[0] + " [-t] [-s] <-p|-m> [<vt-options>] <--vt-only-filter argument>")
     print()
+    print(" <vt-options> are additional filters enclosed by quotes, e.g. '--vt-type libvirt --vt-machine-type s390-virtio'")
     print(" <vt-only-filter> is a comma separated list of test names")
     print(" by passing flag -t the avocado command result is printed and not filtered")
     print(" by passing flag -s the number of original test names and filtered ones are shown")
+    print(" -p selects the pairwise filter selecting all available test cases with maximal coverage of pairs, -s selects the minimal filter selecting all available test cases covering at least all variants")
     sys.exit(exit_code)
 
 def parse_args():
     if len(sys.argv) == 1:
         print_help(1)
 
     global ONLY
+    global PRINT_ORIGINAL
+    global SHOW_STATS
+    global APPLY_FILTER
+    global OPTIONS
+
     ONLY = sys.argv[-1]
 
     if ONLY.startswith("-h"):
         print_help(0)
+
+    if not ("-m" in sys.argv or "-p" in sys.argv):
+        print_help(1)
 
     if "-t" in sys.argv: 
-        global PRINT_ORIGINAL
         PRINT_ORIGINAL = True
 
     if "-s" in sys.argv:
-        global SHOW_STATS
         SHOW_STATS = True
 
+    if "-p" in sys.argv:
+        APPLY_FILTER = pairwise.pairwise
+
+    if "-m" in sys.argv:
+        APPLY_FILTER = minimal.minimal
+
+    if "--" in sys.argv[-2]:
+        OPTIONS = sys.argv[-2]
+
+
 def get_avocado_list():
     test_names = []
-    command = ("avocado list --vt-type libvirt --vt-machine-type s390-virtio"
-               " --vt-only-filter %s --paginator off" % ONLY)
+    command = ("avocado list %s --vt-only-filter %s --paginator off" %
+               (OPTIONS, ONLY))
     output = subprocess.check_output(command, shell=True).decode()
     for row in output.split('\n'):
         if not row.startswith("VT"):
@@ -54,7 +74,7 @@ def get_avocado_list():
             print(name)
         sys.exit(0)
 
-    filtered_test_names = minimal(test_names)
+    filtered_test_names = APPLY_FILTER(test_names)
     for name in filtered_test_names:
         print(name)
 

diff --git a/minimal_filter.py → filters/minimal.py b/minimal_filter.py → filters/minimal.py
diff --git a/filters/pairwise.py b/filters/pairwise.py
@@ -0,0 +1,64 @@
+SEP = "."
+
+def parts(line):
+    p = line.split(SEP)
+    return p
+
+def pairwise_covered(candidates, line_parts):
+    for i in range(len(line_parts) - 1):
+        for j in range(i + 1, len(line_parts)):
+            cand_i_j = [x for x in candidates if
+                    x[i] == line_parts[i] and
+                    x[j] == line_parts[j]]
+            if not cand_i_j:
+                return False
+    return True
+
+def pairwise(lines):
+    diag = [] 
+
+    for line in lines:
+
+        line_parts = parts(line)
+        if line_parts in diag:
+            continue
+
+        candidates = [x for x in diag if len(x) == len(line_parts)]
+        if not (candidates and
+                pairwise_covered(candidates, line_parts)):
+            diag.append(line_parts)
+            continue
+
+    return [".".join(x) for x in diag] 
+
+
+import unittest
+
+test_lines = [
+        "a1.b1.c1",
+        "a1.b1.c2",
+        "a1.b2.c1",
+        "a1.b2.c2",
+        "a2.b1.c1",
+        "a2.b1.c2",
+        "a2.b2.c1",
+        "a2.b2.c2",
+        "a1.d1"
+        ]
+
+class TestDiagonal(unittest.TestCase):
+
+    def test_pairwise(self):
+        pairwised = pairwise(test_lines)
+        self.assertEqual(pairwised, [
+            "a1.b1.c1",
+            "a1.b1.c2",
+            "a1.b2.c1",
+            "a1.b2.c2",
+            "a2.b1.c1",
+            "a2.b1.c2",
+            "a2.b2.c1",
+            "a1.d1"])
+
+if __name__ == '__main__':
+    unittest.main()