Merge the perfimprov branch

Performance improvement, mainly adding read_variables and
write_variables accessor to the database to enable cache-based matching

integration/test_unittest.py

@@ -10,7 +10,7 @@
 from npf.test import Test
 from npf.build import Build
 from npf.variable import dtype, numeric_dict
-from npf.types.dataset import Run, ImmutableRun
+from npf.types.dataset import Run
 
 import numpy as np
 
@@ -95,11 +95,13 @@ def test_runequality():
     rb["A"] = 1
     b = Run(rb)
     assert a == b
-    assert ImmutableRun(ra) == ImmutableRun(rb)
-    assert ImmutableRun(ra) == b
     assert a.inside(b)
     assert b.inside(a)
     assert a.__hash__() == b.__hash__()
+    h = a.__hash__()
+    a.write_variables()["A"] = 3
+    assert a.__hash__() != h
+    assert a != b
 
 def test_local_executor():
     l = LocalExecutor()

modules/fastclick-echo.npf

@@ -0,0 +1,2 @@
+%script deps=fastclick sudo=true
+click --dpdk -l 0-7 -a ${self:0:pci} -- -e "FromDPDKDevice(0, PROMISC true) -> EtherMirror -> ToDPDKDevice(0);"

modules/fastclick-play-single-mt.npf

@@ -30,6 +30,7 @@ RCV_NIC=0
 LIMIT=50000000
 LIMIT_TIME=10
 GEN_THREADS=4
+GEN_RX_THREADS=4
 GEN_PIPELINE=1
 udpgen:GEN_PIPELINE=0
 GEN_TX_PIPELINE=0
@@ -38,7 +39,7 @@ GEN_BURST=32
 PROMISC=false
 promisc:PROMISC=true
 IGNORE?=0
-PAUSE=none
+GEN_PAUSE=none
 GEN_DESC=0
 END_AFTER=0
 GEN_PROGRESSIVE=0
@@ -169,6 +170,7 @@ Idle->[1]output;
 elementclass Numberise { $magic |
     input-> Strip(14)
     $GEN_NUMBER
+    $GEN_FNT
 $(( "" if $GEN_IPV6 else " -> ResetIPChecksum()  " ))
     -> Unstrip(14) -> output
 }
@@ -201,7 +203,7 @@ $(( " StaticThreadSched(fdIN0 0/0, unqueue0 0/0);" if $GEN_PIPELINE else "" ))
 
 $(( " StaticThreadSched(tdIN 0/%d);" % ($GEN_THREADS + 1) if $GEN_TX_PIPELINE else "" ))
 
-receiveIN :: FromDPDKDevice($port, VERBOSE $rxverbose, MAC $INsrcmac, PROMISC $PROMISC, PAUSE $PAUSE, NDESC $GEN_DESC, MAXTHREADS $GEN_THREADS, NUMA false, ACTIVE $GEN_RX)
+receiveIN :: FromDPDKDevice($port, VERBOSE $rxverbose, MAC $INsrcmac, PROMISC $PROMISC, PAUSE $GEN_PAUSE, NDESC $GEN_DESC, MAXTHREADS $GEN_RX_THREADS, NUMA false, ACTIVE $GEN_RX)
 
 elementclass Receiver { $mac, $dir |
     input[0]
@@ -214,7 +216,7 @@ $GEN_MAGIC
     c[1] //Not for this computer or broadcasts
     -> Discard;
 
-$((  "\n".join([ "magic[%d] -> tsd%d :: $GEN_TSDIFF(gen%d/rt, OFFSET %d, N $NRECORD, SAMPLE $SAMPLE, NANO $GEN_NANO ) -> Unstrip(14) ->  avg%d :: AverageCounterMP(IGNORE $ignore) -> Discard;  tsd%d[1] -> Print('WARNING: Untimestamped packet on thread %d', 64) -> Discard;" % (t,t,t,${NUMBEROFFSET} + 2,t,t,t) for t in range($GEN_THREADS) ]) ))
+$((  "\n".join([ "magic[%d] -> tsd%d :: $GEN_TSDIFF(gen%d/rt, OFFSET %d, N $NRECORD, SAMPLE $SAMPLE, NANO $GEN_NANO ) -> Unstrip(14) ->  avg%d :: AverageCounterIMP(IGNORE $ignore) -> Discard;  tsd%d[1] -> Print('WARNING: Untimestamped packet on thread %d', 64) -> Discard;" % (t,t,t,${NUMBEROFFSET} + 2,t,t,t) for t in range($GEN_THREADS) ]) ))
 
 
 $(( (  "avg :: HandlerAggregate( " + ",".join(["ELEMENT avg%d" % (i) for i in range($GEN_THREADS) ]) + " );" ) if 1 else "" ))

modules/fastclick-replay-single-mt.npf

@@ -63,6 +63,7 @@ SND_NIC?=EXPAND($CLIENT_NIC)
 RCV_NIC?=EXPAND($CLIENT_NIC)
 DODUMP?=0
 GEN_DUMP?=0
+udpgen:GEN_SEQUENTIAL?=1
 GEN_SEQUENTIAL?=0
 gdb:GDB=gdb -ex run -ex "signal 2" -ex bt -batch -args
 udpgen:GEN_TRACE_SUP+=->MarkMACHeader->EnsureDPDKBuffer->Pad

npf/build.py

@@ -115,7 +115,7 @@ def _writeversion(self, filename, all_results, allow_overwrite):
         f.seek(0)
         for run, results in all_results.items():
             v = []
-            for key, val in sorted(run.variables.items()):
+            for key, val in sorted(run.read_variables().items()):
                 if type(val) is tuple:
                     val = val[1]
                 v.append((key + ":" + str(val).replace('\\:', ':').replace(':','\\:')).replace('\\,', ',').replace(',','\\,'))
@@ -145,14 +145,27 @@ def load_results(self, test, kind=False, cache=True):
                 if os.path.basename(filename) in f:
                     kind = f[f.rfind("-") + 1 :]
                     f = filename + kind
-                    kr[kind] = self._load_results(test, f, cache)
+                    kr[kind] = self._load_results(f, cache)
             return kr
 
         else:
             filename = self.__resultFilename(test)
-            return self._load_results(test, filename, cache)
+            return self._load_results(filename, cache)
 
-    def _load_results(self, test, filename, cache):
+    def _load_results(self, filename, cache):
+        """
+        The function `_load_results` reads data from a file, parses it, and returns the results in a
+        dictionary format.
+
+        :param filename: The `filename` parameter is a string that represents the name of the file from
+        which the results will be loaded
+        :param cache: The `cache` parameter is a boolean value that determines whether or not to use a
+        cache to store and retrieve previously loaded results. If `cache` is `True`, the function will
+        check if the `filename` is already in the cache and return the cached results if available. If
+        `cache`
+        :return: the variable "all_results", which is a dictionary containing the parsed results from
+        the file.
+        """
         if not Path(filename).exists():
             return None
         if cache:
@@ -171,7 +184,7 @@ def _load_results(self, test, filename, cache):
                 for v_data in re.split(r'(?<!\\),', variables_data):
                     if v_data.strip():
                         k, v = re.split(r'(?<!\\):', v_data)
-                        variables[k] = variable.get_numeric(v) if test.variables.is_numeric(k) else str(v).replace('\\:',':')
+                        variables[k] = variable.get_numeric(v) if variable.is_numeric(k) else str(v).replace('\\:',':')
                 results = {}
 
                 results_data = results_data.strip()[1:-1].split('},{')

npf/executor/executor.py

@@ -1,6 +1,5 @@
 import re
 
-
 from colorama import Fore, Back, Style
 
 foreColors = [Fore.BLACK, Fore.RED, Fore.GREEN, Fore.YELLOW, Fore.BLUE, Fore.MAGENTA, Fore.CYAN, Fore.WHITE]

npf/executor/sshexecutor.py

@@ -66,17 +66,17 @@ def exec(self, cmd, bin_paths : List[str] = None,
             print("Executing on %s%s (PATH+=%s) :\n%s" % (self.addr,(' with sudo' if sudo and self.user != "root" else ''),':'.join(path_list) + (("NS:"  + virt) if virt else ""), cmd.strip()))
 
         # The pre-command goes into the test folder
-        pre = 'cd '+ self.path + ';'
+        pre = 'cd '+ self.path + ';\n'
 
         if self.path:
             env['NPF_ROOT'] = self.path
             env['NPF_CWD_PATH'] = os.path.relpath(npf.cwd_path(options),self.path)
             env['NPF_EXPERIMENT_PATH'] = '../' + os.path.relpath(npf.experiment_path(), self.path)
             env['NPF_ROOT_PATH'] = '../' + os.path.relpath(npf.npf_root_path(), self.path)
-
+        env_str=""
         for k,v in env.items():
             if v is not None:
-                pre += 'export ' + k + '='+v+'\n'
+                env_str += 'export ' + k + '='+v+'\n'
         if path_list:
             path_cmd = 'export PATH="%s:$PATH"\n' % (':'.join(path_list))
         else:
@@ -89,6 +89,7 @@ def exec(self, cmd, bin_paths : List[str] = None,
             if stdin is not None:
                 unbuffer = unbuffer + " -p"
 
+        cmd = env_str + cmd
         if sudo and self.user != "root":
             cmd = "sudo -E  mkdir -p "+testdir+" && sudo -E " + virt +" "+unbuffer+" bash -c '"+path_cmd + cmd.replace("'", "'\"'\"'") + "'";
         else: