diff --git a/integration/test_unittest.py b/integration/test_unittest.py index 3e37f2f9..b166f902 100644 --- a/integration/test_unittest.py +++ b/integration/test_unittest.py @@ -10,7 +10,7 @@ from npf.test import Test from npf.build import Build from npf.variable import dtype, numeric_dict -from npf.types.dataset import Run, ImmutableRun +from npf.types.dataset import Run import numpy as np @@ -95,11 +95,13 @@ def test_runequality(): rb["A"] = 1 b = Run(rb) assert a == b - assert ImmutableRun(ra) == ImmutableRun(rb) - assert ImmutableRun(ra) == b assert a.inside(b) assert b.inside(a) assert a.__hash__() == b.__hash__() + h = a.__hash__() + a.write_variables()["A"] = 3 + assert a.__hash__() != h + assert a != b def test_local_executor(): l = LocalExecutor() diff --git a/modules/fastclick-echo.npf b/modules/fastclick-echo.npf new file mode 100644 index 00000000..5e2613df --- /dev/null +++ b/modules/fastclick-echo.npf @@ -0,0 +1,2 @@ +%script deps=fastclick sudo=true +click --dpdk -l 0-7 -a ${self:0:pci} -- -e "FromDPDKDevice(0, PROMISC true) -> EtherMirror -> ToDPDKDevice(0);" diff --git a/modules/fastclick-play-single-mt.npf b/modules/fastclick-play-single-mt.npf index bcf0f48d..5e95596f 100644 --- a/modules/fastclick-play-single-mt.npf +++ b/modules/fastclick-play-single-mt.npf @@ -30,6 +30,7 @@ RCV_NIC=0 LIMIT=50000000 LIMIT_TIME=10 GEN_THREADS=4 +GEN_RX_THREADS=4 GEN_PIPELINE=1 udpgen:GEN_PIPELINE=0 GEN_TX_PIPELINE=0 @@ -38,7 +39,7 @@ GEN_BURST=32 PROMISC=false promisc:PROMISC=true IGNORE?=0 -PAUSE=none +GEN_PAUSE=none GEN_DESC=0 END_AFTER=0 GEN_PROGRESSIVE=0 @@ -169,6 +170,7 @@ Idle->[1]output; elementclass Numberise { $magic | input-> Strip(14) $GEN_NUMBER + $GEN_FNT $(( "" if $GEN_IPV6 else " -> ResetIPChecksum() " )) -> Unstrip(14) -> output } @@ -201,7 +203,7 @@ $(( " StaticThreadSched(fdIN0 0/0, unqueue0 0/0);" if $GEN_PIPELINE else "" )) $(( " StaticThreadSched(tdIN 0/%d);" % ($GEN_THREADS + 1) if $GEN_TX_PIPELINE else "" )) -receiveIN :: FromDPDKDevice($port, VERBOSE $rxverbose, MAC $INsrcmac, PROMISC $PROMISC, PAUSE $PAUSE, NDESC $GEN_DESC, MAXTHREADS $GEN_THREADS, NUMA false, ACTIVE $GEN_RX) +receiveIN :: FromDPDKDevice($port, VERBOSE $rxverbose, MAC $INsrcmac, PROMISC $PROMISC, PAUSE $GEN_PAUSE, NDESC $GEN_DESC, MAXTHREADS $GEN_RX_THREADS, NUMA false, ACTIVE $GEN_RX) elementclass Receiver { $mac, $dir | input[0] @@ -214,7 +216,7 @@ $GEN_MAGIC c[1] //Not for this computer or broadcasts -> Discard; -$(( "\n".join([ "magic[%d] -> tsd%d :: $GEN_TSDIFF(gen%d/rt, OFFSET %d, N $NRECORD, SAMPLE $SAMPLE, NANO $GEN_NANO ) -> Unstrip(14) -> avg%d :: AverageCounterMP(IGNORE $ignore) -> Discard; tsd%d[1] -> Print('WARNING: Untimestamped packet on thread %d', 64) -> Discard;" % (t,t,t,${NUMBEROFFSET} + 2,t,t,t) for t in range($GEN_THREADS) ]) )) +$(( "\n".join([ "magic[%d] -> tsd%d :: $GEN_TSDIFF(gen%d/rt, OFFSET %d, N $NRECORD, SAMPLE $SAMPLE, NANO $GEN_NANO ) -> Unstrip(14) -> avg%d :: AverageCounterIMP(IGNORE $ignore) -> Discard; tsd%d[1] -> Print('WARNING: Untimestamped packet on thread %d', 64) -> Discard;" % (t,t,t,${NUMBEROFFSET} + 2,t,t,t) for t in range($GEN_THREADS) ]) )) $(( ( "avg :: HandlerAggregate( " + ",".join(["ELEMENT avg%d" % (i) for i in range($GEN_THREADS) ]) + " );" ) if 1 else "" )) diff --git a/modules/fastclick-replay-single-mt.npf b/modules/fastclick-replay-single-mt.npf index f6006c2f..619a9ee6 100644 --- a/modules/fastclick-replay-single-mt.npf +++ b/modules/fastclick-replay-single-mt.npf @@ -63,6 +63,7 @@ SND_NIC?=EXPAND($CLIENT_NIC) RCV_NIC?=EXPAND($CLIENT_NIC) DODUMP?=0 GEN_DUMP?=0 +udpgen:GEN_SEQUENTIAL?=1 GEN_SEQUENTIAL?=0 gdb:GDB=gdb -ex run -ex "signal 2" -ex bt -batch -args udpgen:GEN_TRACE_SUP+=->MarkMACHeader->EnsureDPDKBuffer->Pad diff --git a/npf/build.py b/npf/build.py index f86f3e37..f18b7bff 100755 --- a/npf/build.py +++ b/npf/build.py @@ -115,7 +115,7 @@ def _writeversion(self, filename, all_results, allow_overwrite): f.seek(0) for run, results in all_results.items(): v = [] - for key, val in sorted(run.variables.items()): + for key, val in sorted(run.read_variables().items()): if type(val) is tuple: val = val[1] v.append((key + ":" + str(val).replace('\\:', ':').replace(':','\\:')).replace('\\,', ',').replace(',','\\,')) @@ -145,14 +145,27 @@ def load_results(self, test, kind=False, cache=True): if os.path.basename(filename) in f: kind = f[f.rfind("-") + 1 :] f = filename + kind - kr[kind] = self._load_results(test, f, cache) + kr[kind] = self._load_results(f, cache) return kr else: filename = self.__resultFilename(test) - return self._load_results(test, filename, cache) + return self._load_results(filename, cache) - def _load_results(self, test, filename, cache): + def _load_results(self, filename, cache): + """ + The function `_load_results` reads data from a file, parses it, and returns the results in a + dictionary format. + + :param filename: The `filename` parameter is a string that represents the name of the file from + which the results will be loaded + :param cache: The `cache` parameter is a boolean value that determines whether or not to use a + cache to store and retrieve previously loaded results. If `cache` is `True`, the function will + check if the `filename` is already in the cache and return the cached results if available. If + `cache` + :return: the variable "all_results", which is a dictionary containing the parsed results from + the file. + """ if not Path(filename).exists(): return None if cache: @@ -171,7 +184,7 @@ def _load_results(self, test, filename, cache): for v_data in re.split(r'(? 2: @@ -155,10 +153,7 @@ def __init__(self, fname): self[re.compile(k)] = v def search(self, map_v): - for k,v in self.items(): - if re.search(k,str(map_v)): - return v - return None + return next((v for k, v in self.items() if re.search(k,str(map_v))), None) def guess_type(d): @@ -418,7 +413,7 @@ def combine_variables(self, run_list, variables_to_merge): if len(variables_to_merge) == 1: for run in run_list: s = [] - for k, v in run.variables.items(): + for k, v in run.read_variables().items(): if k in variables_to_merge: s.append("%s" % str(v[1] if type(v) is tuple else v)) ss.append(','.join(s)) @@ -428,7 +423,7 @@ def combine_variables(self, run_list, variables_to_merge): for run in run_list: s = [] short_s = {} - for k, v in run.variables.items(): + for k, v in run.read_variables().items(): if k in variables_to_merge: v = str(v[1] if type(v) is tuple else v) s.append("%s = %s" % (self.var_name(k), v)) @@ -459,10 +454,10 @@ def aggregate_variable(self, key, series, method): # continue newrun = run.copy() for k in key.split("+"): - if k in newrun.variables: - del newrun.variables[k] + if k in newrun.write_variables(): + del newrun.write_variables()[k] - newrun.variables[key] = 'AGG' + newrun.write_variables()[key] = 'AGG' aggregates.setdefault(newrun,[]).append(run_results) @@ -509,9 +504,9 @@ def extract_variable_to_series(self, key, vars_values, all_results, dyns, build, for run, run_results in all_results.items(): # if (graph_variables and not run in graph_variables): # continue - if run.variables[key] == value: + if run.read_variables()[key] == value: newrun = run.copy() - del newrun.variables[key] + del newrun.write_variables()[key] newserie[newrun] = run_results new_varsall.add(newrun) @@ -641,15 +636,42 @@ def map_variables(self, map_k, fmap, series, vars_values): transformed_series.append((test, build, new_results)) return transformed_series - def graph(self, filename, options, fileprefix=None, graph_variables: List[Run] = None, title=False, series=None): - """series is a list of triplet (script,build,results) where - result is the output of a script.execute_all()""" + def graph(self, filename, options, fileprefix=None, graph_variables: List[Run] = None, title=False, series:Series=None): + """ + The function "graph" is used to create a graph based on the given parameters and save it to a + file. + + :param filename: The filename parameter is a string that represents the name of the file where + the graph will be saved + :param options: The "options" parameter is a dictionary that contains various options for + customizing the graph. It is the result of the parsing of the arguments, equal to npf.options. + It is passed for legacy reasons. + :param fileprefix: The `fileprefix` parameter is an optional prefix that can be added to the + filename of the graph. It is useful when you want to differentiate between multiple graphs that + are being generated. If `fileprefix` is provided, it will be added to the beginning of the + filename followed by an underscore + :param graph_variables: The `graph_variables` parameter is a list of `Run` objects. Each `Run` + object represents a set of data points that will be plotted on the graph. If none, all observations + will be shown. + :type graph_variables: List[Run] + :param title: The `title` parameter is a boolean value that determines whether or not to display + a title for the graph. If `title` is set to `True`, a title will be displayed on the graph. If + `title` is set to `False`, no title will be displayed, defaults to False (optional) + :param series: The "series" parameter is used to specify the data series that will be plotted on + the graph. It is typically a list of values or a list of lists, where each value or sublist + represents a data series. Each data series will be plotted as a separate line or bar on the + graph + """ self.options = options if self.options.graph_size is None: self.options.graph_size = plt.rcParams["figure.figsize"] if series is None: series = [] + + for test, _, _ in series: + self.scripts.add(test) + # If no graph variables, use the first serie if graph_variables is None: graph_variables = OrderedSet() @@ -657,26 +679,6 @@ def graph(self, filename, options, fileprefix=None, graph_variables: List[Run] = for run, results in serie[2].items(): graph_variables.add(run) - # Get all scripts, and execute pypost - for i, (test, build, all_results) in enumerate(series): - self.scripts.add(test) - - if hasattr(test, 'pypost'): - def common_divide(a,b): - m = min(len(a),len(b)) - return np.array(a)[:m] / np.array(b)[:m] - def results_divide(res,a,b): - for RUN, RESULTS in all_results.items(): - if a in RESULTS and b in RESULTS: - all_results[RUN][res] = common_divide(RESULTS[a], RESULTS[b]) - vs = {'ALL_RESULTS': all_results, 'common_divide': common_divide, 'results_divide': results_divide} - try: - exec(test.pypost.content, vs) - except Exception as e: - print("ERROR WHILE EXECUTING PYPOST SCRIPT:") - print(e) - - if not series: print("No data...") return @@ -724,6 +726,8 @@ def results_divide(res,a,b): self.graphlines = self.configlist("graph_lines") # Combine variables as per the graph_combine_variables config parameter + # for instance if A has [1,2] values and B has [yes,no], graph_combine_variables={A+B:Unique} + # will remove A and B, and create a new Unique variables with ["1, yes", "1, no", "2, yes", "2, no"] for tocombine in self.configlist('graph_combine_variables', []): if type(tocombine) is tuple: toname = tocombine[1] @@ -785,7 +789,7 @@ def results_divide(res,a,b): results=np.asarray([0]) new_results.setdefault(run.copy(), OrderedDict())[result_type] = results / ydiv - for k, v in run.variables.items(): + for k, v in run.read_variables().items(): vars_values.setdefault(k, OrderedSet()).add(v) if new_results: @@ -1448,12 +1452,12 @@ def generate_plot_for_graph(self, i, i_subplot, figure, n_cols, n_lines, vars_va barplot = True elif graph_type == "barh" or graph_type=="horizontal_bar": - r, ndata= self.do_barplot(axis,vars_all, dyns, result_type, data, shift, ibrokenY==0, horizontal=True) + r, ndata= self.do_barplot(axis,vars_all, dyns, result_type, data, shift, ibrokenY==0, horizontal=True, data_types=data_types) barplot = True horizontal = True else: """Barplot. X is all seen variables combination, series are version""" - r, ndata= self.do_barplot(axis,vars_all, dyns, result_type, data, shift, ibrokenY==0) + r, ndata= self.do_barplot(axis,vars_all, dyns, result_type, data, shift, ibrokenY==0, data_types=data_types) barplot = True except Exception as e: print("ERROR : could not graph %s" % result_type) @@ -1570,12 +1574,16 @@ def generate_plot_for_graph(self, i, i_subplot, figure, n_cols, n_lines, vars_va xticks = np.delete(xticks,np.delete(np.array(range(len(xticks))),index)) plt.xticks(xticks) - - d = log(max(ax),base) - log(min(ax),base) + ma = min(ax) + mal = (log(ma,base) if ma > 0 else 0) + d = log(max(ax),base) - mal #Force recomputation of xmin - if not xmin: - xmin = log(min(ax),base) - (d * plt.margins()[0]) - plt.xlim(xmin=pow(base,xmin)) + if not xmin or ma==0: + xmin = mal - (d * plt.margins()[0]) + if (xmin <= 1): + plt.xlim(xmin=xmin) + else: + plt.xlim(xmin=pow(base,xmin)) if not xmax: xmax = log(max(ax),base) + (d * plt.margins()[0]) plt.xlim(xmax=pow(base,xmax)) @@ -1828,6 +1836,40 @@ def autolabel(rects, ax): ha='center', va='bottom') autolabel(rects, plt) + + def mask_from_filter(self, fl, data_types, build, ax, y): + #The result type to filter on + fl_min=1 + fl_op = '>' + flm = re.match("(.*)([><=])(.*)", fl) + if flm: + fl = flm.group(1) + fl_min=float(flm.group(3)) + fl_op=flm.group(2) + if not fl in data_types: + print("ERROR: graph_filter_by's %s not found" % fl) + return + fl_data = data_types[fl] + fl_y = None + for fl_xyeb in fl_data: + if fl_xyeb[3] == build: + fl_y = np.array(fl_xyeb[1]) + break + if fl_op == '>': + mask = fl_y > fl_min + elif fl_op == '<': + mask = fl_y < fl_min + elif fl_op == '=': + mask = fl_y == fl_min + else: + raise Exception("Unknown operator in filter_by : " + fl_op ) + + + if len(mask) != len(ax) or len(mask) != len(y): + print("ERROR: graph_filter_by cannot be applied, because length of X is %d, length of Y is %d but length of mask is %d" % (len(ax), len(y), len(mask))) + return None + return mask + def do_simple_barplot(self,axis, result_type, data,shift=0,isubplot=0): i = 0 interbar = 0.1 @@ -2119,36 +2161,9 @@ def do_line_plot(self, axis, key, result_type, data : XYEB, data_types, shift,id rects = axis.scatter(ax, y, label=lab, color=c, marker=marker, facecolors=fillstyle) else: if result_type in filters: - #The result type to filter on - fl = filters[result_type] - fl_min=1 - fl_op = '>' - flm = re.match("(.*)([><=])(.*)", fl) - if flm: - fl = flm.group(1) - fl_min=float(flm.group(3)) - fl_op=flm.group(2) - if not fl in data_types: - print("ERROR: graph_filter_by's %s not found" % fl) - return - fl_data = data_types[fl] - fl_y = None - for fl_xyeb in fl_data: - if fl_xyeb[3] == build: - fl_y = np.array(fl_xyeb[1]) - break - if fl_op == '>': - mask = fl_y > fl_min - elif fl_op == '<': - mask = fl_y < fl_min - elif fl_op == '=': - mask = fl_y == fl_min - else: - raise Exception("Unknown operator in filter_by : " + fl_op ) - - if len(mask) != len(ax) or len(mask) != len(y): - print("ERROR: graph_filter_by cannot be applied, because length of X is %d, length of Y is %d but length of mask is %d" % (len(ax), len(y), len(mask))) + mask=self.mask_from_filter(filters[result_type],data_types,build,ax,y) + if mask is None: continue rects = axis.plot(ax[~mask], y[~mask], label=lab, color=c, linestyle=build._line, marker=marker,markevery=(1 if len(ax) < 20 else math.ceil(len(ax) / 20)),drawstyle=drawstyle, fillstyle=fillstyle, **line_params) @@ -2277,7 +2292,7 @@ def format_figure(self, axis, result_type, shift, key = None, default_format = N ticks = [variable.get_numeric(npf.parseUnit(y)) for y in yticks.split('+')] plt.yticks(ticks) - def do_barplot(self, axis,vars_all, dyns, result_type, data, shift, show_vals, horizontal=False): + def do_barplot(self, axis,vars_all, dyns, result_type, data, shift, show_vals, horizontal=False, data_types=None): nseries = len(data) self.format_figure(axis,result_type,shift) @@ -2292,6 +2307,9 @@ def do_barplot(self, axis,vars_all, dyns, result_type, data, shift, show_vals, h edgecolor = None interbar = 0.1 + #Filters allow to hatch bars where a value of another variable is higher than something. Stack not implemented yet. + filters = self.configdict("graph_filter_by", default={}) + interbar = self.config('graph_bar_inter', default=interbar) stack = self.config_bool('graph_bar_stack') do_hatch = self.config_bool('graph_bar_hatch', default=False) @@ -2307,6 +2325,27 @@ def do_barplot(self, axis,vars_all, dyns, result_type, data, shift, show_vals, h func=axis.bar ticks = plt.xticks + def common_args(build): + return { + 'label':str(build.pretty_name()), + 'yerr':std if not horizontal else None, + 'xerr':std if horizontal else None, + } + + def hatch_args(build): + return { + 'color':lighter(build._color, 0.6, 255), + 'edgecolor':lighter(build._color, 0.6, 0), + 'hatch':patterns[i] + } + + def nohatch_args(build): + return { + 'color':build._color, + 'edgecolor':edgecolor, + 'hatch':None + } + if stack: last = 0 for i, (x, y, e, build) in enumerate(data): @@ -2317,18 +2356,38 @@ def do_barplot(self, axis,vars_all, dyns, result_type, data, shift, show_vals, h y = np.asarray([0.0 if np.isnan(x) else x for x in y]) std = np.asarray([std for mean,std,raw in e]) rects = func(ind, last, width, - label=str(build.pretty_name()), color=build._color, - yerr=std if not horizontal else None, xerr=std if horizontal else None, - edgecolor=edgecolor) + **common_args(build), + **(hatch_args(build) if do_hatch else nohatch_args(build)) + ) last = last - y else: for i, (x, y, e, build) in enumerate(data): std = np.asarray([std for mean,std,raw in e]) fx = interbar + ind + (i * width) - rects = func(fx, y, width, - label=str(build.pretty_name()), color=lighter(build._color, 0.6, 255) if do_hatch else build._color, - yerr=std if not horizontal else None, xerr=std if horizontal else None, - edgecolor=lighter(build._color, 0.6, 0) if do_hatch else edgecolor, hatch=patterns[i] if do_hatch else None) + if result_type in filters: + mask=self.mask_from_filter(filters[result_type],data_types,build,ax=x,y=y) + + args = common_args(build) + nargs = {'label':args['label']} + if mask is None: + continue + from itertools import compress + rects = func(list(compress(fx, ~mask)), list(compress(y,~mask)), width, + **nargs, + yerr = list(compress(args['yerr'], ~mask)) if args['yerr'] is not None else None, + **hatch_args(build) + ) + + func(list(compress(fx,mask)), list(compress(y,mask)), width, + **nargs, + yerr = list(compress(args['yerr'], mask)) if args['yerr'] is not None else None, + **nohatch_args(build) + ) + else: + rects = func(fx, y, width, + **common_args(build), + **(hatch_args(build) if do_hatch else nohatch_args(build)) + ) if show_vals: self.write_labels(rects, plt, build._color) diff --git a/npf/node.py b/npf/node.py index 56447e33..8e8d2538 100644 --- a/npf/node.py +++ b/npf/node.py @@ -39,7 +39,7 @@ def __init__(self, name, executor : Executor, tags): f = open(clusterFilePath, 'r') for i, line in enumerate(f): line = line.strip() - if not line or line.startswith("#") or line.startswith("//"): + if not line or re.match("^\\s*(#|//)", line): continue match = re.match(r'((?P[a-zA-Z]+[a-zA-Z0-9]*):)?(?P[0-9]+):(?P' + NIC.TYPES + ')=(?P[a-z0-9:_.-]*)', line, re.IGNORECASE) diff --git a/npf/npf.py b/npf/npf.py index 4eb83d46..b192a2b6 100755 --- a/npf/npf.py +++ b/npf/npf.py @@ -1,6 +1,8 @@ import sys import argparse import os + +from pathlib import Path from argparse import ArgumentParser from typing import Dict, List @@ -9,7 +11,6 @@ from decimal import Decimal from npf.node import Node -from .variable import VariableFactory import numpy as np @@ -157,6 +158,8 @@ def add_testing_options(parser: ArgumentParser, regression: bool = False): t.add_argument('--config', metavar='config=value', type=str, nargs='+', action=ExtendAction, help='list of config values to override', default=[]) + t.add_argument('--env', metavar='list of variables', dest='keep_env', type=str, nargs='+', help='list of environment variables to pass in scripts', default=[], action=ExtendAction) + t.add_argument('--test', '--test', '--npf', dest='test_files', metavar='path or test', type=str, nargs='?', default='tests', help='script or script folder. Default is tests') @@ -510,3 +513,13 @@ def all_num(l): if type(x) is not int and type(x) is not Decimal and not isinstance(x, (np.floating, float)): return False return True + + +def ensure_folder_exists(filename): + savedir = Path(os.path.dirname(filename)) + if not savedir.exists(): + os.makedirs(savedir.as_posix()) + + if not os.path.isabs(filename): + filename = os.getcwd() + os.sep + filename + return filename diff --git a/npf/pipeline/__init__.py b/npf/pipeline/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/npf/pipeline/pypost.py b/npf/pipeline/pypost.py new file mode 100644 index 00000000..092f27c4 --- /dev/null +++ b/npf/pipeline/pypost.py @@ -0,0 +1,26 @@ + + + +from typing import List + +from npf.types.series import Series + + +def execute_pypost(series: List[Series]): + # Get all scripts, and execute pypost + for i, (test, build, all_results) in enumerate(series): + + if hasattr(test, 'pypost'): + def common_divide(a,b): + m = min(len(a),len(b)) + return np.array(a)[:m] / np.array(b)[:m] + def results_divide(res,a,b): + for RUN, RESULTS in all_results.items(): + if a in RESULTS and b in RESULTS: + all_results[RUN][res] = common_divide(RESULTS[a], RESULTS[b]) + vs = {'ALL_RESULTS': all_results, 'common_divide': common_divide, 'results_divide': results_divide} + try: + exec(test.pypost.content, vs) + except Exception as e: + print("ERROR WHILE EXECUTING PYPOST SCRIPT:") + print(e) diff --git a/npf/regression.py b/npf/regression.py index c1eb5c5a..83ff8613 100644 --- a/npf/regression.py +++ b/npf/regression.py @@ -144,13 +144,13 @@ def compare(self, build.writeversion(test, all_results, allow_overwrite = True) return tests_passed, tests_total - def regress_all_tests(self, - tests: List['Test'], - options, - history: int = 1, - on_finish = None, - do_compare:bool = True, - iserie=0, nseries=1) -> Tuple[Build, List[Dataset]]: + def regress_all_tests( self, + tests: List['Test'], + options, + history: int = 1, + on_finish = None, + do_compare:bool = True, + i_serie=0, nseries=1) -> Tuple[Build, List[Dataset], List[Dataset]]: """ Execute all tests passed in argument for the last build of the regressor associated repository :param history: Start regression at last build + 1 - history @@ -170,11 +170,14 @@ def regress_all_tests(self, nok = 0 - for itest,test in enumerate(tests): + for i_test, test in enumerate(tests): + print(test) if build.version != "local": - print("[%s] Running test %s on version %s..." % (repo.name, test.filename, build.version)) + print( + f"[{repo.name}] Running test {test.filename} on version {build.version}..." + ) else: - print("[%s] Running test %s..." % (repo.name, test.filename)) + print(f"[{repo.name}] Running test {test.filename}...") regression = self if repo.last_build: try: @@ -199,7 +202,7 @@ def early_results(all_data_results, all_time_results): options=options, do_test=options.do_test, on_finish=early_results, - iserie=iserie*len(tests) + itest, + iserie=i_serie*len(tests) + i_test, nseries=len(tests)*nseries) if all_results is None and time_results is None: @@ -207,10 +210,10 @@ def early_results(all_data_results, all_time_results): except ScriptInitException: return None, None, None - variables_passed, variables_total = regression.compare(test, test.variables, all_results, build, - old_all_results, - repo.last_build, - init_done=init_done, allow_supplementary=options.allow_supplementary) + variables_passed, variables_total = regression.compare( test, test.variables, all_results, build, + old_all_results, + repo.last_build, + init_done=init_done, allow_supplementary=options.allow_supplementary) if variables_passed == variables_total: nok += 1 data_datasets.append(all_results) diff --git a/npf/section.py b/npf/section.py index 9ed23786..5a8d95f2 100644 --- a/npf/section.py +++ b/npf/section.py @@ -625,6 +625,8 @@ def __init__(self): 'series_prop': 'graph_series_prop', 'graph_legend_ncol': 'legend_ncol', 'graph_legend_loc': 'legend_loc', + 'graph_do_legend' : 'graph_legend', + 'do_legend' : 'graph_legend', 'var_label_dir' : 'graph_label_dir', 'graph_max_col' : 'graph_max_col', diff --git a/npf/statistics.py b/npf/statistics.py index 618b6233..0c950850 100644 --- a/npf/statistics.py +++ b/npf/statistics.py @@ -140,7 +140,7 @@ def buildDataset(cls, all_results: Dataset, test: Test) -> List[tuple]: y = OrderedDict() dataset = [] for i, (run, results_types) in enumerate(all_results.items()): - vars = list(run.variables[k] for k in dtype['names']) + vars = list(run.read_variables()[k] for k in dtype['names']) if not results_types is None and len(results_types) > 0: dataset.append([v for v in vars]) diff --git a/npf/test.py b/npf/test.py index 37b0377c..6cbcafb5 100755 --- a/npf/test.py +++ b/npf/test.py @@ -76,7 +76,7 @@ def _parallel_exec(param: RemoteParameters): if wf[0].isdigit(): n=int(wf[0]) wf=wf[1:] - for i in range(n): + for _ in range(n): param.event.listen(wf) param.event.wait_for_termination(param.delay) @@ -167,7 +167,7 @@ def __init__(self, test_path, options, tags=None, role=None, inline=None): self.filename = os.path.basename(test_path) self.path = os.path.dirname(os.path.abspath(test_path)) self.options = options - self.tags = tags if tags else [] + self.tags = tags or [] self.role = role self.pyexits = [] @@ -401,11 +401,20 @@ def test_tags(self): return missings def build_file_list(self, v, self_role=None, files=None) -> List[Tuple[str, str, str]]: + """ + Builds a list of files based on the provided variables and role. + + :param v: A dictionary of variables to be used for file creation + :param self_role: The role associated with the current instance + :param files: A list of files to process + :returns: A list of tuples containing the filename, content, and role for each file + """ create_list = [] if files is None: files = self.files + for s in files: - role = s.get_role() if s.get_role() else self_role + role = s.get_role() or self_role v["NPF_NODE_MAX"] = len(npf.nodes_for_role(role)) if not s.noparse: s.filename = SectionVariable.replace_variables(v, s.filename, role, default_role_map = self.config.get_dict("default_role_map")) @@ -425,12 +434,12 @@ def create_files(self, file_list, path_to_root): unique_list = {} for filename, p, role in file_list: if filename in unique_list: - if unique_list[filename + (role if role else '')][1] != p: + if unique_list[filename + (role or '')][1] != p: raise Exception( "File name conflict ! Some of your scripts try to create some file with the same name but " "different content (%s) !" % filename) else: - unique_list[filename + (role if role else '')] = (filename, p, role) + unique_list[filename + (role or '')] = (filename, p, role) for _, (filename, p, role) in unique_list.items(): if self.options.show_files: @@ -481,7 +490,7 @@ def killall(queue, event): pass i = 0 - while killer.is_alive() and i < 500: + while killer.is_alive() and i < 2000: time.sleep(0.010) i += 1 try: @@ -498,8 +507,8 @@ def update_constants(self, v_internals : dict, build : Build, full_test_folder : tp = os.path.relpath(self.path,abs_test_folder) if node and node.executor.path: - bp = os.path.relpath(bp, npf.experiment_path() + '/testfolder/') - rp = os.path.relpath(rp, npf.experiment_path() + '/testfolder/') + bp = os.path.relpath(bp, f'{npf.experiment_path()}/testfolder/') + rp = os.path.relpath(rp, f'{npf.experiment_path()}/testfolder/') v_internals.update({ 'NPF_REPO':get_valid_filename(build.repo.name), 'NPF_REPO_PATH': rp, @@ -523,9 +532,9 @@ def parse_results(self, regex_list: str, output: str, new_time_results: dict, ne for nr in re.finditer(result_regex, output.strip(), re.IGNORECASE): result_type = nr.group("type") - kind = nr.group("kind") - if kind is None: - kind = "time" + time_ns = nr.group("kind") + if time_ns is None: + time_ns = "time" time_value = nr.group("time_value") if result_type is None: result_type = '' @@ -559,22 +568,22 @@ def parse_results(self, regex_list: str, output: str, new_time_results: dict, ne result_overwrite = self.config.get_bool_or_in("result_overwrite", result_type) if time_value: t = float(time_value) - if result_type in new_time_results.setdefault(kind,{}).setdefault(t, {}): + if result_type in new_time_results.setdefault(time_ns,{}).setdefault(t, {}): #Result is already known if result_add: - new_time_results[kind][t][result_type] += n + new_time_results[time_ns][t][result_type] += n elif result_overwrite: - new_time_results[kind][t][result_type] = n + new_time_results[time_ns][t][result_type] = n else: if not result_append: print(f"WARNING: There are multiple occurences of metric {result_type} for the same time {t}, please add the metric {result_type} in result_add, result_append or result_overwrite. result_appe d is selected by default, add `result_append={{{result_type}}}` to %config to silent this message.") - if type(new_time_results[kind][t][result_type]) is not list: - new_time_results[kind][t][result_type] = [new_time_results[kind][t][result_type]] + if type(new_time_results[time_ns][t][result_type]) is not list: + new_time_results[time_ns][t][result_type] = [new_time_results[time_ns][t][result_type]] - new_time_results[kind][t][result_type].append(n) + new_time_results[time_ns][t][result_type].append(n) else: - new_time_results[kind][t][result_type] = n + new_time_results[time_ns][t][result_type] = n else: if result_append: new_data_results.setdefault(result_type,[]).append(n) @@ -689,7 +698,7 @@ def execute(self, build, run, v, n_runs=1, n_retry=0, allowed_types=SectionScrip v["NPF_ROLE"] = role for script in t.scripts: - srole = role if role else script.get_role() + srole = role or script.get_role() nodes = npf.nodes_for_role(srole) autokill = m.Value('i', 0) if npf.parseBool(script.params.get("autokill", t.config["autokill"])) else None @@ -783,10 +792,13 @@ def execute(self, build, run, v, n_runs=1, n_retry=0, allowed_types=SectionScrip param.name = script.get_name(True) param.autokill = autokill param.env = OrderedDict() + if self.options.keep_env: + param.env.update({key:os.environ[key] for key in self.options.keep_env}) param.env.update(v_internals) param.env.update([(k, v.replace('$NPF_BUILD_PATH', build.repo.get_build_path())) for k, v in build.repo.env.items()]) + if self.options.rand_env: param.env['RANDENV'] = ''.join(random.choice(string.ascii_lowercase) for i in range(random.randint(0,self.options.rand_env))) if 'waitfor' in script.params: @@ -1136,11 +1148,11 @@ def execute_all(self, build, options, prev_results: Dataset = None, prev_time_results: Dict[str, Dataset] = None, iserie=0, nseries=1) -> Tuple[ - Dataset, bool]: + Dataset, Dataset, bool]: """Execute script for all variables combinations for this specific build. All tools rely on this function for execution of the tests. - :param allowed_types:Tyeps of scripts allowed to run. Set with either init, scripts or both + :param allowed_types:Type of scripts allowed to run. Set with either init, scripts or both :param do_test: Actually run the tests :param options: NPF options object :param build: A build object @@ -1167,7 +1179,7 @@ def execute_all(self, build, options, prev_results: Dataset = None, self.do_init_all(build, options, do_test=do_test, allowed_types=allowed_types, v_internals=v_internals) if not self.options.preserve_temp: shutil.rmtree(test_folder) - return {}, True + return {}, {}, True all_data_results = OrderedDict() all_time_results = OrderedDict() @@ -1193,10 +1205,11 @@ def execute_all(self, build, options, prev_results: Dataset = None, for k,v in imp.test.variables.statics().items(): variables[k] = v.makeValues()[0] - run = Run(variables) variables.update(root_variables) - run.variables.update(build.repo.overriden_variables) - variables = run.variables.copy() + run = Run(variables) + + run.write_variables().update(build.repo.overriden_variables) + variables = run.read_variables().copy() if shadow_variables: shadow_variables.update(root_variables) @@ -1248,14 +1261,14 @@ def execute_all(self, build, options, prev_results: Dataset = None, prev_time_results = nprev_time_results if not run_results and options.use_last and build.repo.url: for version in build.repo.method.get_history(build.version, limit=options.use_last): - oldb = Build(build.repo, version) + oldb = Build(build.repo, version, options.result_path) r = oldb.load_results(self) if r and run in r: run_results = r[run] break if not time_results and options.use_last and build.repo.url: for version in build.repo.method.get_history(build.version, limit=options.use_last): - oldb = Build(build.repo, version) + oldb = Build(build.repo, version, options.result_path) r = oldb.load_results(self, kind=True) found = False if r: @@ -1375,7 +1388,7 @@ def print_header(i, i_try): for kind, kresults in new_all_time_results.items(): time_results.setdefault(kind, OrderedDict()) for time, results in sorted(kresults.items()): - time_run = Run(run.variables.copy()) + time_run = Run(run.read_variables().copy()) time_run.variables[kind] = time for result_type, result in results.items(): rt = time_results[kind].setdefault(time_run, {}).setdefault(result_type, []) diff --git a/npf/test_driver.py b/npf/test_driver.py index df4f7e88..86f8c3a2 100644 --- a/npf/test_driver.py +++ b/npf/test_driver.py @@ -1,9 +1,12 @@ -from typing import List +from typing import Dict, List, Tuple from npf import npf +from npf.build import Build from npf.regression import Grapher, OrderedDict, Regression, npf from npf.repository import Repository from npf.statistics import Statistics from npf.test import Test +from npf.types.dataset import Dataset, Run +from npf.types.series import Series """Runs all tests for a given list of tests (or a folder to expand), and a series of repositories. @@ -14,24 +17,27 @@ def __init__(self, repo_list: List[Repository]): self.graphs_series = [] self.time_graphs_series = [] - def build_list(self, on_finish, test, build, data_datasets, time_datasets): + def build_list(self, on_finish, test, build:Build, data_datasets:Dataset, time_datasets): on_finish(self.graphs_series + [(test,build,data_datasets[0])], self.time_graphs_series + [(test,build,time_datasets[0])]) def run(self, test_name, options, tags:List, on_finish=None, do_regress=True): - for irepo,repo in enumerate(self.repo_list): + for i_repo, repo in enumerate(self.repo_list): + build = None regressor = Regression(repo) tests = Test.expand_folder(test_name, options=options, tags=repo.tags + tags) tests = npf.override(options, tests) - for itest,test in enumerate(tests): + for test in tests: build, data_dataset, time_dataset = regressor.regress_all_tests( tests=[test], options=options, do_compare=do_regress, - on_finish=lambda b,dd,td: self.build_list(on_finish,test,b,dd,td) if on_finish else None,iserie=irepo,nseries=len(self.repo_list) + on_finish=lambda b,dd,td: self.build_list(on_finish,test,b,dd,td) if on_finish else None, + i_serie=i_repo, + nseries=len(self.repo_list) ) - if len(tests) > 0 and not build is None: + if len(tests) > 0 and build is not None: build._pretty_name = repo.name self.graphs_series.append((test, build, data_dataset[0])) self.time_graphs_series.append((test, build, time_dataset[0])) @@ -42,18 +48,30 @@ def run(self, test_name, options, tags:List, on_finish=None, do_regress=True): return self.graphs_series, self.time_graphs_series -def group_series(filename, args, series, time_series, options): - +def group_series(filename: str, series: Series , time_series:Series, options) -> Tuple[Dataset,Dict]: + """ + The function merge different series together, finding common variables + + :param filename: The name of the file to which the output will be exported + :param series: The "series" parameter refers to the data series that you want to export. It could be + a list, array, or any other data structure that contains the data you want to export + :param kind_series: The `kind_series` parameter is used to specify the type of series to be + exported. It can take values such as "line", "bar", "scatter", etc., depending on the type of chart + or graph you want to export + :param options: The "options" parameter is a dictionary that contains various options for exporting + the output. These options can include things like the file format, the delimiter to use, whether or + not to include headers, etc + """ if series is None: return #Group repo if asked to do so if options.group_repo: repo_series=OrderedDict() - for i, (test, build, dataset) in enumerate(series): + for test, build, dataset in series: repo_series.setdefault(build.repo.reponame,(test,build,OrderedDict())) for run, run_results in dataset.items(): - run.variables['SERIE'] = build.pretty_name() + run.write_variables()['SERIE'] = build.pretty_name() repo_series[build.repo.reponame][2][run] = run_results series = [] for reponame, (test, build, dataset) in repo_series.items(): @@ -69,7 +87,7 @@ def group_series(filename, args, series, time_series, options): merged_series.setdefault(build.pretty_name(), []).append((test, build, dataset)) series = [] - for sname,slist in merged_series.items(): + for sname, slist in merged_series.items(): if len(slist) == 1: series.append(slist[0]) else: @@ -83,11 +101,15 @@ def group_series(filename, args, series, time_series, options): for test, build, dataset in series: v_list = set() for run, results in dataset.items(): - v_list.update(run.variables.keys()) + v_list.update(run.read_variables().keys()) all_variables.append(v_list) - if args.statistics: - Statistics.run(build,dataset, test, max_depth=args.statistics_maxdepth, filename=args.statistics_filename if args.statistics_filename else npf.build_output_filename(args, [build.repo for t,build,d in series])) + if options.statistics: + Statistics.run(build, + dataset, + test, + max_depth=options.statistics_maxdepth, + filename=options.statistics_filename or npf.build_output_filename(options, [build.repo for t,build,d in series])) common_variables = set.intersection(*map(set, all_variables)) @@ -98,18 +120,16 @@ def group_series(filename, args, series, time_series, options): for variable in common_variables: all_values = set() all_alone=True - for i, (test, build, dataset) in enumerate(series): + for test, build, dataset in series: serie_values = set() for run, result_types in dataset.items(): - if variable in run.variables: - val = run.variables[variable] + if variable in run.read_variables(): + val = run.read_variables()[variable] serie_values.add(val) if len(serie_values) > 1: all_alone = False break - if all_alone: - pass - else: + if not all_alone: useful_variables.append(variable) if options.group_repo: @@ -121,35 +141,45 @@ def group_series(filename, args, series, time_series, options): #Keep only the variables in Run that are usefull as defined above for i, (test, build, dataset) in enumerate(series): - ndataset = OrderedDict() + new_dataset: Dict[Run,List] = OrderedDict() for run, results in dataset.items(): - ndataset[run.intersect(useful_variables)] = results - series[i] = (test, build, ndataset) + m = run.intersect(useful_variables) + if m in new_dataset: + print(f"WARNING: You are comparing series with different variables. Results of series '{build.pretty_name()}' are merged.") + for output, data in results.items(): + if output in new_dataset[m]: + new_dataset[m][output].extend(data) + else: + new_dataset[m][output] = data + else: + new_dataset[m] = results + series[i] = (test, build, new_dataset) #Keep only the variables in Time Run that are usefull as defined above if options.do_time: - n_time_series=OrderedDict() - for i, (test, build, time_dataset) in enumerate(time_series): - for kind, dataset in time_dataset.items(): - ndataset = OrderedDict() - n_time_series.setdefault(kind,[]) - for run, results in dataset.items(): - ndataset[run.intersect(useful_variables + [kind])] = results - if ndataset: - n_time_series[kind].append((test, build, ndataset)) + n_time_series = OrderedDict() + for test, build, time_dataset in time_series: + for kind, dataset in time_dataset.items(): + new_dataset = OrderedDict() + n_time_series.setdefault(kind,[]) + for run, results in dataset.items(): + new_dataset[run.intersect(useful_variables + [kind])] = results + if new_dataset: + n_time_series[kind].append((test, build, new_dataset)) grapher = Grapher() print("Generating graphs...") + g = grapher.graph(series=series, filename=filename, - options=args, - title=args.graph_title) + options=options, + title=options.graph_title) if options.do_time: - for kind,series in n_time_series.items(): - print("Generating graph for time serie '%s'..." % kind) - g = grapher.graph(series=series, - filename=filename, - fileprefix=kind, - options=args, - title=args.graph_title) + for time_ns,series in n_time_series.items(): + print(f"Generating graph for time serie '{time_ns}'...") + g = grapher.graph( series=series, + filename=filename, + fileprefix=time_ns, + options=options, + title=options.graph_title) return series, time_series diff --git a/npf/types/dataset.py b/npf/types/dataset.py index e359c9af..435e1abf 100644 --- a/npf/types/dataset.py +++ b/npf/types/dataset.py @@ -1,7 +1,8 @@ import numpy as np -from typing import Dict, List, Tuple +from typing import Dict, Final, List, Tuple from collections import OrderedDict import sys + if sys.version_info < (3, 7): from orderedset import OrderedSet else: @@ -16,13 +17,27 @@ class Run: def __init__(self, variables): - self.variables = variables + self._variables = variables + self._hash = None + self._data_repr = None + + def read_variables(self) -> Final[Dict]: + return self._variables + + def write_variables(self) -> Dict: + self._hash = None + self._data_repr = None + return self._variables + + @property + def variables(self): + return self.write_variables() def format_variables(self, hide=None): if hide is None: hide = {} s = [] - for k, v in self.variables.items(): + for k, v in self._variables.items(): if k in hide: continue if type(v) is tuple: s.append('%s = %s' % (k, v[1])) @@ -31,21 +46,21 @@ def format_variables(self, hide=None): return ', '.join(s) def print_variable(self, k, default=None): - v = self.variables.get(k,default) + v = self._variables.get(k,default) if type(v) is tuple: return v[1] else: return v def copy(self): - newrun = Run(self.variables.copy()) + newrun = Run(self._variables.copy()) return newrun def inside(self, o): - for k, v in self.variables.items(): - if not k in o.variables: + for k, v in self._variables.items(): + if not k in o._variables: return False - ov = o.variables[k] + ov = o._variables[k] if type(v) is tuple: v = v[1] if type(ov) is tuple: @@ -59,35 +74,35 @@ def inside(self, o): return True def intersect(self, common): - difs = set.difference(set(self.variables.keys()), common) + + difs = set.difference(set(self._variables.keys()), common) for dif in difs: - del self.variables[dif] + self._hash = None + self._data_repr = None + del self._variables[dif] return self - def __eq__(self, o): - if len(self.variables) != len(o.variables): - return False - for k, v in self.variables.items(): - if not k in o.variables: - return False - ov = o.variables[k] - if v == ov: - continue + def data_repr(self): + if self._data_repr is not None: + return self._data_repr + r="" + for k, v in sorted(self._variables.items()): if type(v) is tuple: v = v[1] - if type(ov) is tuple: - ov = ov[1] - if is_numeric(v) and is_numeric(ov): - if not get_numeric(v) == get_numeric(ov): - return False - else: - if not v == ov: - return False - return True + r = r+str(k)+str(v) + self._data_repr = r + return r + + def __eq__(self, o): + if len(self._variables) != len(o._variables): + return False + return self.data_repr() == o.data_repr() def __hash__(self): + if self._hash is not None: + return self._hash n = 0 - for k, v in self.variables.items(): + for k, v in self._variables.items(): if type(v) is tuple: v = v[1] if is_numeric(v): @@ -95,15 +110,17 @@ def __hash__(self): else: n += str(v).__hash__() n += k.__hash__() + self._hash = n return n + def __repr__(self): return "Run(" + self.format_variables() + ")" def __cmp__(self, o): - for k, v in self.variables.items(): - if not k in o.variables: return 1 - ov = o.variables[k] + for k, v in self._variables.items(): + if not k in o._variables: return 1 + ov = o._variables[k] if is_numeric(v) and is_numeric(ov): return get_numeric(v) - get_numeric(ov) @@ -123,21 +140,7 @@ def __lt__(self, o): return self.__cmp__(o) < 0 def __len__(self): - return len(self.variables) - -class ImmutableRun: - def __init__(self, variables): - self._run = Run(numeric_dict(variables)) - self._hash = self._run.__hash__() - - def __hash__(self): - return self._hash - - def __eq__(self, o): - if type(o) is Run: - return self._run.__eq__(o) - else: - return self._run.__eq__(o._run) + return len(self._variables) @@ -229,12 +232,12 @@ def prepare_csvs(all_result_types, datasets, statics, run_list, options, kind=No row = [] for t in options.output_columns: if t == 'x': - for var,val in run.variables.items(): + for var,val in run._variables.items(): if var in statics: continue row.append(val) elif t == 'all_x': - for var,val in run.variables.items(): + for var,val in run._variables.items(): row.append(val) elif t == 'raw': row.extend(result) diff --git a/npf/types/series.py b/npf/types/series.py new file mode 100644 index 00000000..e5fdb413 --- /dev/null +++ b/npf/types/series.py @@ -0,0 +1,3 @@ +from typing import List, Tuple + +Series = List[Tuple['Test','Build','Dataset']] \ No newline at end of file diff --git a/npf_compare.py b/npf_compare.py index 3edee039..f419f369 100755 --- a/npf_compare.py +++ b/npf_compare.py @@ -8,24 +8,32 @@ supported in comparator. """ import argparse +from typing import Dict +from npf.pipeline import pypost from npf import npf from npf.test_driver import Comparator from npf.regression import * -from pathlib import Path from npf.test import Test import multiprocessing +from npf import npf + +from npf.types.series import Series + from npf.test_driver import group_series def main(): + """ + The main function for running the NPF cross-repository comparator. + """ parser = argparse.ArgumentParser(description='NPF cross-repository comparator') npf.add_verbosity_options(parser) - parser.add_argument('repos', metavar='repo', type=str, nargs='+', help='names of the repositories to compares. Use a format such as repo+VAR=VAL:Title to overwrite variables and serie name.') + parser.add_argument('repos', metavar='repo', type=str, nargs='+', help='names of the repositories to compare. Use a format such as repo+VAR=VAL:Title to overwrite variables and serie name.') parser.add_argument('--graph-title', type=str, nargs='?', help='Graph title') b = npf.add_building_options(parser) @@ -33,6 +41,7 @@ def main(): g = npf.add_graph_options(parser) args = parser.parse_args() + # Parse the cluster options npf.parse_nodes(args) # Parsing repo list and getting last_build @@ -43,24 +52,20 @@ def main(): comparator = Comparator(repo_list) + # Create a proper file name for the output filename = npf.build_output_filename(args, repo_list) - savedir = Path(os.path.dirname(filename)) - if not savedir.exists(): - os.makedirs(savedir.as_posix()) - - if not os.path.isabs(filename): - filename = os.getcwd() + os.sep + filename + filename = npf.ensure_folder_exists(filename) series, time_series = comparator.run(test_name=args.test_files, - tags=args.tags, - options=args, - on_finish= + tags=args.tags, + options=args, + on_finish= lambda series,time_series: group_series(filename,args,series,time_series,options=args) if args.iterative else None ) - group_series(filename, args, series, time_series, options=args) + group_series(filename, series, time_series, options=args) if __name__ == "__main__": multiprocessing.set_start_method('forkserver') diff --git a/npf_run.py b/npf_run.py index bfd75e2c..83f897dd 100755 --- a/npf_run.py +++ b/npf_run.py @@ -8,6 +8,7 @@ import sys from npf import npf +from npf.pipeline import pypost from npf.regression import * from npf.statistics import Statistics from npf.test import Test, ScriptInitException @@ -109,7 +110,7 @@ def main(): builds : List[Build] = [] for version in versions: - builds.append(Build(repo, version)) + builds.append(Build(repo, version, args.result_path)) last_rebuilds = [] @@ -259,23 +260,27 @@ def main(): except FileNotFoundError: print("Previous build %s could not be found, we will not graph it !" % g_build.version) - filename = args.graph_filename if args.graph_filename else build.result_path(test.filename, 'pdf') - grapher.graph(series=[(test, build, all_results)] + g_series, + + + filename = args.graph_filename or build.result_path(test.filename, 'pdf') + series_with_history = [(test, build, all_results)] + g_series + pypost.execute_pypost(series=series_with_history) + grapher.graph(series=series_with_history, title=test.get_title(), filename=filename, graph_variables=[Run(x) for x in test.variables], options = args) if time_results: - for ns, results in time_results.items(): + for time_ns, results in time_results.items(): if not results: continue series_with_history = [(test, build, results)] grapher.graph(series=series_with_history, title=test.get_title(), - filename=filename, - options = args, - fileprefix=ns) + filename = filename, + fileprefix = time_ns, + options = args) if last_build and args.graph_num > 0: graph_builds = [last_build] + graph_builds[:-1] last_build = build diff --git a/repo/fastclick-light-debug.repo b/repo/fastclick-light-debug.repo new file mode 100644 index 00000000..97efb6ca --- /dev/null +++ b/repo/fastclick-light-debug.repo @@ -0,0 +1,3 @@ +parent=fastclick-light +name=FastClick - Light + Debug +configure+=--enable-dpdk CFLAGS="-O2 -gdwarf" CXXFLAGS="-std=gnu++11 -O2 -gdwarf" --disable-bound-port-transfer diff --git a/repo/iperf2-git.repo b/repo/iperf2-git.repo new file mode 100644 index 00000000..01721e60 --- /dev/null +++ b/repo/iperf2-git.repo @@ -0,0 +1,8 @@ +parent=iperf2 +url=https://git.code.sf.net/p/iperf2/code +method=git +bin_folder=src/ +bin_name=iperf2 +configure=./configure +make=make +clean=make clean