diff --git a/integration/zlt.npf b/integration/zlt.npf new file mode 100644 index 0000000..4d10843 --- /dev/null +++ b/integration/zlt.npf @@ -0,0 +1,21 @@ +%config +graph_filter_by={THROUGHPUT:DROPPEDPC>10} +var_format={THROUGHPUT:%d} +var_names={RATE:Input rate (Gbps),THROUGHPUT:Throughput (Gbps)} +graph_legend=0 +var_lim={result:0-100} + +%variables +RATE=[10-100#5] +THRESH={50,90} + +%script +if [ $RATE -lt $THRESH ] ; then + d=0 +else + d=$(echo "($RATE - $THRESH) / 2" | bc) +fi + +t=$(echo "$RATE*(100-$d)/100" | bc) +echo "RESULT-DROPPEDPC $d" +echo "RESULT-THROUGHPUT $t" diff --git a/npf/expdesign/__init__.py b/npf/expdesign/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/npf/expdesign/fullexp.py b/npf/expdesign/fullexp.py new file mode 100644 index 0000000..22c7711 --- /dev/null +++ b/npf/expdesign/fullexp.py @@ -0,0 +1,35 @@ +from npf.variable import OrderedDict + + +from collections import OrderedDict + + +class FullVariableExpander: + """Expand all variables building the full + matrix first.""" + + def __init__(self, vlist, overriden): + self.expanded = [OrderedDict()] + for k, v in vlist.items(): + if k in overriden: + continue + newList = [] + l = v.makeValues() + + for nvalue in l: + for ovalue in self.expanded: + z = ovalue.copy() + z.update(nvalue if type(nvalue) is OrderedDict else {k: nvalue}) + newList.append(z) + + self.expanded = newList + self.it = self.expanded.__iter__() + + def __iter__(self): + return self.expanded.__iter__() + + def __next__(self): + return self.it.__next__() + + def __len__(self): + return len(self.expanded) \ No newline at end of file diff --git a/npf/expdesign/optimexp.py b/npf/expdesign/optimexp.py new file mode 100644 index 0000000..e974054 --- /dev/null +++ b/npf/expdesign/optimexp.py @@ -0,0 +1,39 @@ +from npf.variable import OrderedDict + + +from collections import OrderedDict + +from skopt import Optimizer +from skopt.space import Real +from joblib import Parallel, delayed +# example objective taken from skopt +from skopt.benchmarks import branin + +class OptimizeVariableExpander: + """Use scipy optimize function""" + + def __init__(self, vlist, overriden): + dimensions = [] + for k, v in vlist.items(): + if k in overriden: + continue + + l = v.makeValues() + dimensions.append(l) + + self.optimizer = Optimizer( + dimensions=dimensions, + random_state=1, + base_estimator='gp' + ) + + + def __iter__(self): + x = self.optimizer.ask(n_points=1) # x is a list of n_points points + return x + + def tell(x, y): + self.optimizer.tell(x, y) + + def __next__(self): + return self.it.__next__() \ No newline at end of file diff --git a/npf/expdesign/randomexp.py b/npf/expdesign/randomexp.py new file mode 100644 index 0000000..1686009 --- /dev/null +++ b/npf/expdesign/randomexp.py @@ -0,0 +1,11 @@ +from random import shuffle + +from npf.expdesign.fullexp import FullVariableExpander + + +class RandomVariableExpander(FullVariableExpander): + """Same as BruteVariableExpander but shuffle the series to test""" + + def __init__(self, vlist, overriden): + super().__init__(vlist, overriden) + shuffle(self.expanded) diff --git a/npf/expdesign/zltexp.py b/npf/expdesign/zltexp.py new file mode 100644 index 0000000..f8f14c6 --- /dev/null +++ b/npf/expdesign/zltexp.py @@ -0,0 +1,101 @@ +from collections import OrderedDict +from typing import Dict + +import numpy as np +from npf.expdesign.fullexp import FullVariableExpander +from npf.types.dataset import Run +from npf.variable import Variable + + +class ZLTVariableExpander(FullVariableExpander): + + def __init__(self, vlist:Dict[str,Variable], results, overriden, input, output): + + + if not input in vlist: + raise Exception(f"{input} is not in the variables, please define a variable in the %variable section.") + self.results = results + self.input = input + self.input_values = vlist[input].makeValues() + del vlist[input] + self.current = None + self.output = output + self.passed = 0 + super().__init__(vlist, overriden) + + def __iter__(self): + self.it = self.expanded.__iter__() + self.passed = 0 + return self + + def __len__(self): + return len(self.expanded) * len(self.input_values) - self.passed + + def __next__(self): + margin=1.01 + if self.current == None: + self.current = self.it.__next__() + + # get all outputs for all inputs + vals_for_current = {} + acceptable_rates = [] + max_r = max(self.input_values) + for r, vals in self.results.items(): + if Run(self.current).inside(r): + try: + if self.output: + r_out = np.mean(vals[self.output]) + r_in = r.variables[self.input] + vals_for_current[r_in] = r_out + if r_out >= r_in/margin: + acceptable_rates.append(r_in) + else: + max_r = min(max_r, r_out) + except KeyError: + raise Exception(f"{self.output} is not in the results. Sample of last result : {vals}") + + #Step 1 : try the max output + if len(vals_for_current) == 0: + next_val = max_r + elif len(vals_for_current) == 1: + #If we're lucky, the max rate is doable + + if len(acceptable_rates) == 1: + self.current = None + self.passed += len(self.input_values) - 1 + return self.__next__() + + #Step 2 : go for the rate below the max output + maybe_achievable_inputs = list(filter(lambda x : x <= max_r, self.input_values)) + next_val = max(maybe_achievable_inputs) + else: + + maybe_achievable_inputs = list(filter(lambda x : x <= max_r*margin, self.input_values)) + left_to_try = set(maybe_achievable_inputs).difference(vals_for_current.keys()) + + #Step 3...K : try to get an acceptable rate. This step might be skiped if we got an acceptable rate already + if len(acceptable_rates) == 0: + #Try the rate below the min already tried rate - its drop count. For instance if we tried 70 last run but got 67 of throughput, try the rate below 64 + min_input = min(vals_for_current.keys()) + min_output = vals_for_current[min_input] + target = min_output - (min_input - min_output) + next_val = max(filter(lambda x : x < target,left_to_try)) + else: + #Step K... n : we do a binary search between the maximum acceptable rate and the minimal rate observed + max_acceptable = max(acceptable_rates) + #Consider we tried 100->95 (max_r=95), 90->90 (acceptable) we have to try values between 90..95 + left_to_try_over_acceptable = list(filter(lambda x: x > max_acceptable, left_to_try)) + if len(left_to_try_over_acceptable) == 0: + #Found! + self.current = None + self.passed += len(self.input_values) - len(vals_for_current) + return self.__next__() + #Binary search + next_val = left_to_try_over_acceptable[int(len(left_to_try_over_acceptable) / 2)] + + + copy = self.current.copy() + copy.update({self.input : next_val}) + return copy + + \ No newline at end of file diff --git a/npf/npf.py b/npf/npf.py index b192a2b..0d2f7a4 100755 --- a/npf/npf.py +++ b/npf/npf.py @@ -172,9 +172,9 @@ def add_testing_options(parser: ArgumentParser, regression: bool = False): t.add_argument('--no-mp', dest='allow_mp', action='store_false', default=True, help='Run tests in the same thread. If there is multiple script, they will run ' 'one after the other, hence breaking most of the tests.') - t.add_argument('--expand', type=str, default=None, dest="expand") - t.add_argument('--rand-env', type=int, default=65536, dest="rand_env") - t.add_argument('--experimental-design', type=str, default="matrix.csv", help="The path towards the experimental design point selection file") + t.add_argument('--exp-design', type=str, default="full", dest="design", help="Experimental design method") + t.add_argument('--spacefill', type=str, default="matrix.csv", dest="spacefill", help="The path towards the space filling values matrix") + t.add_argument('--rand-env', type=int, default=65536, dest="rand_env", help="Add an environmental variable of a random size to prevent bias") c = parser.add_argument_group('Cluster options') c.add_argument('--cluster', metavar='role=user@address:path [...]', type=str, nargs='*', default=[], diff --git a/npf/section.py b/npf/section.py index 5a8d95f..04e8716 100644 --- a/npf/section.py +++ b/npf/section.py @@ -1,19 +1,11 @@ -import ast - -from typing import List, Set -from collections.abc import Mapping - from npf import npf -from npf.repository import Repository -from .variable import * -from collections import OrderedDict -from random import shuffle +from npf.sections import * +from npf.variable import * from spellwise import Editex import re - known_sections = ['info', 'config', 'variables', 'exit', 'pypost' , 'pyexit', 'late_variables', 'include', 'file', 'require', 'import', 'script', 'init', 'exit'] class SpellChecker: @@ -151,701 +143,3 @@ def build(test, data): return s -class Section: - def __init__(self, name): - self.name = name - self.content = '' - self.noparse = False - - def get_content(self): - return self.content - - def finish(self, test): - pass - - -class SectionNull(Section): - def __init__(self, name='null'): - super().__init__(name) - -class SectionSendFile(Section): - def __init__(self, role, path): - super().__init__('sendfile') - self._role = role - self.path = path - - def finish(self, test): - test.sendfile.setdefault(self._role,[]).append(self.path) - - def set_role(self, role): - self._role = role - -class SectionScript(Section): - TYPE_INIT = "init" - TYPE_SCRIPT = "script" - TYPE_EXIT = "exit" - ALL_TYPES_SET = {TYPE_INIT, TYPE_SCRIPT, TYPE_EXIT} - - num = 0 - - def __init__(self, role=None, params=None, jinja=False): - super().__init__('script') - if params is None: - params = {} - self.params = params - self._role = role - self.type = self.TYPE_SCRIPT - self.index = ++self.num - self.multi = None - self.jinja = jinja - - def get_role(self): - return self._role - - def set_role(self, role): - self._role = role - - def get_name(self, full=False): - if 'name' in self.params: - return self.params['name'] - elif full: - if self.get_role(): - return "%s [%s]" % (self.get_role(), str(self.index)) - else: - return "[%s]" % (str(self.index)) - else: - return str(self.index) - - def get_type(self): - return self.type - - def finish(self, test): - test.scripts.append(self) - - def delay(self): - return float(self.params.get("delay", 0)) - - def get_deps_repos(self, options) -> List[Repository]: - repos = [] - for dep in self.get_deps(): - repos.append(Repository.get_instance(dep, options)) - return repos - - def get_deps(self) -> Set[str]: - deps = set() - if not "deps" in self.params: - return deps - for dep in self.params["deps"].split(","): - deps.add(dep) - return deps - -class SectionPyExit(Section): - num = 0 - - def __init__(self, name = ""): - super().__init__('pyexit') - self.index = ++self.num - self.name = name - - def finish(self, test): - test.pyexits.append(self) - - def get_name(self, full=False): - if self.name != "": - return "pyexit_"+str(self.index) - else: - return "pyexit_"+self.name - - - -class SectionImport(Section): - def __init__(self, role=None, module=None, params=None, is_include=False): - super().__init__('import') - if params is None: - params = {} - self.params = params - self.is_include = is_include - self.multi = None - if is_include: - self.module = module - elif module is not None and module != '': - self.module = 'modules/' + module - else: - if not 'test' in params: - raise Exception("%import section must define a module name or a test=[path] to import") - self.module = params['test'] - del params['test'] - - self._role = role - - def get_role(self): - return self._role - - def finish(self, test): - content = self.get_content().strip() - if content != '': - raise Exception("%%import section does not support any content (got %s)" % content) - test.imports.append(self) - - -class SectionFile(Section): - def __init__(self, filename, role=None, noparse=False, jinja=False): - super().__init__('file') - self.content = '' - self.filename = filename - self._role = role - self.noparse = noparse - self.jinja = jinja - - def get_role(self): - return self._role - - def finish(self, test): - test.files.append(self) - - -class SectionInitFile(SectionFile): - def __init__(self, filename, role=None, noparse=False): - super().__init__(filename, role, noparse) - - def finish(self, test): - test.init_files.append(self) - - -class SectionRequire(Section): - def __init__(self, jinja=False): - super().__init__('require') - self.content = '' - self.jinja = jinja - - def role(self): - # For now, require is only on one node, the default one - return 'default' - - def finish(self, test): - test.requirements.append(self) - - -class BruteVariableExpander: - """Expand all variables building the full - matrix first.""" - - def __init__(self, vlist, overriden): - self.expanded = [OrderedDict()] - for k, v in vlist.items(): - if k in overriden: - continue - newList = [] - l = v.makeValues() - - for nvalue in l: - for ovalue in self.expanded: - z = ovalue.copy() - z.update(nvalue if type(nvalue) is OrderedDict else {k: nvalue}) - newList.append(z) - - self.expanded = newList - self.it = self.expanded.__iter__() - - def __iter__(self): - return self.expanded.__iter__() - - def __next__(self): - return self.it.__next__() - - -class RandomVariableExpander(BruteVariableExpander): - """Same as BruteVariableExpander but shuffle the series to test""" - - def __init__(self, vlist): - super().__init__(vlist) - shuffle(self.expanded) - self.it == self.expanded.__iter__() - - -class SectionVariable(Section): - def __init__(self, name='variables'): - super().__init__(name) - self.content = '' - self.vlist = OrderedDict() - self.aliases = {} - - @staticmethod - def replace_variables(v: dict, content: str, self_role=None,self_node=None, default_role_map={}): - return replace_variables(v, content, self_role, self_node, default_role_map) - - def replace_all(self, value): - """Return a list of all possible replacement in values for each combination of variables""" - values = [] - for v in self: - values.append(SectionVariable.replace_variables(v, value)) - return values - - def expand(self, method=None, overriden=set()): - if method == "shuffle" or method == "rand" or method == "random": - return RandomVariableExpander(self.vlist) - else: - return BruteVariableExpander(self.vlist, overriden) - - def __iter__(self): - return self.expand() - - def __len__(self): - if len(self.vlist) == 0: - return 0 - n = 1 - for k, v in self.vlist.items(): - n *= v.count() - return n - - def dynamics(self): - """List of non-constants variables""" - dyn = OrderedDict() - for k, v in self.vlist.items(): - if v.count() > 1: dyn[k] = v - return dyn - - def is_numeric(self, k): - v = self.vlist.get(k, None) - if v is None: - return True - return v.is_numeric() - - def statics(self) -> OrderedDict: - """List of constants variables""" - dyn = OrderedDict() - for k, v in self.vlist.items(): - if v.count() <= 1: dyn[k] = v - return dyn - - def override_all(self, d): - for k, v in d.items(): - self.override(k, v) - - @staticmethod - def _assign(vlist, assign, var, val): - cov = None - for k,v in vlist.items(): - if isinstance(v,CoVariable): - if var in v.vlist: - cov = v.vlist - break - - if assign == '+=': - #If in covariable, remove that one - if cov: - print("NOTE: %s is overwriting a covariable, the covariable will be ignored" % var) - del cov[var] - if var in vlist: - vlist[var] += val - else: - vlist[var] = val - elif assign == '?=': - #If in covariable or already in vlist, do nothing - if not cov and not var in vlist: - vlist[var] = val - else: - #If in covariable, remove it as we overwrite the value - if cov: - del cov[var] - vlist[var] = val - def override(self, var, val): - found = False - for k,v in self.vlist.items(): - if isinstance(v, CoVariable): - if var in v.vlist: - found = True - break - if k == var: - found = True - break - if not found: - print("WARNING : %s does not override anything" % var) - - if not isinstance(val, Variable): - val = SimpleVariable(var, val) - else: - if val.is_default: - return - - self._assign(self.vlist, val.assign, var, val) - - @staticmethod - def match_tags(text, tags): - if not text or text == ':': - return True - if text.endswith(':'): - text = text[:-1] - var_tags_ors = text.split('|') - valid = False - for var_tags_or in var_tags_ors: - var_tags = var_tags_or.split(',') - has_this_or = True - for t in var_tags: - if (t in tags) or (t.startswith('-') and not t[1:] in tags): - pass - else: - has_this_or = False - break - if has_this_or: - valid = True - break - return valid - - @staticmethod - def parse_variable(line, tags, vsection=None, fail=True): - try: - if not line: - return None, None, False - match = re.match( - r'(?P' + Variable.TAGS_REGEX + r':)?(?P' + Variable.NAME_REGEX + r')(?P=|[+?]=)(?P.*)', - line) - if not match: - raise Exception("Invalid variable '%s'" % line) - if not SectionVariable.match_tags(match.group('tags'), tags): - return None, None, False - - name = match.group('name') - return name, VariableFactory.build(name, match.group('value'), vsection), match.group('assignType') - except: - if fail: - print("Error parsing line %s" % line) - raise - else: - return None, None, False - - def build(self, content:str, test, check_exists:bool=False, fail:bool=True): - sections_stack = [self] - for line in content.split("\n"): - if line.strip() == "{": - c = CoVariable() - sections_stack.append(c) - self.vlist[c.name] = c - elif line.strip() == "}": - sections_stack.pop() - for k in c.vlist.keys(): - if k in self.vlist: - del self.vlist[k] - else: - line = line.lstrip() - sect = sections_stack[-1] - var, val, assign = self.parse_variable(line, test.tags, vsection=sect, fail=fail) - if not var is None and not val is None: - # If check_exists, we verify that we overwrite a variable. This is used by config section to ensure we write known parameters - if check_exists and not var in sect.vlist: - if var.endswith('s') and var[:-1] in sect.vlist: - var = var[:-1] - elif var + 's' in sect.vlist: - var = var + 's' - else: - if var in self.aliases: - var = self.aliases[var] - else: - raise Exception("Unknown variable %s" % var) - self._assign(sect.vlist, assign, var, val) - return OrderedDict(self.vlist.items()) - - def finish(self, test): - self.vlist = self.build(self.content, test) - - def dtype(self): - formats = [] - names = [] - for k, v in self.vlist.items(): - k, f = v.format() - if type(f) is list: - formats.extend(f) - names.extend(k) - else: - formats.append(f) - names.append(k) - return OrderedDict(names=names, formats=formats) - - -class SectionLateVariable(SectionVariable): - def __init__(self, name='late_variables'): - super().__init__(name) - - def finish(self, test): - test.late_variables.append(self) - - def execute(self, variables, test, fail=True): - self.vlist = OrderedDict() - for k, v in variables.items(): - self.vlist[k] = SimpleVariable(k, v) - content = self.content - - vlist = self.build(content, test, fail=fail) - final = OrderedDict() - for k, v in vlist.items(): - vals = v.makeValues() - if len(vals) > 0: - final[k] = vals[0] - - return final - - -class SectionConfig(SectionVariable): - def __add(self, var, val): - v = SimpleVariable(var, val) - v.is_default = True - self.vlist[var.lower()] = v - - def __add_list(self, var, list): - v = ListVariable(var, list) - v.is_default = True - self.vlist[var.lower()] = v - - def __add_dict(self, var, dict): - v = DictVariable(var, dict) - v.is_default = True - self.vlist[var.lower()] = v - - def __init__(self): - super().__init__('config') - self.content = '' - self.vlist = {} - - self.aliases = { - 'graph_variable_as_series': 'graph_variables_as_series', - 'graph_variable_as_serie': 'graph_variables_as_series', - 'graph_variables_as_serie': 'graph_variables_as_series', - 'graph_subplot_variables' : 'graph_subplot_variable', - 'graph_grid': 'var_grid', - 'graph_ticks' : 'var_ticks', - 'graph_serie': 'var_serie', - 'graph_types':'graph_type', - 'graph_linestyle': 'graph_lines', - 'var_combine': 'graph_combine_variables', - 'series_as_variables': 'graph_series_as_variables', - 'var_as_series': 'graph_variables_as_series', - 'result_as_variables': 'graph_result_as_variables', - 'y_group': 'graph_y_group', - 'graph_serie_sort':'graph_series_sort', - 'series_prop': 'graph_series_prop', - 'graph_legend_ncol': 'legend_ncol', - 'graph_legend_loc': 'legend_loc', - 'graph_do_legend' : 'graph_legend', - 'do_legend' : 'graph_legend', - 'var_label_dir' : 'graph_label_dir', - 'graph_max_col' : 'graph_max_col', - - } - - # Environment - self.__add("default_repo", "local") - - # Regression related - self.__add_list("accept_zero", ["time","DROP", "DROPPED"]) - self.__add("n_supplementary_runs", 3) - self.__add("acceptable", 0.01) - self.__add("accept_outliers_mult", 1) - self.__add("accept_variance", 1) - - # Test related - - self.__add_list("time_kinds", []) - self.__add("n_runs", 3) - self.__add("n_retry", 0) - self.__add_dict("var_n_runs", {}) - self.__add_dict("var_markers", {}) #Do not set CDF here, small CDF may want them, and then scatterplot would not work - self.__add("result_add", False) - self.__add("result_append", False) - self.__add("result_overwrite", False) - self.__add_list("result_regex", [ - r"(:?(:?(?P[A-Z0-9_]+)-)?(?P[0-9.]+)-)?RESULT(:?-(?P[A-Z0-9_:~.@()-]+))?[ \t]+(?P[0-9.]+(e[+-][0-9]+)?)[ ]*(?P[nµugmkKGT]?)(?Ps|sec|b|byte|bits)?"]) - self.__add_list("results_expect", []) - self.__add("autokill", True) - self.__add("critical", False) - self.__add_dict("env", {}) # Unimplemented yet - self.__add("timeout", 30) - self.__add("hardkill", 5000) - - # Role related - self.__add_dict("default_role_map", {}) - self.__add_list("role_exclude", []) - - # Graph options - self.__add_dict("graph_combine_variables", {}) - self.__add_dict("graph_subplot_results", {}) - self.__add("graph_subplot_variable", None) - self.__add("graph_subplot_unique_legend", False) - self.__add_list("graph_display_statics", []) - self.__add_list("graph_variables_as_series", []) - self.__add("graph_variables_explicit", False) - self.__add_list("graph_hide_variables", []) - self.__add_dict('graph_result_as_variable', {}) - self.__add_dict('graph_map', {}) - self.__add_dict('graph_x_sort', {}) - self.__add("graph_scatter", False) - self.__add("graph_show_values", False) - self.__add("graph_show_ylabel", True) - self.__add("graph_show_xlabel", True) - self.__add("graph_subplot_type", "subplot") - self.__add("graph_max_series", None) - self.__add("graph_max_cols", 2) - self.__add("graph_series_as_variables", False) - self.__add("graph_series_prop", False) - self.__add("graph_series_sort", None) - self.__add("graph_series_label", None) - self.__add_dict("graph_filter_by", {}) - self.__add("graph_bar_stack", False) - self.__add("graph_text",'') - self.__add("graph_legend", None), #the default behavior depends upon the type of graph - self.__add("graph_error_fill",False) - self.__add_dict("graph_error", {"CDF":"none"}) - self.__add("graph_mode",None) - self.__add_dict("graph_y_group",{}) - self.__add_list("graph_color", []) - self.__add_list("graph_markers", ['o', '^', 's', 'D', '*', 'x', '.', '_', 'H', '>', '<', 'v', 'd']) - self.__add_list("graph_lines", ['-', '--', '-.', ':']) - self.__add_list("legend_bbox", [0, 1, 1, .05]) - self.__add("legend_loc", "best") - - self.__add("legend_frameon", True) - self.__add("legend_ncol", 1) - self.__add("var_hide", {}) - self.__add_list("var_log", []) - self.__add_dict("var_log_base", {}) - self.__add_dict("var_divider", {'result': 1}) - self.__add_dict("var_lim", {}) - self.__add_dict("var_format", {}) - self.__add_dict("var_ticks", {}) - - self.__add_dict("graph_legend_params", {}) - self.__add_list("var_grid", ["result"]) - self.__add("graph_grid_linestyle", ":") - - self.__add("graph_fillstyle", "full") - self.__add_dict("graph_tick_params", {}) - self.__add("var_serie",None) - self.__add_dict("var_names", {"result-LATENCY":"Latency (µs)", "result-THROUGHPUT":"Throughput", "^THROUGHPUT$":"Throughput", "boxplot":"", "^PARALLEL$":"Number of parallel connections", "^ZEROCOPY$":"Zero-Copy", "CDFLATPC":"CDF", "CDFLATVAL":"Latency"}) - self.__add_dict("var_unit", {"result": "","result-LATENCY":"us","latency":"us","throughput":"bps"}) - self.__add("graph_show_fliers", True) - self.__add_dict("graph_cross_reference", {}) - self.__add_dict("graph_background", {}) - self.__add_dict("var_round", {}) - self.__add_dict("var_aggregate", {}) - self.__add_dict("var_drawstyle", {}) - self.__add_list("graph_type", []) - self.__add("title", None) - self.__add_list("require_tags", []) - self.__add_dict("graph_label_dir", {}) - self.__add("graph_force_diagonal_labels", False) - self.__add("graph_smooth", 1) - - # Time series - self.__add("time_precision", 1) - self.__add("time_sync", False) - self.__add_list("glob_sync", []) - self.__add_list("var_sync", ["time"]) - self.__add_dict("var_shift", {}) - self.__add_dict("var_repeat", {}) - - def var_name(self, key): - key = key.lower() - if key in self["var_names"]: - return self["var_names"][key] - else: - return key - - def get_list(self, key): - key = key.lower() - var = self.vlist[key] - v = var.makeValues() - return v - - def get_dict(self, key): - key = key.lower() - var = self.vlist[key] - try: - v = OrderedDict() - for k,l in var.vdict.items(): - v[k.strip()] = l - except AttributeError: - print("WARNING : Error in configuration of %s" % key) - return {key: var.makeValues()[0]} - return v - - def get_dict_value(self, var, key, result_type=None, default=None): - best_l = -1 - best = default - if var in self: - if not isinstance(self.vlist[var], DictVariable) and self.vlist[var]: - return self[var] - - d = self.get_dict(var) - if result_type is not None: - #Search for "key-result_type", such as result-throughput - kr = key + "-" + result_type - for k, v in d.items(): - m = re.search(k,kr,re.IGNORECASE) - if m: - l = len(m.group(0)) - if (best_l < l): - best_l = l - best = v - - #Search for result type alone such as throughput - for k, v in d.items(): - m = re.search(k, result_type,re.IGNORECASE) - if m: - l = len(m.group(0)) - if (best_l < l): - best_l = l - best = v - - if var in d: - return d[var] - - #Search for the exact key if there is no result_type - for k, v in d.items(): - m = re.search(k, key, re.IGNORECASE) - if m: - l = len(m.group(0)) - if (best_l < l): - best_l = l - best = v - - return best - - def get_bool(self, key): - return get_bool(self[key]) - - def get_bool_or_in(self, var, obj, default=None): - val = self[var] - - if type(val) == type(obj) and val == obj: - return True - - if isinstance(val, list): - return obj in val - if is_bool(val): - return get_bool(val) - return default - - def __contains__(self, key): - return key.lower() in self.vlist - - def __getitem__(self, key): - var = self.vlist[key.lower()] - v = var.makeValues() - if type(v) is list and len(v) == 1: - return v[0] - else: - return v - - def __setitem__(self, key, val): - self.__add(key.lower(), val) - - def match(self, key, val): - try: - for match in self.get_list(key): - if re.match(match, val): - return True - except Exception: - print("ERROR : Regex %s does not work" % key) - return False - - def finish(self, test): - self.vlist = self.build(self.content, test, check_exists=True) diff --git a/npf/sections/__init__.py b/npf/sections/__init__.py new file mode 100644 index 0000000..65d230e --- /dev/null +++ b/npf/sections/__init__.py @@ -0,0 +1,674 @@ +from collections import OrderedDict +import re +from typing import List, Set +from npf.expdesign.fullexp import FullVariableExpander +from npf.expdesign.randomexp import RandomVariableExpander +from npf.expdesign.zltexp import ZLTVariableExpander +from npf.repository import Repository +from npf.variable import CoVariable, DictVariable, ListVariable, SimpleVariable, Variable, VariableFactory, get_bool, is_bool, replace_variables + + +class Section: + def __init__(self, name): + self.name = name + self.content = '' + self.noparse = False + + def get_content(self): + return self.content + + def finish(self, test): + pass + + +class SectionNull(Section): + def __init__(self, name='null'): + super().__init__(name) + +class SectionSendFile(Section): + def __init__(self, role, path): + super().__init__('sendfile') + self._role = role + self.path = path + + def finish(self, test): + test.sendfile.setdefault(self._role,[]).append(self.path) + + def set_role(self, role): + self._role = role + +class SectionScript(Section): + TYPE_INIT = "init" + TYPE_SCRIPT = "script" + TYPE_EXIT = "exit" + ALL_TYPES_SET = {TYPE_INIT, TYPE_SCRIPT, TYPE_EXIT} + + num = 0 + + def __init__(self, role=None, params=None, jinja=False): + super().__init__('script') + if params is None: + params = {} + self.params = params + self._role = role + self.type = self.TYPE_SCRIPT + self.index = ++self.num + self.multi = None + self.jinja = jinja + + def get_role(self): + return self._role + + def set_role(self, role): + self._role = role + + def get_name(self, full=False): + if 'name' in self.params: + return self.params['name'] + elif full: + if self.get_role(): + return "%s [%s]" % (self.get_role(), str(self.index)) + else: + return "[%s]" % (str(self.index)) + else: + return str(self.index) + + def get_type(self): + return self.type + + def finish(self, test): + test.scripts.append(self) + + def delay(self): + return float(self.params.get("delay", 0)) + + def get_deps_repos(self, options) -> List[Repository]: + repos = [] + for dep in self.get_deps(): + repos.append(Repository.get_instance(dep, options)) + return repos + + def get_deps(self) -> Set[str]: + deps = set() + if not "deps" in self.params: + return deps + for dep in self.params["deps"].split(","): + deps.add(dep) + return deps + +class SectionPyExit(Section): + num = 0 + + def __init__(self, name = ""): + super().__init__('pyexit') + self.index = ++self.num + self.name = name + + def finish(self, test): + test.pyexits.append(self) + + def get_name(self, full=False): + if self.name != "": + return "pyexit_"+str(self.index) + else: + return "pyexit_"+self.name + + + +class SectionImport(Section): + def __init__(self, role=None, module=None, params=None, is_include=False): + super().__init__('import') + if params is None: + params = {} + self.params = params + self.is_include = is_include + self.multi = None + if is_include: + self.module = module + elif module is not None and module != '': + self.module = 'modules/' + module + else: + if not 'test' in params: + raise Exception("%import section must define a module name or a test=[path] to import") + self.module = params['test'] + del params['test'] + + self._role = role + + def get_role(self): + return self._role + + def finish(self, test): + content = self.get_content().strip() + if content != '': + raise Exception("%%import section does not support any content (got %s)" % content) + test.imports.append(self) + + +class SectionFile(Section): + def __init__(self, filename, role=None, noparse=False, jinja=False): + super().__init__('file') + self.content = '' + self.filename = filename + self._role = role + self.noparse = noparse + self.jinja = jinja + + def get_role(self): + return self._role + + def finish(self, test): + test.files.append(self) + + +class SectionInitFile(SectionFile): + def __init__(self, filename, role=None, noparse=False): + super().__init__(filename, role, noparse) + + def finish(self, test): + test.init_files.append(self) + + +class SectionRequire(Section): + def __init__(self, jinja=False): + super().__init__('require') + self.content = '' + self.jinja = jinja + + def role(self): + # For now, require is only on one node, the default one + return 'default' + + def finish(self, test): + test.requirements.append(self) + +class SectionVariable(Section): + def __init__(self, name='variables'): + super().__init__(name) + self.content = '' + self.vlist = OrderedDict() + self.aliases = {} + + @staticmethod + def replace_variables(v: dict, content: str, self_role=None,self_node=None, default_role_map={}): + return replace_variables(v, content, self_role, self_node, default_role_map) + + def replace_all(self, value): + """Return a list of all possible replacement in values for each combination of variables""" + values = [] + for v in self: + values.append(SectionVariable.replace_variables(v, value)) + return values + + def expand(self, results, method=None, overriden=set()): + if method == "shuffle" or method == "rand" or method == "random": + return RandomVariableExpander(self.vlist) + elif method.lower().startswith("zlt"): + params = method[4:-1].split(",") + return ZLTVariableExpander(self.vlist, overriden=overriden, results=results, input=params[0], output=params[1]) + else: + return FullVariableExpander(self.vlist, overriden) + + def __iter__(self): + return self.expand() + + def __len__(self): + if len(self.vlist) == 0: + return 0 + n = 1 + for k, v in self.vlist.items(): + n *= v.count() + return n + + def dynamics(self): + """List of non-constants variables""" + dyn = OrderedDict() + for k, v in self.vlist.items(): + if v.count() > 1: dyn[k] = v + return dyn + + def is_numeric(self, k): + v = self.vlist.get(k, None) + if v is None: + return True + return v.is_numeric() + + def statics(self) -> OrderedDict: + """List of constants variables""" + dyn = OrderedDict() + for k, v in self.vlist.items(): + if v.count() <= 1: dyn[k] = v + return dyn + + def override_all(self, d): + for k, v in d.items(): + self.override(k, v) + + @staticmethod + def _assign(vlist, assign, var, val): + cov = None + for k,v in vlist.items(): + if isinstance(v,CoVariable): + if var in v.vlist: + cov = v.vlist + break + + if assign == '+=': + #If in covariable, remove that one + if cov: + print("NOTE: %s is overwriting a covariable, the covariable will be ignored" % var) + del cov[var] + if var in vlist: + vlist[var] += val + else: + vlist[var] = val + elif assign == '?=': + #If in covariable or already in vlist, do nothing + if not cov and not var in vlist: + vlist[var] = val + else: + #If in covariable, remove it as we overwrite the value + if cov: + del cov[var] + vlist[var] = val + def override(self, var, val): + found = False + for k,v in self.vlist.items(): + if isinstance(v, CoVariable): + if var in v.vlist: + found = True + break + if k == var: + found = True + break + if not found: + print("WARNING : %s does not override anything" % var) + + if not isinstance(val, Variable): + val = SimpleVariable(var, val) + else: + if val.is_default: + return + + self._assign(self.vlist, val.assign, var, val) + + @staticmethod + def match_tags(text, tags): + if not text or text == ':': + return True + if text.endswith(':'): + text = text[:-1] + var_tags_ors = text.split('|') + valid = False + for var_tags_or in var_tags_ors: + var_tags = var_tags_or.split(',') + has_this_or = True + for t in var_tags: + if (t in tags) or (t.startswith('-') and not t[1:] in tags): + pass + else: + has_this_or = False + break + if has_this_or: + valid = True + break + return valid + + @staticmethod + def parse_variable(line, tags, vsection=None, fail=True): + try: + if not line: + return None, None, False + match = re.match( + r'(?P' + Variable.TAGS_REGEX + r':)?(?P' + Variable.NAME_REGEX + r')(?P=|[+?]=)(?P.*)', + line) + if not match: + raise Exception("Invalid variable '%s'" % line) + if not SectionVariable.match_tags(match.group('tags'), tags): + return None, None, False + + name = match.group('name') + return name, VariableFactory.build(name, match.group('value'), vsection), match.group('assignType') + except: + if fail: + print("Error parsing line %s" % line) + raise + else: + return None, None, False + + def build(self, content:str, test, check_exists:bool=False, fail:bool=True): + sections_stack = [self] + for line in content.split("\n"): + if line.strip() == "{": + c = CoVariable() + sections_stack.append(c) + self.vlist[c.name] = c + elif line.strip() == "}": + sections_stack.pop() + for k in c.vlist.keys(): + if k in self.vlist: + del self.vlist[k] + else: + line = line.lstrip() + sect = sections_stack[-1] + var, val, assign = self.parse_variable(line, test.tags, vsection=sect, fail=fail) + if not var is None and not val is None: + # If check_exists, we verify that we overwrite a variable. This is used by config section to ensure we write known parameters + if check_exists and not var in sect.vlist: + if var.endswith('s') and var[:-1] in sect.vlist: + var = var[:-1] + elif var + 's' in sect.vlist: + var = var + 's' + else: + if var in self.aliases: + var = self.aliases[var] + else: + raise Exception("Unknown variable %s" % var) + self._assign(sect.vlist, assign, var, val) + return OrderedDict(self.vlist.items()) + + def finish(self, test): + self.vlist = self.build(self.content, test) + + def dtype(self): + formats = [] + names = [] + for k, v in self.vlist.items(): + k, f = v.format() + if type(f) is list: + formats.extend(f) + names.extend(k) + else: + formats.append(f) + names.append(k) + return OrderedDict(names=names, formats=formats) + + +class SectionLateVariable(SectionVariable): + def __init__(self, name='late_variables'): + super().__init__(name) + + def finish(self, test): + test.late_variables.append(self) + + def execute(self, variables, test, fail=True): + self.vlist = OrderedDict() + for k, v in variables.items(): + self.vlist[k] = SimpleVariable(k, v) + content = self.content + + vlist = self.build(content, test, fail=fail) + final = OrderedDict() + for k, v in vlist.items(): + vals = v.makeValues() + if len(vals) > 0: + final[k] = vals[0] + + return final + + +class SectionConfig(SectionVariable): + def __add(self, var, val): + v = SimpleVariable(var, val) + v.is_default = True + self.vlist[var.lower()] = v + + def __add_list(self, var, list): + v = ListVariable(var, list) + v.is_default = True + self.vlist[var.lower()] = v + + def __add_dict(self, var, dict): + v = DictVariable(var, dict) + v.is_default = True + self.vlist[var.lower()] = v + + def __init__(self): + super().__init__('config') + self.content = '' + self.vlist = {} + + self.aliases = { + 'graph_variable_as_series': 'graph_variables_as_series', + 'graph_variable_as_serie': 'graph_variables_as_series', + 'graph_variables_as_serie': 'graph_variables_as_series', + 'graph_subplot_variables' : 'graph_subplot_variable', + 'graph_grid': 'var_grid', + 'graph_ticks' : 'var_ticks', + 'graph_serie': 'var_serie', + 'graph_types':'graph_type', + 'graph_linestyle': 'graph_lines', + 'var_combine': 'graph_combine_variables', + 'series_as_variables': 'graph_series_as_variables', + 'var_as_series': 'graph_variables_as_series', + 'result_as_variables': 'graph_result_as_variables', + 'y_group': 'graph_y_group', + 'graph_serie_sort':'graph_series_sort', + 'series_prop': 'graph_series_prop', + 'graph_legend_ncol': 'legend_ncol', + 'graph_legend_loc': 'legend_loc', + 'graph_do_legend' : 'graph_legend', + 'do_legend' : 'graph_legend', + 'var_label_dir' : 'graph_label_dir', + 'graph_max_col' : 'graph_max_col', + + } + + # Environment + self.__add("default_repo", "local") + + # Regression related + self.__add_list("accept_zero", ["time","DROP", "DROPPED"]) + self.__add("n_supplementary_runs", 3) + self.__add("acceptable", 0.01) + self.__add("accept_outliers_mult", 1) + self.__add("accept_variance", 1) + + # Test related + + self.__add_list("time_kinds", []) + self.__add("n_runs", 3) + self.__add("n_retry", 0) + self.__add_dict("var_n_runs", {}) + self.__add_dict("var_markers", {}) #Do not set CDF here, small CDF may want them, and then scatterplot would not work + self.__add("result_add", False) + self.__add("result_append", False) + self.__add("result_overwrite", False) + self.__add_list("result_regex", [ + r"(:?(:?(?P[A-Z0-9_]+)-)?(?P[0-9.]+)-)?RESULT(:?-(?P[A-Z0-9_:~.@()-]+))?[ \t]+(?P[0-9.]+(e[+-][0-9]+)?)[ ]*(?P[nµugmkKGT]?)(?Ps|sec|b|byte|bits)?"]) + self.__add_list("results_expect", []) + self.__add("autokill", True) + self.__add("critical", False) + self.__add_dict("env", {}) # Unimplemented yet + self.__add("timeout", 30) + self.__add("hardkill", 5000) + + # Role related + self.__add_dict("default_role_map", {}) + self.__add_list("role_exclude", []) + + # Graph options + self.__add_dict("graph_combine_variables", {}) + self.__add_dict("graph_subplot_results", {}) + self.__add("graph_subplot_variable", None) + self.__add("graph_subplot_unique_legend", False) + self.__add_list("graph_display_statics", []) + self.__add_list("graph_variables_as_series", []) + self.__add("graph_variables_explicit", False) + self.__add_list("graph_hide_variables", []) + self.__add_dict('graph_result_as_variable', {}) + self.__add_dict('graph_map', {}) + self.__add_dict('graph_x_sort', {}) + self.__add("graph_scatter", False) + self.__add("graph_show_values", False) + self.__add("graph_show_ylabel", True) + self.__add("graph_show_xlabel", True) + self.__add("graph_subplot_type", "subplot") + self.__add("graph_max_series", None) + self.__add("graph_max_cols", 2) + self.__add("graph_series_as_variables", False) + self.__add("graph_series_prop", False) + self.__add("graph_series_sort", None) + self.__add("graph_series_label", None) + self.__add_dict("graph_filter_by", {}) + self.__add("graph_bar_stack", False) + self.__add("graph_text",'') + self.__add("graph_legend", None), #the default behavior depends upon the type of graph + self.__add("graph_error_fill",False) + self.__add_dict("graph_error", {"CDF":"none"}) + self.__add("graph_mode",None) + self.__add_dict("graph_y_group",{}) + self.__add_list("graph_color", []) + self.__add_list("graph_markers", ['o', '^', 's', 'D', '*', 'x', '.', '_', 'H', '>', '<', 'v', 'd']) + self.__add_list("graph_lines", ['-', '--', '-.', ':']) + self.__add_list("legend_bbox", [0, 1, 1, .05]) + self.__add("legend_loc", "best") + + self.__add("legend_frameon", True) + self.__add("legend_ncol", 1) + self.__add("var_hide", {}) + self.__add_list("var_log", []) + self.__add_dict("var_log_base", {}) + self.__add_dict("var_divider", {'result': 1}) + self.__add_dict("var_lim", {}) + self.__add_dict("var_format", {}) + self.__add_dict("var_ticks", {}) + + self.__add_dict("graph_legend_params", {}) + self.__add_list("var_grid", ["result"]) + self.__add("graph_grid_linestyle", ":") + + self.__add("graph_fillstyle", "full") + self.__add_dict("graph_tick_params", {}) + self.__add("var_serie",None) + self.__add_dict("var_names", {"result-LATENCY":"Latency (µs)", "result-THROUGHPUT":"Throughput", "^THROUGHPUT$":"Throughput", "boxplot":"", "^PARALLEL$":"Number of parallel connections", "^ZEROCOPY$":"Zero-Copy", "CDFLATPC":"CDF", "CDFLATVAL":"Latency"}) + self.__add_dict("var_unit", {"result": "","result-LATENCY":"us","latency":"us","throughput":"bps"}) + self.__add("graph_show_fliers", True) + self.__add_dict("graph_cross_reference", {}) + self.__add_dict("graph_background", {}) + self.__add_dict("var_round", {}) + self.__add_dict("var_aggregate", {}) + self.__add_dict("var_drawstyle", {}) + self.__add_list("graph_type", []) + self.__add("title", None) + self.__add_list("require_tags", []) + self.__add_dict("graph_label_dir", {}) + self.__add("graph_force_diagonal_labels", False) + self.__add("graph_smooth", 1) + + # Time series + self.__add("time_precision", 1) + self.__add("time_sync", False) + self.__add_list("glob_sync", []) + self.__add_list("var_sync", ["time"]) + self.__add_dict("var_shift", {}) + self.__add_dict("var_repeat", {}) + + def var_name(self, key): + key = key.lower() + if key in self["var_names"]: + return self["var_names"][key] + else: + return key + + def get_list(self, key): + key = key.lower() + var = self.vlist[key] + v = var.makeValues() + return v + + def get_dict(self, key): + key = key.lower() + var = self.vlist[key] + try: + v = OrderedDict() + for k,l in var.vdict.items(): + v[k.strip()] = l + except AttributeError: + print("WARNING : Error in configuration of %s" % key) + return {key: var.makeValues()[0]} + return v + + def get_dict_value(self, var, key, result_type=None, default=None): + best_l = -1 + best = default + if var in self: + if not isinstance(self.vlist[var], DictVariable) and self.vlist[var]: + return self[var] + + d = self.get_dict(var) + if result_type is not None: + #Search for "key-result_type", such as result-throughput + kr = key + "-" + result_type + for k, v in d.items(): + m = re.search(k,kr,re.IGNORECASE) + if m: + l = len(m.group(0)) + if (best_l < l): + best_l = l + best = v + + #Search for result type alone such as throughput + for k, v in d.items(): + m = re.search(k, result_type,re.IGNORECASE) + if m: + l = len(m.group(0)) + if (best_l < l): + best_l = l + best = v + + if var in d: + return d[var] + + #Search for the exact key if there is no result_type + for k, v in d.items(): + m = re.search(k, key, re.IGNORECASE) + if m: + l = len(m.group(0)) + if (best_l < l): + best_l = l + best = v + + return best + + def get_bool(self, key): + return get_bool(self[key]) + + def get_bool_or_in(self, var, obj, default=None): + val = self[var] + + if type(val) == type(obj) and val == obj: + return True + + if isinstance(val, list): + return obj in val + if is_bool(val): + return get_bool(val) + return default + + def __contains__(self, key): + return key.lower() in self.vlist + + def __getitem__(self, key): + var = self.vlist[key.lower()] + v = var.makeValues() + if type(v) is list and len(v) == 1: + return v[0] + else: + return v + + def __setitem__(self, key, val): + self.__add(key.lower(), val) + + def match(self, key, val): + try: + for match in self.get_list(key): + if re.match(match, val): + return True + except Exception: + print("ERROR : Regex %s does not work" % key) + return False + + def finish(self, test): + self.vlist = self.build(self.content, test, check_exists=True) + diff --git a/npf/test.py b/npf/test.py index 6cbcafb..1f444ab 100755 --- a/npf/test.py +++ b/npf/test.py @@ -17,6 +17,7 @@ from npf.node import NIC from npf.section import * from npf.npf import get_valid_filename +from npf.sections import Section, SectionNull from npf.types.dataset import Run, Dataset from npf.eventbus import EventBus from .variable import get_bool @@ -1192,7 +1193,7 @@ def execute_all(self, build, options, prev_results: Dataset = None, for runs_this_pass in total_runs: # Number of results to ensure for this run n = 0 overriden = set(build.repo.overriden_variables.keys()) - all_variables = list(self.variables.expand(method=options.expand, overriden=overriden)) + all_variables = self.variables.expand(method=options.design, overriden=overriden, results=all_data_results) n_tests = len(all_variables) for root_variables in all_variables: n += 1 @@ -1344,7 +1345,7 @@ def print_header(i, i_try): if desc: print(desc, end=' ') print( - ("[%srun %d/%d for test %d/%d"+(" of serie %d/%d" %(iserie+1,nseries) if nseries > 1 else "")+"]") % ( ("retrying %d/%d " % (i_try + 1,n_try)) if i_try > 0 else "", i+1, n_runs, n, n_tests)) + ("[%srun %d/%d for test %d/%d"+(" of serie %d/%d" %(iserie+1,nseries) if nseries > 1 else "")+"]") % ( ("retrying %d/%d " % (i_try + 1,n_try)) if i_try > 0 else "", i+1, n_runs, n, len(all_variables))) new_data_results, new_all_time_results, output, err, n_exec, n_err = self.execute(build, run, variables, n_runs, diff --git a/npf/variable.py b/npf/variable.py index 57898c1..6899abc 100644 --- a/npf/variable.py +++ b/npf/variable.py @@ -267,7 +267,7 @@ def getVals(cls, v:Variable): @classmethod def load(cls): - path = npf.find_local(sys.modules["npf.npf"].options.experimental_design) + path = npf.find_local(sys.modules["npf.npf"].options.spacefill) assert path is not None with open(path) as fd: