From 39acbd318a942219f8876fe17cd2efaa7faa66f3 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 20 Aug 2024 14:35:19 +0200 Subject: [PATCH] Feat/flavor0103 add extras (#645) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * flavor-add-extra-specs.py: Adds extra specs to SCS flavors. This helper inspects existing SCS- flavors, does some snaity checks and and adds the needed extra specs defined in scs-0104-v1. * The relevant standard is 0103 not 0104. * Add scs:cpu-type and disk0-type checks and setting. * Improve documentation. * You can explicitly pass --cpu-type and FLAVORS now. This generates the SCS extra specs even for non-SCS flavors. Mostly for operators, that are still finding their way into the SCS ecosystem and would want to see how their existing flavors map. * Add scs:name-v3 as requested in #630. * Mute flake8. * Add -n|--no-change mode. * use isinstance * Remove commented out imports. * Simplify OS_CLOUD default pickup from environment. As suggested by Matthias Büchse . * Drop setting scs:name-v3 again. * Improve check_name_extra(). This is however not yet the big rewrite as intended ... * Implement flavor_filter as suggested by mbuechse. * Save one variable (compute). * simplify logic using extant (partly improved) library * adapt unit test * simplify flavor filter logic, improve error output * Fix computation of which extra_specs to remove * appease flake8 * implement actions report, ask, apply * Bugfix: do not report existing values when no value exists * appease flake8 * Demote error to warning because we can't tell if it's a mistake * Convert our RAM in GiB to OpenStack RAM in MiB. * Option -A/--all-names to produce all scs:name-vN. Normally, if one of them exists, we produce no more. * Revert "Convert our RAM in GiB to OpenStack RAM in MiB." This reverts commit 90a75ab75ca115ec83a2ff20d3b85cef3710299b. While it does match what OpenStack expects, it's unused and breaks the CI. Needs more investigation. * Use longest SCS- name to derive reference specs. * Update help text to document -A|--all-names * Reenable -q|--quiet. Report changes without it. * Fix import order: builtins, external, internal * use logger consistently, decouple `usage` a bit * remove redundant code * use all flavornames, not just one reference name Signed-off-by: Kurt Garloff Signed-off-by: Matthias Büchse Co-authored-by: Matthias Büchse --- Tests/iaas/flavor-naming/check_yaml.py | 6 +- Tests/iaas/flavor-naming/check_yaml_test.py | 6 +- Tests/iaas/flavor-naming/cli.py | 63 +--- .../flavor-naming/flavor-add-extra-specs.py | 337 ++++++++++++++++++ Tests/iaas/flavor-naming/flavor_names.py | 95 ++++- Tests/iaas/scs-0103-v1-flavors.yaml | 174 ++++----- .../standard-flavors/flavors-openstack.py | 11 +- Tests/testing/scs-0103-v1-flavors-wrong.yaml | 170 ++++----- 8 files changed, 617 insertions(+), 245 deletions(-) create mode 100755 Tests/iaas/flavor-naming/flavor-add-extra-specs.py diff --git a/Tests/iaas/flavor-naming/check_yaml.py b/Tests/iaas/flavor-naming/check_yaml.py index b2ecbe407..3d6738917 100755 --- a/Tests/iaas/flavor-naming/check_yaml.py +++ b/Tests/iaas/flavor-naming/check_yaml.py @@ -17,8 +17,8 @@ from flavor_names import parser_v2, flavorname_to_dict -REQUIRED_FIELDS = ['name-v1', 'name-v2', 'name', 'cpus', 'ram', 'cpu-type'] -DEFAULTS = {'disk0-type': 'network'} +REQUIRED_FIELDS = ['scs:name-v1', 'scs:name-v2', 'name', 'cpus', 'ram', 'scs:cpu-type'] +DEFAULTS = {'scs:disk0-type': 'network'} class Undefined: @@ -47,7 +47,7 @@ def check_spec(self, flavor_spec): self.emit(f"flavor spec missing keys {', '.join(missing)}: {flavor_spec}") return name = flavor_spec['name'] - name_v2 = flavor_spec['name-v2'] + name_v2 = flavor_spec['scs:name-v2'] try: flavorname = parser_v2(name_v2) except Exception: diff --git a/Tests/iaas/flavor-naming/check_yaml_test.py b/Tests/iaas/flavor-naming/check_yaml_test.py index 3bb3bc12c..09cb01e62 100644 --- a/Tests/iaas/flavor-naming/check_yaml_test.py +++ b/Tests/iaas/flavor-naming/check_yaml_test.py @@ -17,12 +17,12 @@ BUGGY_YAML_DIR = Path(TEST_ROOT, "testing") EXPECTED_ERRORS = """ -ERROR: flavor 'SCS-1V-4': field 'cpu-type' contradicting name-v2 'SCS-1V-4'; found 'crowded-core', expected 'shared-core' -ERROR: flavor 'SCS-2V-8': field 'name-v1' contradicting name-v2 'SCS-2V-8'; found 'SCS-2V-8', expected 'SCS-2V:8' +ERROR: flavor 'SCS-1V-4': field 'scs:cpu-type' contradicting name-v2 'SCS-1V-4'; found 'crowded-core', expected 'shared-core' +ERROR: flavor 'SCS-2V-8': field 'scs:name-v1' contradicting name-v2 'SCS-2V-8'; found 'SCS-2V-8', expected 'SCS-2V:8' ERROR: flavor 'SCS-4V-16': field 'ram' contradicting name-v2 'SCS-4V-16'; found 12, expected 16.0 ERROR: flavor 'SCS-8V-32': field 'disk' contradicting name-v2 'SCS-8V-32'; found 128, expected undefined ERROR: flavor 'SCS-1V-2': field 'cpus' contradicting name-v2 'SCS-1V-2'; found 2, expected 1 -ERROR: flavor 'SCS-2V-4-20s': field 'disk0-type' contradicting name-v2 'SCS-2V-4-20s'; found 'network', expected 'ssd' +ERROR: flavor 'SCS-2V-4-20s': field 'scs:disk0-type' contradicting name-v2 'SCS-2V-4-20s'; found 'network', expected 'ssd' ERROR: flavor 'SCS-4V-16-100s': field 'disk' contradicting name-v2 'SCS-4V-16-100s'; found 10, expected 100 ERROR: file 'scs-0103-v1-flavors-wrong.yaml': found 7 errors """.strip() diff --git a/Tests/iaas/flavor-naming/cli.py b/Tests/iaas/flavor-naming/cli.py index ff3021e75..86969cbbb 100755 --- a/Tests/iaas/flavor-naming/cli.py +++ b/Tests/iaas/flavor-naming/cli.py @@ -6,55 +6,18 @@ import click import yaml -from flavor_names import parser_v1, parser_v2, parser_v3, inputflavor, outputter, flavorname_to_dict, prettyname - - -logger = logging.getLogger(__name__) - - -class ParsingStrategy: - """class to model parsing that accepts multiple versions of the syntax in different ways""" - - def __init__(self, parsers=(), tolerated_parsers=(), invalid_parsers=()): - self.parsers = parsers - self.tolerated_parsers = tolerated_parsers - self.invalid_parsers = invalid_parsers - - def parse(self, namestr): - exc = None - for parser in self.parsers: - try: - return parser(namestr) - except Exception as e: - if exc is None: - exc = e - # at this point, if `self.parsers` is not empty, then `exc` is not `None` - for parser in self.tolerated_parsers: - try: - result = parser(namestr) - except Exception: - pass - else: - logger.warning(f"Name is merely tolerated {parser.vstr}: {namestr}") - return result - for parser in self.invalid_parsers: - try: - result = parser(namestr) - except Exception: - pass - else: - raise ValueError(f"Name is non-tolerable {parser.vstr}") - raise exc +from flavor_names import parser_v1, parser_v2, parser_v3, inputflavor, outputter, flavorname_to_dict, \ + prettyname, ParsingStrategy -VERSIONS = { - 'v1': ParsingStrategy(parsers=(parser_v1, ), invalid_parsers=(parser_v2, )), - 'v1/v2': ParsingStrategy(parsers=(parser_v1, ), tolerated_parsers=(parser_v2, )), - 'v2/v1': ParsingStrategy(parsers=(parser_v2, ), tolerated_parsers=(parser_v1, )), - 'v2': ParsingStrategy(parsers=(parser_v2, ), invalid_parsers=(parser_v1, )), - 'v3': ParsingStrategy(parsers=(parser_v3, ), invalid_parsers=(parser_v1, )), -} -_, VERSIONS['latest'] = max(VERSIONS.items()) +PARSERS = {ps.vstr: ps for ps in [ + ParsingStrategy(vstr='v1', parsers=(parser_v1, ), invalid_parsers=(parser_v2, )), + ParsingStrategy(vstr='v1/v2', parsers=(parser_v1, ), tolerated_parsers=(parser_v2, )), + ParsingStrategy(vstr='v2/v1', parsers=(parser_v2, ), tolerated_parsers=(parser_v1, )), + ParsingStrategy(vstr='v2', parsers=(parser_v2, ), invalid_parsers=(parser_v1, )), + ParsingStrategy(vstr='v3', parsers=(parser_v3, ), invalid_parsers=(parser_v1, )), +]} +_, PARSERS['latest'] = max(PARSERS.items()) def noop(*args, **kwargs): @@ -84,7 +47,7 @@ def process_pipeline(rc, *args, **kwargs): @cli.command() -@click.argument('version', type=click.Choice(list(VERSIONS), case_sensitive=False)) +@click.argument('version', type=click.Choice(list(PARSERS), case_sensitive=False)) @click.argument('name', nargs=-1) @click.option('-o', '--output', 'output', type=click.Choice(['none', 'prose', 'yaml']), help='select output format (default: none)') @@ -96,12 +59,12 @@ def parse(cfg, version, name, output='none'): validation. With 'v1/v2', flavor names of both kinds are accepted, but warnings are emitted for v2, and similarly with 'v2/v1', where warnings are emitted for v1. """ - version = VERSIONS.get(version) + parser = PARSERS.get(version) printv = cfg.printv errors = 0 for namestr in name: try: - flavorname = version.parse(namestr) + flavorname = parser(namestr) except ValueError as exc: print(f"{exc}: {namestr}") errors += 1 diff --git a/Tests/iaas/flavor-naming/flavor-add-extra-specs.py b/Tests/iaas/flavor-naming/flavor-add-extra-specs.py new file mode 100755 index 000000000..f901f50f1 --- /dev/null +++ b/Tests/iaas/flavor-naming/flavor-add-extra-specs.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +""" +flavor-add-extra-specs.py + +Cycles through all SCS- openstack flavors and adds properties specified in +scs-0103-v1 . + +Usage: flavor-add-extra-specs.py [options] [FLAVORS] +Options: + -h|--help: Print usage information + -d|--debug: Output verbose debugging info + -q|--quiet: Only output warnings and errors + -A|--all-names: Overwrite scs:name-vN with systematic names (each + name will be kept, but may appear w/another key) + -t|--disk0-type TYPE: Assumes disk TYPE for flavors w/ unspec disk0-type + -p|--cpu-type TYPE: Assumes CPU TYPE for flavors w/o SCS name + -c|--os-cloud CLOUD: Cloud to work on (default: OS_CLOUD env) + -a|--action ACTION: What action to perform: + report: only report what changes would be performed + ask: (default) report, then ask whether to perform + apply: perform changes without asking + +By default, all SCS- flavors are processed; by passing flavor names FLAVORS as +arguments, only those are processed. +You can pass non-SCS FLAVORS and specify --cpu-type to generate SCS names and +set the SCS extra_specs. + +On most clouds, to add properties (extra_specs) to flavors, you need to have +admin power; this program will otherwise report the failed settings. +Add -d|--debug for more verbose output. + +(c) Kurt Garloff , 6/2024 +(c) Matthias Büchse , 8/2024 +SPDX-License-Identifier: CC-BY-SA-4.0 +""" + +import getopt +import logging +import os +import sys + +import openstack + +from flavor_names import parser_vN, CPUTYPE_KEY, DISKTYPE_KEY, Flavorname, Main, Disk, flavorname_to_dict, \ + SCS_NAME_PATTERN + + +logger = logging.getLogger(__name__) +DEFAULTS = {'scs:disk0-type': 'network'} + + +def usage(file=sys.stderr): + "Output usage information (help)" + print(__doc__.strip(), file=file) + + +def min_max_check(real, claim, valnm, flvnm, extra): + """Check whether property valnm real is at least claim. + Prints ERROR is lower and returns False + Prints WARNING if higher (and returns True) + Returns True if no problem detected. + For floats, we allow for 1% tolerance in both directions. + """ + # 1% tolerance for floats (RAM) + if isinstance(claim, float): + chkval = real*1.01 + chkval2 = real*0.99 + else: + chkval = real + chkval2 = real + if chkval < claim: + logger.error(f"Flavor {flvnm} claims {claim} {valnm}{extra}, but only has {real}. Needs fixing.") + return False + if chkval2 > claim: + logger.warning(f"Flavor {flvnm} claims {claim} {valnm}{extra}, but overdelivers with {real}.") + return True + + +def check_std_props(flavor, flvnm, extra=""): + """Check consistency of openstack props with parsed SCS name specs + Return no of errors found.""" + errors = 0 + # vcpus + if not min_max_check(flavor.vcpus, flvnm.cpuram.cpus, "CPUs", flavor.name, extra): + errors += 1 + # ram + if not min_max_check(flavor.ram, flvnm.cpuram.ram*1024, "MiB RAM", flavor.name, extra): + errors += 1 + # disk + disksz = 0 + if flvnm.disk: + disksz = flvnm.disk.disksize + if not min_max_check(flavor.disk, disksz, "GiB Disk", flavor.name, extra): + errors += 1 + return errors + + +def generate_flavorname(flavor, cpu_type, disk0_type): + """Generate an SCS- v2 name for flavor, + using cpu_type (and disk0_type if needed). + Returns string.""" + cpuram = Main() + cpuram.cpus = flavor.vcpus + cpuram.cputype = cpu_type + cpuram.ram = int((flavor.ram+12)/512)/2.0 + flavorname = Flavorname(main) + if flavor.disk: + disk = Disk() + disk.disksize = flavor.disk + disk.disktype = disk0_type + flavorname.disk = disk + return flavorname + + +def revert_dict(value, dct, extra=""): + "Return key that matches val, None if no match" + for key, val in dct.items(): + if val == value: + return key + logger.error(f"ERROR: {extra} {value} should be in {dct.items()}") + + +def _extract_core_items(flavorname: Flavorname): + cputype = flavorname.cpuram.cputype + disktype = None if flavorname.disk is None else flavorname.disk.disktype + return cputype, disktype + + +def _extract_core(flavorname: Flavorname): + cputype, disktype = _extract_core_items(flavorname) + return f"cputype={cputype}, disktype={disktype}" + + +class ActionReport: + @staticmethod + def set_extra_spec(flavor, key, value): + print(f'Flavor {flavor.name}: SET {key}={value}') + + @staticmethod + def del_extra_spec(flavor, key): + print(f'Flavor {flavor.name}: DELETE {key}') + + +class ActionApply: + def __init__(self, compute): + self.compute = compute + + def set_extra_spec(self, flavor, key, value): + logger.info(f'Flavor {flavor.name}: SET {key}={value}') + try: + flavor.update_extra_specs_property(self.compute, key, value) + except openstack.exceptions.SDKException as exc: + logger.error(f"{exc!r} while setting {key}={value} for {flavor.name}") + + def del_extra_spec(self, flavor, key): + logger.info(f'Flavor {flavor.name}: DELETE {key}') + try: + flavor.delete_extra_specs_property(self.compute, key) + except openstack.exceptions.SDKException as exc: + logger.error(f"{exc!r} while deleting {key} for {flavor.name}") + + +class SetCommand: + def __init__(self, flavor, key, value): + self.flavor = flavor + self.key = key + self.value = value + + def apply(self, action): + action.set_extra_spec(self.flavor, self.key, self.value) + + +class DelCommand: + def __init__(self, flavor, key): + self.flavor = flavor + self.key = key + + def apply(self, action): + action.del_extra_spec(self.flavor, self.key) + + +def handle_commands(action, compute, commands): + if not commands: + return + if action in ('ask', 'report'): + action_report = ActionReport() + print(f'Proposing the following {len(commands)} changes to extra_specs:') + for command in commands: + command.apply(action_report) + if action == 'ask': + print('Do you want to apply these changes? y/n') + if input() == 'y': + action = 'apply' + else: + print('No changes will be applied.') + if action == 'apply': + action_apply = ActionApply(compute) + for command in commands: + command.apply(action_apply) + + +def main(argv): + action = "ask" # or "report" or "apply" + + errors = 0 + disk0_type = None + cpu_type = None + gen_all_names = False + + cloud = os.environ.get("OS_CLOUD") + try: + opts, flvs = getopt.gnu_getopt(argv, "hdqAt:p:c:a:", + ("help", "debug", "quiet", "all-names", + "disk0-type=", "cpu-type=", "os-cloud=", "action=")) + except getopt.GetoptError as exc: + logger.critical(repr(exc)) + usage() + return 1 + for opt in opts: + if opt[0] == "-h" or opt[0] == "--help": + usage(file=sys.stdout) + return 0 + if opt[0] == "-q" or opt[0] == "--quiet": + logging.getLogger().setLevel(logging.WARNING) + if opt[0] == "-d" or opt[0] == "--debug": + logging.getLogger().setLevel(logging.DEBUG) + if opt[0] == "-A" or opt[0] == "--all-names": + gen_all_names = True + if opt[0] == "-a" or opt[0] == "--action": + action = opt[1].strip().lower() + if opt[0] == "-c" or opt[0] == "--os-cloud": + cloud = opt[1] + if opt[0] == "-t" or opt[0] == "--disk0-type": + disk0_type = opt[1] + if disk0_type not in DISKTYPE_KEY: + disk0_type = revert_dict(disk0_type, DISKTYPE_KEY) + if not disk0_type: + return 2 + if opt[0] == "-p" or opt[0] == "--cpu-type": + cpu_type = opt[1] + if cpu_type not in CPUTYPE_KEY: + cpu_type = revert_dict(cpu_type, CPUTYPE_KEY) + if not cpu_type: + return 2 + + if action not in ('ask', 'report', 'apply'): + logger.error("action needs to be one of ask, report, apply") + usage() + return 4 + + if not cloud: + logger.error("Need to pass -c|--os-cloud|OS_CLOUD env") + usage() + return 3 + + conn = openstack.connect(cloud) + conn.authorize() + + # select relevant flavors: either given via name, or all SCS flavors + predicate = (lambda fn: fn in flvs) if flvs else (lambda fn: fn.startswith('SCS-')) + flavors = [flavor for flavor in conn.compute.flavors() if predicate(flavor.name)] + # This is likely a user error, so make them aware + if len(flavors) < len(flvs): + missing = set(flvs) - set(flavor.name for flavor in flavors) + logger.warning("Flavors not found: " + ", ".join(missing)) + + commands = [] + for flavor in flavors: + extra_names_to_check = [ + (key, value) + for key, value in flavor.extra_specs.items() + if SCS_NAME_PATTERN.match(key) + ] + names_to_check = [('name', flavor.name)] if flavor.name.startswith('SCS-') else [] + names_to_check.extend(extra_names_to_check) + + # syntax check: compute flavorname instances + # sanity check: claims must be true wrt actual flavor + flavornames = {} + for key, name_str in names_to_check: + try: + flavornames[key] = flavorname = parser_vN(name_str) + except ValueError as exc: + logger.error(f"could not parse {key}={name_str}: {exc!r}") + errors += 1 + else: + errors += check_std_props(flavor, flavorname, " by name") + + if not flavornames: + # we need cputype and disktype from user + if not cpu_type: + logger.warning(f"Need to specify cpu-type for generating name for {flavor.name}, skipping") + continue + if flavor.disk and not disk0_type: + logger.warning(f"Need to specify disk0-type for generating name for {flavor.name}, skipping") + continue + flavornames['_generated'] = generate_flavorname(flavor, cpu_type, disk0_type) + + expected = flavorname_to_dict(*flavornames.values(), ctx=flavor.name) + # determine invalid keys (within scs namespace) + # scs:name-vN is always permissible + removals = [ + key + for key in flavor.extra_specs + if key.startswith('scs:') and not SCS_NAME_PATTERN.match(key) + if expected.get(key, DEFAULTS.get(key)) is None + ] + logger.debug(f"Flavor {flavor.name}: expected={expected}, removals={removals}") + + for key in removals: + commands.append(DelCommand(flavor, key)) + + # generate or rectify extra_specs + for key, value in expected.items(): + if not key.startswith("scs:"): + continue + if not gen_all_names and key.startswith("scs:name-v") and extra_names_to_check: + continue # do not generate names if names are present + current = flavor.extra_specs.get(key) + if current == value: + continue + if current is None and DEFAULTS.get(key) == value: + continue + if current is not None: + logger.warning(f"{flavor.name}: resetting {key} because {current} != expected {value}") + commands.append(SetCommand(flavor, key, value)) + + handle_commands(action, conn.compute, commands) + logger.info(f"Processed {len(flavors)} flavors, {len(commands)} changes") + return errors + + +if __name__ == "__main__": + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + openstack.enable_logging(debug=False) + sys.exit(min(127, main(sys.argv[1:]))) # cap at 127 due to OS restrictions diff --git a/Tests/iaas/flavor-naming/flavor_names.py b/Tests/iaas/flavor-naming/flavor_names.py index d8573af38..08b6d11d1 100644 --- a/Tests/iaas/flavor-naming/flavor_names.py +++ b/Tests/iaas/flavor-naming/flavor_names.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +from collections import defaultdict +import logging import os import os.path import re @@ -9,8 +11,13 @@ import yaml +logger = logging.getLogger(__name__) + +SCS_NAME_PATTERN = re.compile(r"scs:name-v\d+\Z") CPUTYPE_KEY = {'L': 'crowded-core', 'V': 'shared-core', 'T': 'dedicated-thread', 'C': 'dedicated-core'} +CPUTYPE_SORT = {'crowded-core': 0, 'shared-core': 1, 'dedicated-thread': 2, 'dedicated-core': 3} DISKTYPE_KEY = {'n': 'network', 'h': 'hdd', 's': 'ssd', 'p': 'nvme'} +DISKTYPE_SORT = {'network': 0, 'hdd': 1, 'ssd': 2, 'nvme': 3} HERE = Path(__file__).parent @@ -437,6 +444,46 @@ def __call__(self, s: str, pos=0) -> Flavorname: return flavorname +class ParsingStrategy: + """ + Composite parser that accepts multiple versions of the syntax in different ways + + Follows the contract of class `Parser` + """ + + def __init__(self, vstr, parsers=(), tolerated_parsers=(), invalid_parsers=()): + self.vstr = vstr + self.parsers = parsers + self.tolerated_parsers = tolerated_parsers + self.invalid_parsers = invalid_parsers + + def __call__(self, namestr: str) -> Flavorname: + exc = None + for parser in self.parsers: + try: + return parser(namestr) + except Exception as e: + if exc is None: + exc = e + # at this point, if `self.parsers` is not empty, then `exc` is not `None` + for parser in self.tolerated_parsers: + try: + result = parser(namestr) + except Exception: + pass + else: + logger.warning(f"Name is merely tolerated {parser.vstr}: {namestr}") + return result + for parser in self.invalid_parsers: + try: + result = parser(namestr) + except Exception: + pass + else: + raise ValueError(f"Name is non-tolerable {parser.vstr}") + raise exc + + def _convert_user_input(idx, attr, target, val): """auxiliary function that converts user-input string `val` to the target attribute type""" fdesc = attr.name @@ -542,23 +589,49 @@ def __call__(self): parser_v1 = Parser("v1", SyntaxV1) parser_v2 = Parser("v2", SyntaxV2) parser_v3 = Parser("v3", SyntaxV2) # this is the same as parser_v2 except for the vstr +parser_vN = ParsingStrategy(vstr="vN", parsers=(parser_v2, parser_v1)) outname = outputter = Outputter() inputflavor = inputter = Inputter() -def flavorname_to_dict(flavorname: Flavorname) -> dict: - name_v2 = outputter(flavorname) +def flavorname_to_dict(*flavornames: Flavorname, ctx='') -> dict: + if not flavornames: + raise RuntimeError("need to supply at least one Flavorname instance!") + if ctx: + ctx = ctx + ': ' # used for logging warnings + name_collection = set() + collection = defaultdict(set) + for flavorname in flavornames: + collection['cpus'].add(flavorname.cpuram.cpus) + collection['ram'].add(flavorname.cpuram.ram) + collection['scs:cpu-type'].add(CPUTYPE_KEY[flavorname.cpuram.cputype]) + if flavorname.disk: + collection['disk'].add(flavorname.disk.disksize) + collection['nrdisks'].add(flavorname.disk.nrdisks) # this will need some postprocessing + collection['scs:disk0-type'].add(DISKTYPE_KEY[flavorname.disk.disktype or 'n']) + name_v2 = outputter(flavorname) + name_collection.add((SyntaxV1.from_v2(name_v2), "v1")) + name_collection.add((name_v2, "v2")) + short_v2 = outputter(flavorname.shorten()) + # could check whether short_v2 != name_v2, but the set will swallow everything + name_collection.add((SyntaxV1.from_v2(short_v2), "v1")) + name_collection.add((short_v2, "v2")) + for key, values in collection.items(): + if len(values) > 1: + logger.warning(f"{ctx}Inconsistent {key}: {', '.join(values)}") result = { - 'cpus': flavorname.cpuram.cpus, - 'cpu-type': CPUTYPE_KEY[flavorname.cpuram.cputype], - 'ram': flavorname.cpuram.ram, - 'name-v1': SyntaxV1.from_v2(name_v2), - 'name-v2': name_v2, + 'cpus': max(collection['cpus']), + 'scs:cpu-type': max(collection['scs:cpu-type'], key=CPUTYPE_SORT.__getitem__), + 'ram': max(collection['ram']), } - if flavorname.disk: - result['disk'] = flavorname.disk.disksize - for i in range(flavorname.disk.nrdisks): - result[f'disk{i}-type'] = DISKTYPE_KEY[flavorname.disk.disktype or 'n'] + if collection['nrdisks']: + result['disk'] = max(collection['disk']) + disktype = max(collection['scs:disk0-type'], key=DISKTYPE_SORT.__getitem__) + for i in range(max(collection['nrdisks'])): + result[f'scs:disk{i}-type'] = disktype + names = [item[0] for item in sorted(name_collection, key=lambda item: (-len(item[0]), item[1]))] + for idx, name in enumerate(names): + result[f'scs:name-v{idx + 1}'] = name return result diff --git a/Tests/iaas/scs-0103-v1-flavors.yaml b/Tests/iaas/scs-0103-v1-flavors.yaml index 9d23b8527..66c02cdd6 100644 --- a/Tests/iaas/scs-0103-v1-flavors.yaml +++ b/Tests/iaas/scs-0103-v1-flavors.yaml @@ -1,194 +1,194 @@ meta: - name_key: name-v2 + name_key: "scs:name-v2" flavor_groups: - status: mandatory list: - name: SCS-1V-4 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 - name-v1: SCS-1V:4 - name-v2: SCS-1V-4 + "scs:name-v1": SCS-1V:4 + "scs:name-v2": SCS-1V-4 - name: SCS-2V-8 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-2V:8 - name-v2: SCS-2V-8 + "scs:name-v1": SCS-2V:8 + "scs:name-v2": SCS-2V-8 - name: SCS-4V-16 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-4V:16 - name-v2: SCS-4V-16 + "scs:name-v1": SCS-4V:16 + "scs:name-v2": SCS-4V-16 - name: SCS-8V-32 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-8V:32 - name-v2: SCS-8V-32 + "scs:name-v1": SCS-8V:32 + "scs:name-v2": SCS-8V-32 - name: SCS-1V-2 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 - name-v1: SCS-1V:2 - name-v2: SCS-1V-2 + "scs:name-v1": SCS-1V:2 + "scs:name-v2": SCS-1V-2 - name: SCS-2V-4 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 - name-v1: SCS-2V:4 - name-v2: SCS-2V-4 + "scs:name-v1": SCS-2V:4 + "scs:name-v2": SCS-2V-4 - name: SCS-4V-8 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-4V:8 - name-v2: SCS-4V-8 + "scs:name-v1": SCS-4V:8 + "scs:name-v2": SCS-4V-8 - name: SCS-8V-16 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-8V:16 - name-v2: SCS-8V-16 + "scs:name-v1": SCS-8V:16 + "scs:name-v2": SCS-8V-16 - name: SCS-16V-32 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-16V:32 - name-v2: SCS-16V-32 + "scs:name-v1": SCS-16V:32 + "scs:name-v2": SCS-16V-32 - name: SCS-1V-8 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-1V:8 - name-v2: SCS-1V-8 + "scs:name-v1": SCS-1V:8 + "scs:name-v2": SCS-1V-8 - name: SCS-2V-16 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-2V:16 - name-v2: SCS-2V-16 + "scs:name-v1": SCS-2V:16 + "scs:name-v2": SCS-2V-16 - name: SCS-4V-32 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-4V:32 - name-v2: SCS-4V-32 + "scs:name-v1": SCS-4V:32 + "scs:name-v2": SCS-4V-32 - name: SCS-1L-1 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 - name-v1: SCS-1L:1 - name-v2: SCS-1L-1 + "scs:name-v1": SCS-1L:1 + "scs:name-v2": SCS-1L-1 - status: mandatory list: - name: SCS-2V-4-20s cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 20 - disk0-type: ssd - name-v1: SCS-2V:4:20s - name-v2: SCS-2V-4-20s + "scs:disk0-type": ssd + "scs:name-v1": SCS-2V:4:20s + "scs:name-v2": SCS-2V-4-20s - name: SCS-4V-16-100s cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 100 - disk0-type: ssd - name-v1: SCS-4V:16:100s - name-v2: SCS-4V-16-100s + "scs:disk0-type": ssd + "scs:name-v1": SCS-4V:16:100s + "scs:name-v2": SCS-4V-16-100s - status: recommended list: - name: SCS-1V-4-10 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-1V:4:10 - name-v2: SCS-1V-4-10 + "scs:name-v1": SCS-1V:4:10 + "scs:name-v2": SCS-1V-4-10 - name: SCS-2V-8-20 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-2V:8:20 - name-v2: SCS-2V-8-20 + "scs:name-v1": SCS-2V:8:20 + "scs:name-v2": SCS-2V-8-20 - name: SCS-4V-16-50 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-4V:16:50 - name-v2: SCS-4V-16-50 + "scs:name-v1": SCS-4V:16:50 + "scs:name-v2": SCS-4V-16-50 - name: SCS-8V-32-100 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-8V:32:100 - name-v2: SCS-8V-32-100 + "scs:name-v1": SCS-8V:32:100 + "scs:name-v2": SCS-8V-32-100 - name: SCS-1V-2-5 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 disk: 5 - name-v1: SCS-1V:2:5 - name-v2: SCS-1V-2-5 + "scs:name-v1": SCS-1V:2:5 + "scs:name-v2": SCS-1V-2-5 - name: SCS-2V-4-10 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-2V:4:10 - name-v2: SCS-2V-4-10 + "scs:name-v1": SCS-2V:4:10 + "scs:name-v2": SCS-2V-4-10 - name: SCS-4V-8-20 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-4V:8:20 - name-v2: SCS-4V-8-20 + "scs:name-v1": SCS-4V:8:20 + "scs:name-v2": SCS-4V-8-20 - name: SCS-8V-16-50 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-8V:16:50 - name-v2: SCS-8V-16-50 + "scs:name-v1": SCS-8V:16:50 + "scs:name-v2": SCS-8V-16-50 - name: SCS-16V-32-100 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-16V:32:100 - name-v2: SCS-16V-32-100 + "scs:name-v1": SCS-16V:32:100 + "scs:name-v2": SCS-16V-32-100 - name: SCS-1V-8-20 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-1V:8:20 - name-v2: SCS-1V-8-20 + "scs:name-v1": SCS-1V:8:20 + "scs:name-v2": SCS-1V-8-20 - name: SCS-2V-16-50 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-2V:16:50 - name-v2: SCS-2V-16-50 + "scs:name-v1": SCS-2V:16:50 + "scs:name-v2": SCS-2V-16-50 - name: SCS-4V-32-100 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-4V:32:100 - name-v2: SCS-4V-32-100 + "scs:name-v1": SCS-4V:32:100 + "scs:name-v2": SCS-4V-32-100 - name: SCS-1L-1-5 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 disk: 5 - name-v1: SCS-1L:1:5 - name-v2: SCS-1L-1-5 + "scs:name-v1": SCS-1L:1:5 + "scs:name-v2": SCS-1L-1-5 diff --git a/Tests/iaas/standard-flavors/flavors-openstack.py b/Tests/iaas/standard-flavors/flavors-openstack.py index 61d35fb36..2680fa822 100755 --- a/Tests/iaas/standard-flavors/flavors-openstack.py +++ b/Tests/iaas/standard-flavors/flavors-openstack.py @@ -105,7 +105,6 @@ def main(argv): logger.critical("Flavor definition missing 'flavor_groups' field") name_key = flavor_spec_data['meta']['name_key'] - es_name_key = f"scs:{name_key}" # compute union of all flavor groups, copying group info (mainly "status") to each flavor # check if the spec is complete while we are at it flavor_specs = [] @@ -128,9 +127,9 @@ def main(argv): with openstack.connect(cloud=cloud, timeout=32) as conn: present_flavors = conn.list_flavors(get_extra=True) by_name = { - flavor.extra_specs[es_name_key]: flavor + flavor.extra_specs[name_key]: flavor for flavor in present_flavors - if es_name_key in flavor.extra_specs + if name_key in flavor.extra_specs } by_legacy_name = {flavor.name: flavor for flavor in present_flavors} # for reserved keys, keep track of all flavors that don't have a matching spec @@ -145,7 +144,7 @@ def main(argv): if not flavor: flavor = by_legacy_name.get(flavor_spec[name_key]) if flavor: - logger.warning(f"Flavor '{flavor_spec['name']}' found via name only, missing property {es_name_key!r}") + logger.warning(f"Flavor '{flavor_spec['name']}' found via name only, missing property {name_key!r}") else: status = flavor_spec['_group']['status'] level = {"mandatory": logging.ERROR}.get(status, logging.WARNING) @@ -165,9 +164,9 @@ def main(argv): report = [ f"{key}: {es_value!r} should be {value!r}" for key, value, es_value in [ - (key, value, flavor.extra_specs.get(f"scs:{key}")) + (key, value, flavor.extra_specs.get(key)) for key, value in flavor_spec.items() - if key not in ('_group', 'name', 'cpus', 'ram', 'disk') + if key.startswith("scs:") ] if value != es_value ] diff --git a/Tests/testing/scs-0103-v1-flavors-wrong.yaml b/Tests/testing/scs-0103-v1-flavors-wrong.yaml index 60d1c7bc2..7aff89a0f 100644 --- a/Tests/testing/scs-0103-v1-flavors-wrong.yaml +++ b/Tests/testing/scs-0103-v1-flavors-wrong.yaml @@ -5,191 +5,191 @@ flavor_groups: list: - name: SCS-1V-4 cpus: 1 - cpu-type: crowded-core # wrong: name suggests shared-core + "scs:cpu-type": crowded-core # wrong: name suggests shared-core ram: 4 - name-v1: SCS-1V:4 - name-v2: SCS-1V-4 + "scs:name-v1": SCS-1V:4 + "scs:name-v2": SCS-1V-4 - name: SCS-2V-8 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-2V-8 # wrong: not a v1 name - name-v2: SCS-2V-8 + "scs:name-v1": SCS-2V-8 # wrong: not a v1 name + "scs:name-v2": SCS-2V-8 - name: SCS-4V-16 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 12 # wrong: name suggests 16 - name-v1: SCS-4V:16 - name-v2: SCS-4V-16 + "scs:name-v1": SCS-4V:16 + "scs:name-v2": SCS-4V-16 - name: SCS-8V-32 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 128 # wrong: no disk in name - name-v1: SCS-8V:32 - name-v2: SCS-8V-32 + "scs:name-v1": SCS-8V:32 + "scs:name-v2": SCS-8V-32 - name: SCS-1V-2 cpus: 2 # wrong: name suggests 1 cpu - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 - name-v1: SCS-1V:2 - name-v2: SCS-1V-2 + "scs:name-v1": SCS-1V:2 + "scs:name-v2": SCS-1V-2 - name: SCS-2V-4 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 - name-v1: SCS-2V:4 - name-v2: SCS-2V-4 + "scs:name-v1": SCS-2V:4 + "scs:name-v2": SCS-2V-4 - name: SCS-4V-8 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-4V:8 - name-v2: SCS-4V-8 + "scs:name-v1": SCS-4V:8 + "scs:name-v2": SCS-4V-8 - name: SCS-8V-16 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-8V:16 - name-v2: SCS-8V-16 + "scs:name-v1": SCS-8V:16 + "scs:name-v2": SCS-8V-16 - name: SCS-16V-32 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-16V:32 - name-v2: SCS-16V-32 + "scs:name-v1": SCS-16V:32 + "scs:name-v2": SCS-16V-32 - name: SCS-1V-8 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-1V:8 - name-v2: SCS-1V-8 + "scs:name-v1": SCS-1V:8 + "scs:name-v2": SCS-1V-8 - name: SCS-2V-16 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-2V:16 - name-v2: SCS-2V-16 + "scs:name-v1": SCS-2V:16 + "scs:name-v2": SCS-2V-16 - name: SCS-4V-32 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-4V:32 - name-v2: SCS-4V-32 + "scs:name-v1": SCS-4V:32 + "scs:name-v2": SCS-4V-32 - name: SCS-1L-1 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 - name-v1: SCS-1L:1 - name-v2: SCS-1L-1 + "scs:name-v1": SCS-1L:1 + "scs:name-v2": SCS-1L-1 - status: mandatory list: - name: SCS-2V-4-20s cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 20 # wrong: name suggests disk-type ssd - name-v1: SCS-2V:4:20s - name-v2: SCS-2V-4-20s + "scs:name-v1": SCS-2V:4:20s + "scs:name-v2": SCS-2V-4-20s - name: SCS-4V-16-100s cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 10 # wrong: name suggests 100 - disk0-type: ssd - name-v1: SCS-4V:16:100s - name-v2: SCS-4V-16-100s + "scs:disk0-type": ssd + "scs:name-v1": SCS-4V:16:100s + "scs:name-v2": SCS-4V-16-100s - status: recommended list: - name: SCS-1V-4-10 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-1V:4:10 - name-v2: SCS-1V-4-10 + "scs:name-v1": SCS-1V:4:10 + "scs:name-v2": SCS-1V-4-10 - name: SCS-2V-8-20 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-2V:8:20 - name-v2: SCS-2V-8-20 + "scs:name-v1": SCS-2V:8:20 + "scs:name-v2": SCS-2V-8-20 - name: SCS-4V-16-50 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-4V:16:50 - name-v2: SCS-4V-16-50 + "scs:name-v1": SCS-4V:16:50 + "scs:name-v2": SCS-4V-16-50 - name: SCS-8V-32-100 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-8V:32:100 - name-v2: SCS-8V-32-100 + "scs:name-v1": SCS-8V:32:100 + "scs:name-v2": SCS-8V-32-100 - name: SCS-1V-2-5 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 disk: 5 - name-v1: SCS-1V:2:5 - name-v2: SCS-1V-2-5 + "scs:name-v1": SCS-1V:2:5 + "scs:name-v2": SCS-1V-2-5 - name: SCS-2V-4-10 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-2V:4:10 - name-v2: SCS-2V-4-10 + "scs:name-v1": SCS-2V:4:10 + "scs:name-v2": SCS-2V-4-10 - name: SCS-4V-8-20 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-4V:8:20 - name-v2: SCS-4V-8-20 + "scs:name-v1": SCS-4V:8:20 + "scs:name-v2": SCS-4V-8-20 - name: SCS-8V-16-50 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-8V:16:50 - name-v2: SCS-8V-16-50 + "scs:name-v1": SCS-8V:16:50 + "scs:name-v2": SCS-8V-16-50 - name: SCS-16V-32-100 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-16V:32:100 - name-v2: SCS-16V-32-100 + "scs:name-v1": SCS-16V:32:100 + "scs:name-v2": SCS-16V-32-100 - name: SCS-1V-8-20 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-1V:8:20 - name-v2: SCS-1V-8-20 + "scs:name-v1": SCS-1V:8:20 + "scs:name-v2": SCS-1V-8-20 - name: SCS-2V-16-50 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-2V:16:50 - name-v2: SCS-2V-16-50 + "scs:name-v1": SCS-2V:16:50 + "scs:name-v2": SCS-2V-16-50 - name: SCS-4V-32-100 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-4V:32:100 - name-v2: SCS-4V-32-100 + "scs:name-v1": SCS-4V:32:100 + "scs:name-v2": SCS-4V-32-100 - name: SCS-1L-1-5 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 disk: 5 - name-v1: SCS-1L:1:5 - name-v2: SCS-1L-1-5 + "scs:name-v1": SCS-1L:1:5 + "scs:name-v2": SCS-1L-1-5