diff --git a/Tests/iaas/flavor-naming/check_yaml.py b/Tests/iaas/flavor-naming/check_yaml.py index b2ecbe407..3d6738917 100755 --- a/Tests/iaas/flavor-naming/check_yaml.py +++ b/Tests/iaas/flavor-naming/check_yaml.py @@ -17,8 +17,8 @@ from flavor_names import parser_v2, flavorname_to_dict -REQUIRED_FIELDS = ['name-v1', 'name-v2', 'name', 'cpus', 'ram', 'cpu-type'] -DEFAULTS = {'disk0-type': 'network'} +REQUIRED_FIELDS = ['scs:name-v1', 'scs:name-v2', 'name', 'cpus', 'ram', 'scs:cpu-type'] +DEFAULTS = {'scs:disk0-type': 'network'} class Undefined: @@ -47,7 +47,7 @@ def check_spec(self, flavor_spec): self.emit(f"flavor spec missing keys {', '.join(missing)}: {flavor_spec}") return name = flavor_spec['name'] - name_v2 = flavor_spec['name-v2'] + name_v2 = flavor_spec['scs:name-v2'] try: flavorname = parser_v2(name_v2) except Exception: diff --git a/Tests/iaas/flavor-naming/check_yaml_test.py b/Tests/iaas/flavor-naming/check_yaml_test.py index 3bb3bc12c..09cb01e62 100644 --- a/Tests/iaas/flavor-naming/check_yaml_test.py +++ b/Tests/iaas/flavor-naming/check_yaml_test.py @@ -17,12 +17,12 @@ BUGGY_YAML_DIR = Path(TEST_ROOT, "testing") EXPECTED_ERRORS = """ -ERROR: flavor 'SCS-1V-4': field 'cpu-type' contradicting name-v2 'SCS-1V-4'; found 'crowded-core', expected 'shared-core' -ERROR: flavor 'SCS-2V-8': field 'name-v1' contradicting name-v2 'SCS-2V-8'; found 'SCS-2V-8', expected 'SCS-2V:8' +ERROR: flavor 'SCS-1V-4': field 'scs:cpu-type' contradicting name-v2 'SCS-1V-4'; found 'crowded-core', expected 'shared-core' +ERROR: flavor 'SCS-2V-8': field 'scs:name-v1' contradicting name-v2 'SCS-2V-8'; found 'SCS-2V-8', expected 'SCS-2V:8' ERROR: flavor 'SCS-4V-16': field 'ram' contradicting name-v2 'SCS-4V-16'; found 12, expected 16.0 ERROR: flavor 'SCS-8V-32': field 'disk' contradicting name-v2 'SCS-8V-32'; found 128, expected undefined ERROR: flavor 'SCS-1V-2': field 'cpus' contradicting name-v2 'SCS-1V-2'; found 2, expected 1 -ERROR: flavor 'SCS-2V-4-20s': field 'disk0-type' contradicting name-v2 'SCS-2V-4-20s'; found 'network', expected 'ssd' +ERROR: flavor 'SCS-2V-4-20s': field 'scs:disk0-type' contradicting name-v2 'SCS-2V-4-20s'; found 'network', expected 'ssd' ERROR: flavor 'SCS-4V-16-100s': field 'disk' contradicting name-v2 'SCS-4V-16-100s'; found 10, expected 100 ERROR: file 'scs-0103-v1-flavors-wrong.yaml': found 7 errors """.strip() diff --git a/Tests/iaas/flavor-naming/cli.py b/Tests/iaas/flavor-naming/cli.py index ff3021e75..86969cbbb 100755 --- a/Tests/iaas/flavor-naming/cli.py +++ b/Tests/iaas/flavor-naming/cli.py @@ -6,55 +6,18 @@ import click import yaml -from flavor_names import parser_v1, parser_v2, parser_v3, inputflavor, outputter, flavorname_to_dict, prettyname - - -logger = logging.getLogger(__name__) - - -class ParsingStrategy: - """class to model parsing that accepts multiple versions of the syntax in different ways""" - - def __init__(self, parsers=(), tolerated_parsers=(), invalid_parsers=()): - self.parsers = parsers - self.tolerated_parsers = tolerated_parsers - self.invalid_parsers = invalid_parsers - - def parse(self, namestr): - exc = None - for parser in self.parsers: - try: - return parser(namestr) - except Exception as e: - if exc is None: - exc = e - # at this point, if `self.parsers` is not empty, then `exc` is not `None` - for parser in self.tolerated_parsers: - try: - result = parser(namestr) - except Exception: - pass - else: - logger.warning(f"Name is merely tolerated {parser.vstr}: {namestr}") - return result - for parser in self.invalid_parsers: - try: - result = parser(namestr) - except Exception: - pass - else: - raise ValueError(f"Name is non-tolerable {parser.vstr}") - raise exc +from flavor_names import parser_v1, parser_v2, parser_v3, inputflavor, outputter, flavorname_to_dict, \ + prettyname, ParsingStrategy -VERSIONS = { - 'v1': ParsingStrategy(parsers=(parser_v1, ), invalid_parsers=(parser_v2, )), - 'v1/v2': ParsingStrategy(parsers=(parser_v1, ), tolerated_parsers=(parser_v2, )), - 'v2/v1': ParsingStrategy(parsers=(parser_v2, ), tolerated_parsers=(parser_v1, )), - 'v2': ParsingStrategy(parsers=(parser_v2, ), invalid_parsers=(parser_v1, )), - 'v3': ParsingStrategy(parsers=(parser_v3, ), invalid_parsers=(parser_v1, )), -} -_, VERSIONS['latest'] = max(VERSIONS.items()) +PARSERS = {ps.vstr: ps for ps in [ + ParsingStrategy(vstr='v1', parsers=(parser_v1, ), invalid_parsers=(parser_v2, )), + ParsingStrategy(vstr='v1/v2', parsers=(parser_v1, ), tolerated_parsers=(parser_v2, )), + ParsingStrategy(vstr='v2/v1', parsers=(parser_v2, ), tolerated_parsers=(parser_v1, )), + ParsingStrategy(vstr='v2', parsers=(parser_v2, ), invalid_parsers=(parser_v1, )), + ParsingStrategy(vstr='v3', parsers=(parser_v3, ), invalid_parsers=(parser_v1, )), +]} +_, PARSERS['latest'] = max(PARSERS.items()) def noop(*args, **kwargs): @@ -84,7 +47,7 @@ def process_pipeline(rc, *args, **kwargs): @cli.command() -@click.argument('version', type=click.Choice(list(VERSIONS), case_sensitive=False)) +@click.argument('version', type=click.Choice(list(PARSERS), case_sensitive=False)) @click.argument('name', nargs=-1) @click.option('-o', '--output', 'output', type=click.Choice(['none', 'prose', 'yaml']), help='select output format (default: none)') @@ -96,12 +59,12 @@ def parse(cfg, version, name, output='none'): validation. With 'v1/v2', flavor names of both kinds are accepted, but warnings are emitted for v2, and similarly with 'v2/v1', where warnings are emitted for v1. """ - version = VERSIONS.get(version) + parser = PARSERS.get(version) printv = cfg.printv errors = 0 for namestr in name: try: - flavorname = version.parse(namestr) + flavorname = parser(namestr) except ValueError as exc: print(f"{exc}: {namestr}") errors += 1 diff --git a/Tests/iaas/flavor-naming/flavor-add-extra-specs.py b/Tests/iaas/flavor-naming/flavor-add-extra-specs.py new file mode 100755 index 000000000..f901f50f1 --- /dev/null +++ b/Tests/iaas/flavor-naming/flavor-add-extra-specs.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +""" +flavor-add-extra-specs.py + +Cycles through all SCS- openstack flavors and adds properties specified in +scs-0103-v1 . + +Usage: flavor-add-extra-specs.py [options] [FLAVORS] +Options: + -h|--help: Print usage information + -d|--debug: Output verbose debugging info + -q|--quiet: Only output warnings and errors + -A|--all-names: Overwrite scs:name-vN with systematic names (each + name will be kept, but may appear w/another key) + -t|--disk0-type TYPE: Assumes disk TYPE for flavors w/ unspec disk0-type + -p|--cpu-type TYPE: Assumes CPU TYPE for flavors w/o SCS name + -c|--os-cloud CLOUD: Cloud to work on (default: OS_CLOUD env) + -a|--action ACTION: What action to perform: + report: only report what changes would be performed + ask: (default) report, then ask whether to perform + apply: perform changes without asking + +By default, all SCS- flavors are processed; by passing flavor names FLAVORS as +arguments, only those are processed. +You can pass non-SCS FLAVORS and specify --cpu-type to generate SCS names and +set the SCS extra_specs. + +On most clouds, to add properties (extra_specs) to flavors, you need to have +admin power; this program will otherwise report the failed settings. +Add -d|--debug for more verbose output. + +(c) Kurt Garloff , 6/2024 +(c) Matthias Büchse , 8/2024 +SPDX-License-Identifier: CC-BY-SA-4.0 +""" + +import getopt +import logging +import os +import sys + +import openstack + +from flavor_names import parser_vN, CPUTYPE_KEY, DISKTYPE_KEY, Flavorname, Main, Disk, flavorname_to_dict, \ + SCS_NAME_PATTERN + + +logger = logging.getLogger(__name__) +DEFAULTS = {'scs:disk0-type': 'network'} + + +def usage(file=sys.stderr): + "Output usage information (help)" + print(__doc__.strip(), file=file) + + +def min_max_check(real, claim, valnm, flvnm, extra): + """Check whether property valnm real is at least claim. + Prints ERROR is lower and returns False + Prints WARNING if higher (and returns True) + Returns True if no problem detected. + For floats, we allow for 1% tolerance in both directions. + """ + # 1% tolerance for floats (RAM) + if isinstance(claim, float): + chkval = real*1.01 + chkval2 = real*0.99 + else: + chkval = real + chkval2 = real + if chkval < claim: + logger.error(f"Flavor {flvnm} claims {claim} {valnm}{extra}, but only has {real}. Needs fixing.") + return False + if chkval2 > claim: + logger.warning(f"Flavor {flvnm} claims {claim} {valnm}{extra}, but overdelivers with {real}.") + return True + + +def check_std_props(flavor, flvnm, extra=""): + """Check consistency of openstack props with parsed SCS name specs + Return no of errors found.""" + errors = 0 + # vcpus + if not min_max_check(flavor.vcpus, flvnm.cpuram.cpus, "CPUs", flavor.name, extra): + errors += 1 + # ram + if not min_max_check(flavor.ram, flvnm.cpuram.ram*1024, "MiB RAM", flavor.name, extra): + errors += 1 + # disk + disksz = 0 + if flvnm.disk: + disksz = flvnm.disk.disksize + if not min_max_check(flavor.disk, disksz, "GiB Disk", flavor.name, extra): + errors += 1 + return errors + + +def generate_flavorname(flavor, cpu_type, disk0_type): + """Generate an SCS- v2 name for flavor, + using cpu_type (and disk0_type if needed). + Returns string.""" + cpuram = Main() + cpuram.cpus = flavor.vcpus + cpuram.cputype = cpu_type + cpuram.ram = int((flavor.ram+12)/512)/2.0 + flavorname = Flavorname(main) + if flavor.disk: + disk = Disk() + disk.disksize = flavor.disk + disk.disktype = disk0_type + flavorname.disk = disk + return flavorname + + +def revert_dict(value, dct, extra=""): + "Return key that matches val, None if no match" + for key, val in dct.items(): + if val == value: + return key + logger.error(f"ERROR: {extra} {value} should be in {dct.items()}") + + +def _extract_core_items(flavorname: Flavorname): + cputype = flavorname.cpuram.cputype + disktype = None if flavorname.disk is None else flavorname.disk.disktype + return cputype, disktype + + +def _extract_core(flavorname: Flavorname): + cputype, disktype = _extract_core_items(flavorname) + return f"cputype={cputype}, disktype={disktype}" + + +class ActionReport: + @staticmethod + def set_extra_spec(flavor, key, value): + print(f'Flavor {flavor.name}: SET {key}={value}') + + @staticmethod + def del_extra_spec(flavor, key): + print(f'Flavor {flavor.name}: DELETE {key}') + + +class ActionApply: + def __init__(self, compute): + self.compute = compute + + def set_extra_spec(self, flavor, key, value): + logger.info(f'Flavor {flavor.name}: SET {key}={value}') + try: + flavor.update_extra_specs_property(self.compute, key, value) + except openstack.exceptions.SDKException as exc: + logger.error(f"{exc!r} while setting {key}={value} for {flavor.name}") + + def del_extra_spec(self, flavor, key): + logger.info(f'Flavor {flavor.name}: DELETE {key}') + try: + flavor.delete_extra_specs_property(self.compute, key) + except openstack.exceptions.SDKException as exc: + logger.error(f"{exc!r} while deleting {key} for {flavor.name}") + + +class SetCommand: + def __init__(self, flavor, key, value): + self.flavor = flavor + self.key = key + self.value = value + + def apply(self, action): + action.set_extra_spec(self.flavor, self.key, self.value) + + +class DelCommand: + def __init__(self, flavor, key): + self.flavor = flavor + self.key = key + + def apply(self, action): + action.del_extra_spec(self.flavor, self.key) + + +def handle_commands(action, compute, commands): + if not commands: + return + if action in ('ask', 'report'): + action_report = ActionReport() + print(f'Proposing the following {len(commands)} changes to extra_specs:') + for command in commands: + command.apply(action_report) + if action == 'ask': + print('Do you want to apply these changes? y/n') + if input() == 'y': + action = 'apply' + else: + print('No changes will be applied.') + if action == 'apply': + action_apply = ActionApply(compute) + for command in commands: + command.apply(action_apply) + + +def main(argv): + action = "ask" # or "report" or "apply" + + errors = 0 + disk0_type = None + cpu_type = None + gen_all_names = False + + cloud = os.environ.get("OS_CLOUD") + try: + opts, flvs = getopt.gnu_getopt(argv, "hdqAt:p:c:a:", + ("help", "debug", "quiet", "all-names", + "disk0-type=", "cpu-type=", "os-cloud=", "action=")) + except getopt.GetoptError as exc: + logger.critical(repr(exc)) + usage() + return 1 + for opt in opts: + if opt[0] == "-h" or opt[0] == "--help": + usage(file=sys.stdout) + return 0 + if opt[0] == "-q" or opt[0] == "--quiet": + logging.getLogger().setLevel(logging.WARNING) + if opt[0] == "-d" or opt[0] == "--debug": + logging.getLogger().setLevel(logging.DEBUG) + if opt[0] == "-A" or opt[0] == "--all-names": + gen_all_names = True + if opt[0] == "-a" or opt[0] == "--action": + action = opt[1].strip().lower() + if opt[0] == "-c" or opt[0] == "--os-cloud": + cloud = opt[1] + if opt[0] == "-t" or opt[0] == "--disk0-type": + disk0_type = opt[1] + if disk0_type not in DISKTYPE_KEY: + disk0_type = revert_dict(disk0_type, DISKTYPE_KEY) + if not disk0_type: + return 2 + if opt[0] == "-p" or opt[0] == "--cpu-type": + cpu_type = opt[1] + if cpu_type not in CPUTYPE_KEY: + cpu_type = revert_dict(cpu_type, CPUTYPE_KEY) + if not cpu_type: + return 2 + + if action not in ('ask', 'report', 'apply'): + logger.error("action needs to be one of ask, report, apply") + usage() + return 4 + + if not cloud: + logger.error("Need to pass -c|--os-cloud|OS_CLOUD env") + usage() + return 3 + + conn = openstack.connect(cloud) + conn.authorize() + + # select relevant flavors: either given via name, or all SCS flavors + predicate = (lambda fn: fn in flvs) if flvs else (lambda fn: fn.startswith('SCS-')) + flavors = [flavor for flavor in conn.compute.flavors() if predicate(flavor.name)] + # This is likely a user error, so make them aware + if len(flavors) < len(flvs): + missing = set(flvs) - set(flavor.name for flavor in flavors) + logger.warning("Flavors not found: " + ", ".join(missing)) + + commands = [] + for flavor in flavors: + extra_names_to_check = [ + (key, value) + for key, value in flavor.extra_specs.items() + if SCS_NAME_PATTERN.match(key) + ] + names_to_check = [('name', flavor.name)] if flavor.name.startswith('SCS-') else [] + names_to_check.extend(extra_names_to_check) + + # syntax check: compute flavorname instances + # sanity check: claims must be true wrt actual flavor + flavornames = {} + for key, name_str in names_to_check: + try: + flavornames[key] = flavorname = parser_vN(name_str) + except ValueError as exc: + logger.error(f"could not parse {key}={name_str}: {exc!r}") + errors += 1 + else: + errors += check_std_props(flavor, flavorname, " by name") + + if not flavornames: + # we need cputype and disktype from user + if not cpu_type: + logger.warning(f"Need to specify cpu-type for generating name for {flavor.name}, skipping") + continue + if flavor.disk and not disk0_type: + logger.warning(f"Need to specify disk0-type for generating name for {flavor.name}, skipping") + continue + flavornames['_generated'] = generate_flavorname(flavor, cpu_type, disk0_type) + + expected = flavorname_to_dict(*flavornames.values(), ctx=flavor.name) + # determine invalid keys (within scs namespace) + # scs:name-vN is always permissible + removals = [ + key + for key in flavor.extra_specs + if key.startswith('scs:') and not SCS_NAME_PATTERN.match(key) + if expected.get(key, DEFAULTS.get(key)) is None + ] + logger.debug(f"Flavor {flavor.name}: expected={expected}, removals={removals}") + + for key in removals: + commands.append(DelCommand(flavor, key)) + + # generate or rectify extra_specs + for key, value in expected.items(): + if not key.startswith("scs:"): + continue + if not gen_all_names and key.startswith("scs:name-v") and extra_names_to_check: + continue # do not generate names if names are present + current = flavor.extra_specs.get(key) + if current == value: + continue + if current is None and DEFAULTS.get(key) == value: + continue + if current is not None: + logger.warning(f"{flavor.name}: resetting {key} because {current} != expected {value}") + commands.append(SetCommand(flavor, key, value)) + + handle_commands(action, conn.compute, commands) + logger.info(f"Processed {len(flavors)} flavors, {len(commands)} changes") + return errors + + +if __name__ == "__main__": + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + openstack.enable_logging(debug=False) + sys.exit(min(127, main(sys.argv[1:]))) # cap at 127 due to OS restrictions diff --git a/Tests/iaas/flavor-naming/flavor_names.py b/Tests/iaas/flavor-naming/flavor_names.py index d8573af38..08b6d11d1 100644 --- a/Tests/iaas/flavor-naming/flavor_names.py +++ b/Tests/iaas/flavor-naming/flavor_names.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +from collections import defaultdict +import logging import os import os.path import re @@ -9,8 +11,13 @@ import yaml +logger = logging.getLogger(__name__) + +SCS_NAME_PATTERN = re.compile(r"scs:name-v\d+\Z") CPUTYPE_KEY = {'L': 'crowded-core', 'V': 'shared-core', 'T': 'dedicated-thread', 'C': 'dedicated-core'} +CPUTYPE_SORT = {'crowded-core': 0, 'shared-core': 1, 'dedicated-thread': 2, 'dedicated-core': 3} DISKTYPE_KEY = {'n': 'network', 'h': 'hdd', 's': 'ssd', 'p': 'nvme'} +DISKTYPE_SORT = {'network': 0, 'hdd': 1, 'ssd': 2, 'nvme': 3} HERE = Path(__file__).parent @@ -437,6 +444,46 @@ def __call__(self, s: str, pos=0) -> Flavorname: return flavorname +class ParsingStrategy: + """ + Composite parser that accepts multiple versions of the syntax in different ways + + Follows the contract of class `Parser` + """ + + def __init__(self, vstr, parsers=(), tolerated_parsers=(), invalid_parsers=()): + self.vstr = vstr + self.parsers = parsers + self.tolerated_parsers = tolerated_parsers + self.invalid_parsers = invalid_parsers + + def __call__(self, namestr: str) -> Flavorname: + exc = None + for parser in self.parsers: + try: + return parser(namestr) + except Exception as e: + if exc is None: + exc = e + # at this point, if `self.parsers` is not empty, then `exc` is not `None` + for parser in self.tolerated_parsers: + try: + result = parser(namestr) + except Exception: + pass + else: + logger.warning(f"Name is merely tolerated {parser.vstr}: {namestr}") + return result + for parser in self.invalid_parsers: + try: + result = parser(namestr) + except Exception: + pass + else: + raise ValueError(f"Name is non-tolerable {parser.vstr}") + raise exc + + def _convert_user_input(idx, attr, target, val): """auxiliary function that converts user-input string `val` to the target attribute type""" fdesc = attr.name @@ -542,23 +589,49 @@ def __call__(self): parser_v1 = Parser("v1", SyntaxV1) parser_v2 = Parser("v2", SyntaxV2) parser_v3 = Parser("v3", SyntaxV2) # this is the same as parser_v2 except for the vstr +parser_vN = ParsingStrategy(vstr="vN", parsers=(parser_v2, parser_v1)) outname = outputter = Outputter() inputflavor = inputter = Inputter() -def flavorname_to_dict(flavorname: Flavorname) -> dict: - name_v2 = outputter(flavorname) +def flavorname_to_dict(*flavornames: Flavorname, ctx='') -> dict: + if not flavornames: + raise RuntimeError("need to supply at least one Flavorname instance!") + if ctx: + ctx = ctx + ': ' # used for logging warnings + name_collection = set() + collection = defaultdict(set) + for flavorname in flavornames: + collection['cpus'].add(flavorname.cpuram.cpus) + collection['ram'].add(flavorname.cpuram.ram) + collection['scs:cpu-type'].add(CPUTYPE_KEY[flavorname.cpuram.cputype]) + if flavorname.disk: + collection['disk'].add(flavorname.disk.disksize) + collection['nrdisks'].add(flavorname.disk.nrdisks) # this will need some postprocessing + collection['scs:disk0-type'].add(DISKTYPE_KEY[flavorname.disk.disktype or 'n']) + name_v2 = outputter(flavorname) + name_collection.add((SyntaxV1.from_v2(name_v2), "v1")) + name_collection.add((name_v2, "v2")) + short_v2 = outputter(flavorname.shorten()) + # could check whether short_v2 != name_v2, but the set will swallow everything + name_collection.add((SyntaxV1.from_v2(short_v2), "v1")) + name_collection.add((short_v2, "v2")) + for key, values in collection.items(): + if len(values) > 1: + logger.warning(f"{ctx}Inconsistent {key}: {', '.join(values)}") result = { - 'cpus': flavorname.cpuram.cpus, - 'cpu-type': CPUTYPE_KEY[flavorname.cpuram.cputype], - 'ram': flavorname.cpuram.ram, - 'name-v1': SyntaxV1.from_v2(name_v2), - 'name-v2': name_v2, + 'cpus': max(collection['cpus']), + 'scs:cpu-type': max(collection['scs:cpu-type'], key=CPUTYPE_SORT.__getitem__), + 'ram': max(collection['ram']), } - if flavorname.disk: - result['disk'] = flavorname.disk.disksize - for i in range(flavorname.disk.nrdisks): - result[f'disk{i}-type'] = DISKTYPE_KEY[flavorname.disk.disktype or 'n'] + if collection['nrdisks']: + result['disk'] = max(collection['disk']) + disktype = max(collection['scs:disk0-type'], key=DISKTYPE_SORT.__getitem__) + for i in range(max(collection['nrdisks'])): + result[f'scs:disk{i}-type'] = disktype + names = [item[0] for item in sorted(name_collection, key=lambda item: (-len(item[0]), item[1]))] + for idx, name in enumerate(names): + result[f'scs:name-v{idx + 1}'] = name return result diff --git a/Tests/iaas/scs-0103-v1-flavors.yaml b/Tests/iaas/scs-0103-v1-flavors.yaml index 9d23b8527..66c02cdd6 100644 --- a/Tests/iaas/scs-0103-v1-flavors.yaml +++ b/Tests/iaas/scs-0103-v1-flavors.yaml @@ -1,194 +1,194 @@ meta: - name_key: name-v2 + name_key: "scs:name-v2" flavor_groups: - status: mandatory list: - name: SCS-1V-4 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 - name-v1: SCS-1V:4 - name-v2: SCS-1V-4 + "scs:name-v1": SCS-1V:4 + "scs:name-v2": SCS-1V-4 - name: SCS-2V-8 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-2V:8 - name-v2: SCS-2V-8 + "scs:name-v1": SCS-2V:8 + "scs:name-v2": SCS-2V-8 - name: SCS-4V-16 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-4V:16 - name-v2: SCS-4V-16 + "scs:name-v1": SCS-4V:16 + "scs:name-v2": SCS-4V-16 - name: SCS-8V-32 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-8V:32 - name-v2: SCS-8V-32 + "scs:name-v1": SCS-8V:32 + "scs:name-v2": SCS-8V-32 - name: SCS-1V-2 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 - name-v1: SCS-1V:2 - name-v2: SCS-1V-2 + "scs:name-v1": SCS-1V:2 + "scs:name-v2": SCS-1V-2 - name: SCS-2V-4 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 - name-v1: SCS-2V:4 - name-v2: SCS-2V-4 + "scs:name-v1": SCS-2V:4 + "scs:name-v2": SCS-2V-4 - name: SCS-4V-8 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-4V:8 - name-v2: SCS-4V-8 + "scs:name-v1": SCS-4V:8 + "scs:name-v2": SCS-4V-8 - name: SCS-8V-16 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-8V:16 - name-v2: SCS-8V-16 + "scs:name-v1": SCS-8V:16 + "scs:name-v2": SCS-8V-16 - name: SCS-16V-32 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-16V:32 - name-v2: SCS-16V-32 + "scs:name-v1": SCS-16V:32 + "scs:name-v2": SCS-16V-32 - name: SCS-1V-8 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-1V:8 - name-v2: SCS-1V-8 + "scs:name-v1": SCS-1V:8 + "scs:name-v2": SCS-1V-8 - name: SCS-2V-16 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-2V:16 - name-v2: SCS-2V-16 + "scs:name-v1": SCS-2V:16 + "scs:name-v2": SCS-2V-16 - name: SCS-4V-32 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-4V:32 - name-v2: SCS-4V-32 + "scs:name-v1": SCS-4V:32 + "scs:name-v2": SCS-4V-32 - name: SCS-1L-1 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 - name-v1: SCS-1L:1 - name-v2: SCS-1L-1 + "scs:name-v1": SCS-1L:1 + "scs:name-v2": SCS-1L-1 - status: mandatory list: - name: SCS-2V-4-20s cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 20 - disk0-type: ssd - name-v1: SCS-2V:4:20s - name-v2: SCS-2V-4-20s + "scs:disk0-type": ssd + "scs:name-v1": SCS-2V:4:20s + "scs:name-v2": SCS-2V-4-20s - name: SCS-4V-16-100s cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 100 - disk0-type: ssd - name-v1: SCS-4V:16:100s - name-v2: SCS-4V-16-100s + "scs:disk0-type": ssd + "scs:name-v1": SCS-4V:16:100s + "scs:name-v2": SCS-4V-16-100s - status: recommended list: - name: SCS-1V-4-10 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-1V:4:10 - name-v2: SCS-1V-4-10 + "scs:name-v1": SCS-1V:4:10 + "scs:name-v2": SCS-1V-4-10 - name: SCS-2V-8-20 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-2V:8:20 - name-v2: SCS-2V-8-20 + "scs:name-v1": SCS-2V:8:20 + "scs:name-v2": SCS-2V-8-20 - name: SCS-4V-16-50 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-4V:16:50 - name-v2: SCS-4V-16-50 + "scs:name-v1": SCS-4V:16:50 + "scs:name-v2": SCS-4V-16-50 - name: SCS-8V-32-100 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-8V:32:100 - name-v2: SCS-8V-32-100 + "scs:name-v1": SCS-8V:32:100 + "scs:name-v2": SCS-8V-32-100 - name: SCS-1V-2-5 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 disk: 5 - name-v1: SCS-1V:2:5 - name-v2: SCS-1V-2-5 + "scs:name-v1": SCS-1V:2:5 + "scs:name-v2": SCS-1V-2-5 - name: SCS-2V-4-10 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-2V:4:10 - name-v2: SCS-2V-4-10 + "scs:name-v1": SCS-2V:4:10 + "scs:name-v2": SCS-2V-4-10 - name: SCS-4V-8-20 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-4V:8:20 - name-v2: SCS-4V-8-20 + "scs:name-v1": SCS-4V:8:20 + "scs:name-v2": SCS-4V-8-20 - name: SCS-8V-16-50 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-8V:16:50 - name-v2: SCS-8V-16-50 + "scs:name-v1": SCS-8V:16:50 + "scs:name-v2": SCS-8V-16-50 - name: SCS-16V-32-100 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-16V:32:100 - name-v2: SCS-16V-32-100 + "scs:name-v1": SCS-16V:32:100 + "scs:name-v2": SCS-16V-32-100 - name: SCS-1V-8-20 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-1V:8:20 - name-v2: SCS-1V-8-20 + "scs:name-v1": SCS-1V:8:20 + "scs:name-v2": SCS-1V-8-20 - name: SCS-2V-16-50 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-2V:16:50 - name-v2: SCS-2V-16-50 + "scs:name-v1": SCS-2V:16:50 + "scs:name-v2": SCS-2V-16-50 - name: SCS-4V-32-100 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-4V:32:100 - name-v2: SCS-4V-32-100 + "scs:name-v1": SCS-4V:32:100 + "scs:name-v2": SCS-4V-32-100 - name: SCS-1L-1-5 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 disk: 5 - name-v1: SCS-1L:1:5 - name-v2: SCS-1L-1-5 + "scs:name-v1": SCS-1L:1:5 + "scs:name-v2": SCS-1L-1-5 diff --git a/Tests/iaas/standard-flavors/flavors-openstack.py b/Tests/iaas/standard-flavors/flavors-openstack.py index 61d35fb36..2680fa822 100755 --- a/Tests/iaas/standard-flavors/flavors-openstack.py +++ b/Tests/iaas/standard-flavors/flavors-openstack.py @@ -105,7 +105,6 @@ def main(argv): logger.critical("Flavor definition missing 'flavor_groups' field") name_key = flavor_spec_data['meta']['name_key'] - es_name_key = f"scs:{name_key}" # compute union of all flavor groups, copying group info (mainly "status") to each flavor # check if the spec is complete while we are at it flavor_specs = [] @@ -128,9 +127,9 @@ def main(argv): with openstack.connect(cloud=cloud, timeout=32) as conn: present_flavors = conn.list_flavors(get_extra=True) by_name = { - flavor.extra_specs[es_name_key]: flavor + flavor.extra_specs[name_key]: flavor for flavor in present_flavors - if es_name_key in flavor.extra_specs + if name_key in flavor.extra_specs } by_legacy_name = {flavor.name: flavor for flavor in present_flavors} # for reserved keys, keep track of all flavors that don't have a matching spec @@ -145,7 +144,7 @@ def main(argv): if not flavor: flavor = by_legacy_name.get(flavor_spec[name_key]) if flavor: - logger.warning(f"Flavor '{flavor_spec['name']}' found via name only, missing property {es_name_key!r}") + logger.warning(f"Flavor '{flavor_spec['name']}' found via name only, missing property {name_key!r}") else: status = flavor_spec['_group']['status'] level = {"mandatory": logging.ERROR}.get(status, logging.WARNING) @@ -165,9 +164,9 @@ def main(argv): report = [ f"{key}: {es_value!r} should be {value!r}" for key, value, es_value in [ - (key, value, flavor.extra_specs.get(f"scs:{key}")) + (key, value, flavor.extra_specs.get(key)) for key, value in flavor_spec.items() - if key not in ('_group', 'name', 'cpus', 'ram', 'disk') + if key.startswith("scs:") ] if value != es_value ] diff --git a/Tests/testing/scs-0103-v1-flavors-wrong.yaml b/Tests/testing/scs-0103-v1-flavors-wrong.yaml index 60d1c7bc2..7aff89a0f 100644 --- a/Tests/testing/scs-0103-v1-flavors-wrong.yaml +++ b/Tests/testing/scs-0103-v1-flavors-wrong.yaml @@ -5,191 +5,191 @@ flavor_groups: list: - name: SCS-1V-4 cpus: 1 - cpu-type: crowded-core # wrong: name suggests shared-core + "scs:cpu-type": crowded-core # wrong: name suggests shared-core ram: 4 - name-v1: SCS-1V:4 - name-v2: SCS-1V-4 + "scs:name-v1": SCS-1V:4 + "scs:name-v2": SCS-1V-4 - name: SCS-2V-8 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-2V-8 # wrong: not a v1 name - name-v2: SCS-2V-8 + "scs:name-v1": SCS-2V-8 # wrong: not a v1 name + "scs:name-v2": SCS-2V-8 - name: SCS-4V-16 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 12 # wrong: name suggests 16 - name-v1: SCS-4V:16 - name-v2: SCS-4V-16 + "scs:name-v1": SCS-4V:16 + "scs:name-v2": SCS-4V-16 - name: SCS-8V-32 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 128 # wrong: no disk in name - name-v1: SCS-8V:32 - name-v2: SCS-8V-32 + "scs:name-v1": SCS-8V:32 + "scs:name-v2": SCS-8V-32 - name: SCS-1V-2 cpus: 2 # wrong: name suggests 1 cpu - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 - name-v1: SCS-1V:2 - name-v2: SCS-1V-2 + "scs:name-v1": SCS-1V:2 + "scs:name-v2": SCS-1V-2 - name: SCS-2V-4 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 - name-v1: SCS-2V:4 - name-v2: SCS-2V-4 + "scs:name-v1": SCS-2V:4 + "scs:name-v2": SCS-2V-4 - name: SCS-4V-8 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-4V:8 - name-v2: SCS-4V-8 + "scs:name-v1": SCS-4V:8 + "scs:name-v2": SCS-4V-8 - name: SCS-8V-16 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-8V:16 - name-v2: SCS-8V-16 + "scs:name-v1": SCS-8V:16 + "scs:name-v2": SCS-8V-16 - name: SCS-16V-32 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-16V:32 - name-v2: SCS-16V-32 + "scs:name-v1": SCS-16V:32 + "scs:name-v2": SCS-16V-32 - name: SCS-1V-8 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 - name-v1: SCS-1V:8 - name-v2: SCS-1V-8 + "scs:name-v1": SCS-1V:8 + "scs:name-v2": SCS-1V-8 - name: SCS-2V-16 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 - name-v1: SCS-2V:16 - name-v2: SCS-2V-16 + "scs:name-v1": SCS-2V:16 + "scs:name-v2": SCS-2V-16 - name: SCS-4V-32 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 - name-v1: SCS-4V:32 - name-v2: SCS-4V-32 + "scs:name-v1": SCS-4V:32 + "scs:name-v2": SCS-4V-32 - name: SCS-1L-1 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 - name-v1: SCS-1L:1 - name-v2: SCS-1L-1 + "scs:name-v1": SCS-1L:1 + "scs:name-v2": SCS-1L-1 - status: mandatory list: - name: SCS-2V-4-20s cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 20 # wrong: name suggests disk-type ssd - name-v1: SCS-2V:4:20s - name-v2: SCS-2V-4-20s + "scs:name-v1": SCS-2V:4:20s + "scs:name-v2": SCS-2V-4-20s - name: SCS-4V-16-100s cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 10 # wrong: name suggests 100 - disk0-type: ssd - name-v1: SCS-4V:16:100s - name-v2: SCS-4V-16-100s + "scs:disk0-type": ssd + "scs:name-v1": SCS-4V:16:100s + "scs:name-v2": SCS-4V-16-100s - status: recommended list: - name: SCS-1V-4-10 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-1V:4:10 - name-v2: SCS-1V-4-10 + "scs:name-v1": SCS-1V:4:10 + "scs:name-v2": SCS-1V-4-10 - name: SCS-2V-8-20 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-2V:8:20 - name-v2: SCS-2V-8-20 + "scs:name-v1": SCS-2V:8:20 + "scs:name-v2": SCS-2V-8-20 - name: SCS-4V-16-50 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-4V:16:50 - name-v2: SCS-4V-16-50 + "scs:name-v1": SCS-4V:16:50 + "scs:name-v2": SCS-4V-16-50 - name: SCS-8V-32-100 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-8V:32:100 - name-v2: SCS-8V-32-100 + "scs:name-v1": SCS-8V:32:100 + "scs:name-v2": SCS-8V-32-100 - name: SCS-1V-2-5 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 2 disk: 5 - name-v1: SCS-1V:2:5 - name-v2: SCS-1V-2-5 + "scs:name-v1": SCS-1V:2:5 + "scs:name-v2": SCS-1V-2-5 - name: SCS-2V-4-10 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 4 disk: 10 - name-v1: SCS-2V:4:10 - name-v2: SCS-2V-4-10 + "scs:name-v1": SCS-2V:4:10 + "scs:name-v2": SCS-2V-4-10 - name: SCS-4V-8-20 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-4V:8:20 - name-v2: SCS-4V-8-20 + "scs:name-v1": SCS-4V:8:20 + "scs:name-v2": SCS-4V-8-20 - name: SCS-8V-16-50 cpus: 8 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-8V:16:50 - name-v2: SCS-8V-16-50 + "scs:name-v1": SCS-8V:16:50 + "scs:name-v2": SCS-8V-16-50 - name: SCS-16V-32-100 cpus: 16 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-16V:32:100 - name-v2: SCS-16V-32-100 + "scs:name-v1": SCS-16V:32:100 + "scs:name-v2": SCS-16V-32-100 - name: SCS-1V-8-20 cpus: 1 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 8 disk: 20 - name-v1: SCS-1V:8:20 - name-v2: SCS-1V-8-20 + "scs:name-v1": SCS-1V:8:20 + "scs:name-v2": SCS-1V-8-20 - name: SCS-2V-16-50 cpus: 2 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 16 disk: 50 - name-v1: SCS-2V:16:50 - name-v2: SCS-2V-16-50 + "scs:name-v1": SCS-2V:16:50 + "scs:name-v2": SCS-2V-16-50 - name: SCS-4V-32-100 cpus: 4 - cpu-type: shared-core + "scs:cpu-type": shared-core ram: 32 disk: 100 - name-v1: SCS-4V:32:100 - name-v2: SCS-4V-32-100 + "scs:name-v1": SCS-4V:32:100 + "scs:name-v2": SCS-4V-32-100 - name: SCS-1L-1-5 cpus: 1 - cpu-type: crowded-core + "scs:cpu-type": crowded-core ram: 1 disk: 5 - name-v1: SCS-1L:1:5 - name-v2: SCS-1L-1-5 + "scs:name-v1": SCS-1L:1:5 + "scs:name-v2": SCS-1L-1-5