Refactor LandscapeMembers, LandscapeOutput, and Members

.gitignore

@@ -197,7 +197,7 @@ Temporary Items
 /*.yml
 /*.csv
 /*.txt
-http_cache.sqlite
+*.sqlite
 !action.yml
 
 # add back in for packaging

action.yml

@@ -25,6 +25,7 @@ runs:
         token: ${{ env.token }}
         repository: jmertic/lfx-landscape-tools
         path: landscape-tools
+        ref: landscapemembers
     - name: Set up Python 3.x
       uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
       with:
@@ -58,11 +59,11 @@ runs:
       env:
         GITHUB_TOKEN: ${{ env.token }}
         GH_TOKEN: ${{ env.token }}
-    - name: Save missing.csv file
+    - name: Save debug.log file
       uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
       with:
-        name: missing-members 
-        path: ./landscape/missing.csv
+        name: debug.log 
+        path: ./landscape/debug.log
     - name: Validate landscape2 data
       uses: cncf/landscape2-validate-action@7f299c46e9b03b4e8bc2896882734fb0b0756b37 # v2.0.0
       with:

lfx_landscape_tools/cli.py

@@ -31,7 +31,8 @@ def __init__(self):
         parser = ArgumentParser("Collection of tools for working with a landscape")
         group = parser.add_mutually_exclusive_group()
         group.add_argument("-s", "--silent", dest="silent", action="store_true", help="Suppress all messages")
-        group.add_argument("-v", "--verbose", dest="verbose", action='store_true', help="Verbose output ( i.e. show all INFO level messages in addition to WARN and above )")
+        group.add_argument("-l", "--log", dest="loglevel", default="error", choices=['debug', 'info', 'warning', 'error', 'critical'], help="logging level")
+        group.add_argument("-v", "--verbose", dest="verbose", action='store_true', help="Verbose output (i.e. show all INFO level messages in addition to WARN and above - equivalent to `--log info`)")
         subparsers = parser.add_subparsers(help='sub-command help')
 
         buildlandscapemembers_parser = subparsers.add_parser("build_members", help="Replace current items with latest from LFX")
@@ -50,25 +51,36 @@ def __init__(self):
         synclandscapeprojects_parser.set_defaults(func=self.syncprojects)
 
         maketextlogo_parser = subparsers.add_parser("maketextlogo", help="Create a text pure SVG logo")
-        maketextlogo_parser.add_argument("-n", "--name", dest="orgname", required=True, help="Name to appear in logo")
+        maketextlogo_parser.add_argument("-n", "--name", dest="name", required=True, help="Name to appear in logo")
         maketextlogo_parser.add_argument("--autocrop", dest="autocrop", action='store_true', help="Process logo with autocrop")
         maketextlogo_parser.add_argument("-o", "--output", dest="filename", help="Filename to save created logo to")
         maketextlogo_parser.set_defaults(func=self.maketextlogo)
 
         args = parser.parse_args()
 
+        levels = {
+            'critical': logging.CRITICAL,   # errors that mean an immediate stop
+            'error': logging.ERROR,         # general errors that will effect the output
+            'warn': logging.WARNING,        # errors that can be caught and corrected
+            'warning': logging.WARNING,
+            'info': logging.INFO,           # infomational messages
+            'debug': logging.DEBUG          # messages to help debug things misbehaving ;-)
+        }
+        if args.verbose:
+            args.loglevel = 'info'
         logging.basicConfig(
-            level=logging.INFO if args.verbose else logging.WARN,
+            level=levels.get(args.loglevel.lower()),
             format="%(asctime)s [%(levelname)s] %(message)s",
             handlers=[
-                logging.FileHandler("debug.log"),
+                logging.FileHandler("debug.log",mode="w"),
                 logging.StreamHandler(sys.stdout) if not args.silent else None
             ]
         )
 
         try:
             args.func(args)
-        except AttributeError:
+        except AttributeError as e:
+            logging.getLogger().debug(e)
             parser.print_help()
 
         logging.getLogger().info("This took {} seconds".format(datetime.now() - self._starttime))
@@ -85,35 +97,41 @@ def _dir_path(self,path):
 
     def buildmembers(self,args):
         config = Config(args.configfile,view='members')
-        landscapeoutput = LandscapeOutput(config, resetCategory=True)
-        logging.getLogger().info("Adding LFX Members data")
-        landscapeoutput.addItems(LFXMembers(config=config))
+        landscapeoutput = LandscapeOutput(config=config)
+        landscapeoutput.load(members=LFXMembers(config=config))
         landscapeoutput.save()
 
-        logging.getLogger().info("Successfully added {} members and skipped {} members".format(landscapeoutput.itemsAdded,landscapeoutput.itemsErrors))
+        logging.getLogger().info("Successfully processed {} members and skipped {} members".format(landscapeoutput.itemsProcessed,landscapeoutput.itemsErrors))
 
     def buildprojects(self,args):
         config = Config(args.configfile,view='projects')
-        landscapeoutput = LandscapeOutput(config, resetCategory=True)
-        logging.getLogger().info("Adding LFX Projects data")
-        landscapeoutput.addItems(LFXProjects(config=config))
+        landscapeoutput = LandscapeOutput(config=config)
+        landscapeoutput.load(members=LFXProjects(config=config))
         landscapeoutput.save()
 
-        logging.getLogger().info("Successfully added {} projects and skipped {} projects".format(landscapeoutput.itemsAdded,landscapeoutput.itemsErrors))
+        logging.getLogger().info("Successfully processed {} projects and skipped {} projects".format(landscapeoutput.itemsProcessed,landscapeoutput.itemsErrors))
 
     def syncprojects(self,args):
         config = Config(args.configfile,view='projects')
-        landscapeoutput = LandscapeOutput(config=config, resetCategory=False)
-        logging.getLogger().info("Syncing TAC Agenda Project data")
-        landscapeoutput.syncItems(TACAgendaProject(config=config))
-        logging.getLogger().info("Syncing LFX Projects data")
-        landscapeoutput.syncItems(LFXProjects(config=config)) 
+        items = LFXProjects(config=config)
+        logging.getLogger().info("Overlaying current Landscape data")
+        items.overlay(memberstooverlay=LandscapeMembers(config=config))
+        logging.getLogger().info("Overlaying TAC Agenda Project data")
+        items.overlay(memberstooverlay=TACAgendaProject(config=config))
+        # yes, this is intentional :). This ensures the LFX data is the predominate source of truth
+        logging.getLogger().info("Overlaying LFX Projects data")
+        items.overlay(memberstooverlay=LFXProjects(config=config))
+        # also intentional, to overlay extra field dates
+        logging.getLogger().info("Overlaying TAC Agenda Project data 'extra' field")
+        items.overlay(memberstooverlay=TACAgendaProject(config=config),onlykeys=['extra'])
+        landscapeoutput = LandscapeOutput(config=config)
+        landscapeoutput.load(members=items)
         landscapeoutput.save()
 
-        logging.getLogger().info("Successfully added {} projects, updated {} projects, and skipped {} projects".format(landscapeoutput.itemsAdded,landscapeoutput.itemsUpdated,landscapeoutput.itemsErrors))
+        logging.getLogger().info("Successfully processed {} projects and skipped {} projects".format(landscapeoutput.itemsProcessed,landscapeoutput.itemsErrors))
 
     def maketextlogo(self,args):
-        svglogo = SVGLogo(name=args.orgname)
+        svglogo = SVGLogo(name=args.name)
 
         if args.autocrop:
             svglogo.autocrop()

lfx_landscape_tools/config.py

@@ -39,7 +39,7 @@ class Config:
     landscapefile = 'landscape.yml'
     missingcsvfile = 'missing.csv'
     hostedLogosDir = 'hosted_logos'
-    memberSuffix = None
+    memberSuffix = ''
     memberUsePublicMembershipLogo = False
     projectsAddTechnologySector = False
     projectsAddIndustrySector = False
@@ -61,8 +61,8 @@ def __init__(self, config_file: io.TextIOWrapper = None, view = None):
             if not self.slug or not self.project:
                 raise ValueError("Invalid project specification in config file")
             self.landscapeProjectsCategory = data_loaded.get('landscapeProjectsCategory',Config.landscapeProjectsCategory)
-            self.landscapeProjectsSubcategories = data_loaded.get('landscapeProjectsSubcategories',Config.landscapeProjectsSubcategories)
             self.landscapeProjectsLevels = data_loaded.get('landscapeProjectsLevels',Config.landscapeProjectsLevels)
+            self.landscapeProjectsSubcategories = data_loaded.get('landscapeProjectsSubcategories',self._getlandscapeProjectsSubcategoriesFromLevels())
             self.landscapeMembersCategory = data_loaded.get('landscapeMembersCategory',Config.landscapeMembersCategory)
             self.landscapeMembersCategory = data_loaded.get('landscapeMemberCategory',Config.landscapeMembersCategory)
             self.landscapeMembersSubcategories = data_loaded.get('landscapeMembersSubcategories',Config.landscapeMembersSubcategories)
@@ -125,4 +125,6 @@ def _lookupSlugFromProject(self,project):
 
         return None
 
-
+    def _getlandscapeProjectsSubcategoriesFromLevels(self):
+        for level in self.landscapeProjectsLevels:
+            self.landscapeProjectsSubcategories.append({'name':level['name'],'category':level['name']})

lfx_landscape_tools/landscapemembers.py

@@ -7,6 +7,9 @@
 
 import logging
 from contextlib import suppress
+import os
+import re
+from urllib.parse import urlparse
 
 ## third party modules
 import ruamel.yaml
@@ -18,71 +21,57 @@
 
 class LandscapeMembers(Members):
 
-    landscapeListYAML = 'https://raw.githubusercontent.com/cncf/landscapeapp/master/landscapes.yml'
-    landscapeSettingsYAML = 'https://raw.githubusercontent.com/{repo}/master/settings.yml'
-    landscapeLandscapeYAML = 'https://raw.githubusercontent.com/{repo}/master/landscape.yml'
-    landscapeLogo = 'https://raw.githubusercontent.com/{repo}/master/hosted_logos/{logo}'
-    skipLandscapes = ['openjsf']
-
-    def __init__(self, landscapeListYAML = None, loadData = True):
-        if landscapeListYAML:
-            self.landscapeListYAML = landscapeListYAML
-        super().__init__(loadData=loadData,config=Config())
+    landscapeCategory = 'Members'
+    landscapeSubcategories = [
+        {"name": "Premier Membership", "category": "Premier"},
+        {"name": "General Membership", "category": "General"},
+        {"name": "Associate Membership", "category": "Associate"}
+    ]
+    landscapefile = 'landscape.yml'
+    memberSuffix = ''
 
     def processConfig(self, config: type[Config]):
-        return
+        self.landscapeCategory = config.landscapeCategory
+        self.landscapeSubcategories = config.landscapeSubcategories
+        self.landscapefile = os.path.join(config.basedir,config.landscapefile)
+        self.memberSuffix = config.memberSuffix if config.view == 'members' else self.memberSuffix
+        self.hostedLogosDir = os.path.join(config.basedir,config.hostedLogosDir)
 
     def loadData(self):
         logger = logging.getLogger()
-        logger.info("Loading other landscape members data")
-
-        response = requests.get(self.landscapeListYAML)
-        landscapeList = ruamel.yaml.YAML().load(response.content)
-
-        for landscape in landscapeList['landscapes']:
-            if landscape['name'] in self.skipLandscapes:
-                continue
-
-            logger.info("Loading {}...".format(landscape['name']))
-
-            # first figure out where memberships live
-            response = requests.get(self.landscapeSettingsYAML.format(repo=landscape['repo']))
-            with suppress(Exception):
-                settingsYaml = ruamel.yaml.YAML().load(response.content) 
-            # skip landscape if not well formed
-            if 'global' not in settingsYaml or settingsYaml['global'] is None or 'membership' not in settingsYaml['global']:
-                continue
-            membershipKey = settingsYaml['global']['membership']
+        logger.info("Loading Current Landscape members in category '{}'".format(self.landscapeCategory))
+        landscape = {}
 
-            # then load in members only
-            response = requests.get(self.landscapeLandscapeYAML.format(repo=landscape['repo']))
-            try:
-                landscapeYaml = ruamel.yaml.YAML().load(response.content)
-            except:
-                continue
-            for category in landscapeYaml['landscape']:
-                if membershipKey in category['name']:
-                    for subcategory in category['subcategories']:
+        try:
+            with open(self.landscapefile, 'r', encoding="utf8", errors='ignore') as fileobject: 
+                logging.getLogger().debug("Successfully opened landscape file '{}'".format(self.landscapefile))
+                landscape = ruamel.yaml.YAML().load(fileobject)
+                logging.getLogger().debug("Successfully parsed yaml output in landscape file '{}'".format(self.landscapefile))
+        except Exception as e:
+            logging.getLogger().error("Error opening landscape file '{}' - will not load current landscape data - '{}'".format(self.landscapefile,e))
+        else:
+            for x in landscape.get('landscape',{}):
+                if x.get('name') == self.landscapeCategory:
+                    for subcategory in x.get('subcategories'):
+                        logger.debug("Processing subcategory '{}'...".format(subcategory['name']))
                         for item in subcategory['items']:
                             member = Member()
+                            member.name = re.sub('{}$'.format(re.escape(self.memberSuffix)),'',item.get('name'))
+                            if item.get('logo'):
+                                if urlparse(item.get('logo')).scheme == '':
+                                    member.logo = os.path.normpath("{}/{}".format(self.hostedLogosDir,item.get('logo')))
+                                else:
+                                    member.logo = item.get('logo')
+                            logger.info("Found Landscape Member '{}'".format(member.name))
                             for key, value in item.items():
-                                with suppress(ValueError):
-                                    if key != 'enduser':
-                                        setattr(member, key, value)
-                            member.orgname = item.get('name')
-                            member.membership = ''
-                            with suppress(ValueError):
-                                member.website = item.get('homepage_url')
-                                member.logo = self.normalizeLogo(item.get('logo'),landscape.get('repo'))
-                                member.crunchbase = item.get('crunchbase')
+                                if key not in ['item','name','homepage_url','logo']:
+                                    logger.debug("Setting '{}' to '{}' for '{}'".format(key,value,member.name))
+                                    setattr(member, key, value)
+                            for landscapeSubcategory in self.landscapeSubcategories:
+                                if subcategory.get('name') == landscapeSubcategory.get('category'):
+                                    logger.debug("Parsing subcategory '{}' to landscapeSubcategory '{}'".format(subcategory.get('name'),landscapeSubcategory.get('name')))
+                                    member.membership = landscapeSubcategory.get('name')
+                                    break
+                            member.homepage_url = item.get('homepage_url')
+                            member.linkedin = item.get('extra',{}).get('linkedin_url')
                             self.members.append(member)
-
-    def normalizeLogo(self, logo, landscapeRepo):
-        if logo is None or logo == '':
-            return ""
-
-        if 'https://' in logo or 'http://' in logo:
-            return logo
-
-        return self.landscapeLogo.format(repo=landscapeRepo,logo=logo)
-