do not collect metadata stuff on client side

insights/client/core_collector.py

@@ -3,114 +3,24 @@
 """
 from __future__ import absolute_import
 
-import json
 import logging
-import os
-
-from itertools import chain
 
 from insights import collect
 from insights.client.archive import InsightsArchive
 from insights.client.constants import InsightsConstants as constants
-from insights.client.utilities import systemd_notify_init_thread, get_version_info, get_tags
-from insights.core.blacklist import BLACKLISTED_SPECS
+from insights.client.utilities import systemd_notify_init_thread
 
 APP_NAME = constants.app_name
 logger = logging.getLogger(__name__)
 
 
 class CoreCollector(object):
     """
-    Collectoer for new core-collector
+    Collector for core collection
     """
-    def __init__(self, config, archive_=None, mountpoint=None, spec_conf=None):
+    def __init__(self, config):
         self.config = config
-        self.archive = archive_ if archive_ else InsightsArchive(config)
-        self.mountpoint = mountpoint if mountpoint else '/'
-        self.spec_conf = spec_conf if spec_conf else {}
-
-    def _write_branch_info(self, branch_info):
-        logger.debug("Writing branch information to archive...")
-        self.archive.add_metadata_to_archive(
-            json.dumps(branch_info), '/branch_info')
-
-    def _write_display_name(self):
-        if self.config.display_name:
-            logger.debug("Writing display_name to archive...")
-            self.archive.add_metadata_to_archive(
-                self.config.display_name, '/display_name')
-
-    def _write_ansible_host(self):
-        if self.config.ansible_host:
-            logger.debug("Writing ansible_host to archive...")
-            self.archive.add_metadata_to_archive(
-                self.config.ansible_host, '/ansible_host')
-
-    def _write_version_info(self):
-        logger.debug("Writing version information to archive...")
-        version_info = get_version_info()
-        self.archive.add_metadata_to_archive(
-            json.dumps(version_info), '/version_info')
-
-    def _write_tags(self):
-        tags = get_tags()
-        # NOTE:
-        # The following code is also used by datasource 'tags'
-        # - insights.specs.datasources.tags
-        # Please keep them consistence before removing this.
-        if tags is not None:
-            def f(k, v):
-                if type(v) is list:
-                    col = []
-                    for val in v:
-                        col.append(f(k, val))
-                    return list(chain.from_iterable(col))
-                elif type(v) is dict:
-                    col = []
-                    for key, val in v.items():
-                        col.append(f(k + ":" + key, val))
-                    return list(chain.from_iterable(col))
-                else:
-                    return [{"key": k, "value": v, "namespace": constants.app_name}]
-            logger.debug("Writing tags to archive...")
-            t = []
-            for k, v in tags.items():
-                iv = f(k, v)
-                t.append(iv)
-            t = list(chain.from_iterable(t))
-            self.archive.add_metadata_to_archive(json.dumps(t), '/tags.json')
-
-    def _write_blacklist_report(self, blacklist_report):
-        logger.debug("Writing blacklist report to archive...")
-        self.archive.add_metadata_to_archive(
-            json.dumps(blacklist_report), '/blacklist_report')
-
-    def _write_blacklisted_specs(self):
-        if BLACKLISTED_SPECS:
-            logger.debug("Writing blacklisted specs to archive...")
-            self.archive.add_metadata_to_archive(
-                json.dumps({"specs": BLACKLISTED_SPECS}), '/blacklisted_specs')
-
-    def _write_egg_release(self):
-        logger.debug("Writing egg release to archive...")
-        egg_release = ''
-        try:
-            with open(constants.egg_release_file) as fil:
-                egg_release = fil.read()
-        except (IOError, MemoryError) as e:
-            logger.debug('Could not read the egg release file: %s', str(e))
-        try:
-            os.remove(constants.egg_release_file)
-        except OSError as e:
-            logger.debug('Could not remove the egg release file: %s', str(e))
-
-        try:
-            self.archive.add_metadata_to_archive(
-                egg_release, '/egg_release')
-        except OSError as e:
-            logger.debug('Could not add the egg release file to the archive: %s', str(e))
-            self.archive.add_metadata_to_archive(
-                '', '/egg_release')
+        self.archive = InsightsArchive(config)
 
     def run_collection(self, rm_conf, branch_info, blacklist_report):
         '''
@@ -134,17 +44,21 @@ def run_collection(self, rm_conf, branch_info, blacklist_report):
 
         logger.debug('Core collection finished.')
 
-        # collect metadata
-        logger.debug('Collecting metadata ...')
-        self._write_branch_info(branch_info)
-        self._write_display_name()
-        self._write_ansible_host()
-        self._write_version_info()
-        self._write_tags()
-        self._write_blacklist_report(blacklist_report)
-        self._write_blacklisted_specs()
-        self._write_egg_release()
-        logger.debug('Metadata collection finished.')
+        # About "metadata":
+        #  The following metadata is now collected by core collection when
+        #  available, and will be stored under the "./data" directory instead
+        #  of the root of the archive.
+        #
+        #   ansible_host = client_metadata.ansible_host
+        #   blacklist_report = client_metadata.blacklist_report
+        #   blacklisted_specs = client_metadata.blacklisted_specs
+        #   branch_info = client_metadata.branch_info
+        #   display_name = client_metadata.display_name
+        #   egg_release = client_metadata.egg_release
+        #   tags = client_metadata.tags
+        #   version_info = client_metadata.version_info
+        #
+        #  See `insights.specs.datasources.client_metadata`
 
     def done(self):
         """

insights/specs/datasources/client_metadata.py

@@ -1,6 +1,7 @@
 """
 Custom datasource for client metadata
 """
+
 import json
 import logging
 import os
@@ -39,8 +40,9 @@ def ansible_host(broker):
     """
     insights_config = broker.get('client_config')
     if insights_config and insights_config.ansible_host:
-        return DatasourceProvider(content=insights_config.ansible_host,
-                                  relative_path='ansible_host')
+        return DatasourceProvider(
+            content=insights_config.ansible_host, relative_path='ansible_host'
+        )
     raise SkipComponent
 
 
@@ -77,6 +79,7 @@ def blacklist_report(broker):
     Returns:
         str: The JSON strings
     """
+
     def length(lst):
         '''
         Because of how the INI remove.conf is parsed,
@@ -115,13 +118,11 @@ def length(lst):
         )
         if isinstance(redact_config.get('patterns'), dict):
             ret.update(
-                patterns=length(redact_config['patterns']['regex']),
-                using_patterns_regex=True
+                patterns=length(redact_config['patterns']['regex']), using_patterns_regex=True
             )
         else:
             ret.update(patterns=length(redact_config.get('patterns')))
-    return DatasourceProvider(content=json.dumps(ret),
-                              relative_path='blacklist_report')
+    return DatasourceProvider(content=json.dumps(ret), relative_path='blacklist_report')
 
 
 @datasource(HostContext)
@@ -137,8 +138,9 @@ def blacklisted_specs(broker):
         str: The JSON strings
     """
     if BLACKLISTED_SPECS:
-        return DatasourceProvider(content=json.dumps({"specs": BLACKLISTED_SPECS}),
-                                  relative_path='blacklisted_specs')
+        return DatasourceProvider(
+            content=json.dumps({"specs": BLACKLISTED_SPECS}), relative_path='blacklisted_specs'
+        )
     raise SkipComponent
 
 
@@ -157,8 +159,7 @@ def branch_info(broker):
             branch_info = constants.default_branch_info
         else:
             branch_info = insights_config.branch_info
-    return DatasourceProvider(content=json.dumps(branch_info),
-                              relative_path='branch_info')
+    return DatasourceProvider(content=json.dumps(branch_info), relative_path='branch_info')
 
 
 @datasource(HostContext)
@@ -175,8 +176,9 @@ def display_name(broker):
     """
     insights_config = broker.get('client_config')
     if insights_config and insights_config.display_name:
-        return DatasourceProvider(content=insights_config.display_name,
-                                  relative_path='display_name')
+        return DatasourceProvider(
+            content=insights_config.display_name, relative_path='display_name'
+        )
     raise SkipComponent
 
 
@@ -198,10 +200,13 @@ def egg_release(broker):
             egg_release = fil.read()
     except (IOError, MemoryError) as e:
         logger.debug('Could not read the egg release file: %s', str(e))
+    try:
+        os.remove(constants.egg_release_file)
+    except OSError as e:
+        logger.debug('Could not remove the egg release file: %s', str(e))
 
     if egg_release:
-        return DatasourceProvider(content=egg_release,
-                                  relative_path='egg_release')
+        return DatasourceProvider(content=egg_release, relative_path='egg_release')
     raise SkipComponent
 
 
@@ -227,13 +232,10 @@ def tags(broker):
         except (yaml.YAMLError, yaml.parser.ParserError) as e:
             # can't parse yaml from conf
             logger.error("Invalid YAML. Unable to load '%s'" % tags_file_path)
-            raise ContentException('ERROR: Cannot parse %s.\n\nError details: \n%s\n' % (tags_file_path, e))
+            raise ContentException(
+                'ERROR: Cannot parse %s.\n\nError details: \n%s\n' % (tags_file_path, e)
+            )
 
-        # --START--
-        # NOTE:
-        # The following code is from the following function
-        # - insights.client.core_collector.CoreCollector._write_tags
-        # Please keep them consistence before removing that.
         def f(k, v):
             if type(v) is list:
                 col = []
@@ -247,18 +249,17 @@ def f(k, v):
                 return list(chain.from_iterable(col))
             else:
                 return [{"key": k, "value": v, "namespace": constants.app_name}]
+
         t = []
         for k, v in tags.items():
             iv = f(k, v)
             t.append(iv)
         t = list(chain.from_iterable(t))
-        # --END--
 
         if t:
             # The actual file path in archive:
             # - insights-archive-xxx/data/tags.json
-            return DatasourceProvider(content=json.dumps(t),
-                                      relative_path='tags.json')
+            return DatasourceProvider(content=json.dumps(t), relative_path='tags.json')
         msg = "Empty YAML. Unable to load '%s'." % tags_file_path
         logger.error(msg)
         raise ContentException(msg)
@@ -280,9 +281,9 @@ def version_info(broker):
         client_version = None
 
     version_info = {}
-    version_info['core_version'] = '{0}-{1}'.format(package_info['VERSION'],
-                                                    package_info['RELEASE'])
+    version_info['core_version'] = '{0}-{1}'.format(
+        package_info['VERSION'], package_info['RELEASE']
+    )
     version_info['client_version'] = client_version
 
-    return DatasourceProvider(content=json.dumps(version_info),
-                              relative_path='version_info')
+    return DatasourceProvider(content=json.dumps(version_info), relative_path='version_info')

insights/tests/client/core_collector/test_run_collection.py

@@ -1,39 +1,18 @@
 from mock.mock import patch
 
-from insights.client.archive import InsightsArchive
 from insights.client.config import InsightsConfig
 from insights.client.core_collector import CoreCollector
 
 
 @patch('insights.client.core_collector.logger')
-@patch('insights.client.core_collector.CoreCollector._write_egg_release', return_value=None)
-@patch('insights.client.core_collector.CoreCollector._write_blacklisted_specs', return_value=None)
-@patch('insights.client.core_collector.CoreCollector._write_blacklist_report', return_value=None)
-@patch('insights.client.core_collector.CoreCollector._write_tags', return_value=None)
-@patch('insights.client.core_collector.CoreCollector._write_version_info', return_value=None)
-@patch('insights.client.core_collector.CoreCollector._write_ansible_host', return_value=None)
-@patch('insights.client.core_collector.CoreCollector._write_display_name', return_value=None)
-@patch('insights.client.core_collector.CoreCollector._write_branch_info', return_value=None)
 @patch('insights.client.core_collector.systemd_notify_init_thread', return_value=None)
 @patch('insights.client.core_collector.InsightsArchive.create_archive_dir', return_value=None)
 @patch('insights.client.core_collector.collect', return_value=None)
-def test_run_collection(
-        collect,
-        create_archive_dir,
-        systemd_notify_init_thread,
-        _write_branch_info,
-        _write_display_name,
-        _write_ansible_host,
-        _write_version_info,
-        _write_tags,
-        _write_blacklist_report,
-        _write_blacklisted_specs,
-        _write_egg_release,
-        logger):
+def test_run_collection(collect, create_archive_dir, systemd_notify_init_thread, logger):
     conf = InsightsConfig()
-    arch = InsightsArchive(conf)
-    cc = CoreCollector(conf, arch)
+    cc = CoreCollector(conf)
     cc.run_collection({}, {}, {})
+    systemd_notify_init_thread.assert_called_once()
     create_archive_dir.assert_called_once()
     collect.collect.assert_called_once()
-    logger.debug.assert_called_with('Metadata collection finished.')
+    logger.debug.assert_called_with('Core collection finished.')