Merge pull request #572 from mendix/develop

Release 2022-08-25

buildpack/stage.py

@@ -146,7 +146,23 @@ def is_source_push():
         return False
 
 
+def cleanup_dependency_cache(cached_dir, dependency_list):
+    # get directory structure
+    for root, dirs, files in os.walk(cached_dir):
+        for file in files:
+            file_full_path = os.path.join(root, file)
+            logging.debug("dependency in cache folder: [{}]".format(file_full_path))
+            if file_full_path not in dependency_list:
+                # delete from cache
+                os.remove(file_full_path)
+                logging.debug(
+                    "deleted unused dependency [{}] from [{}]...".format(file_full_path, root)
+                )
+
+
 if __name__ == "__main__":
+    util.initialize_globals()
+
     logging.basicConfig(
         level=util.get_buildpack_loglevel(),
         stream=sys.stdout,
@@ -207,3 +223,5 @@ def is_source_push():
     databroker.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     nginx.stage(BUILDPACK_DIR, BUILD_DIR, CACHE_DIR)
     logging.info("Mendix Cloud Foundry Buildpack staging completed")
+
+    cleanup_dependency_cache(CACHE_DIR, util.CACHED_DEPENDENCIES)

buildpack/telemetry/metrics.py

@@ -467,7 +467,22 @@ def _inject_storage_stats(self, stats):
 
     def _inject_database_stats(self, stats):
         database_stats = {}
-        index_size = self._get_database_index_size()
+        stats["database"] = database_stats
+
+        try:
+            index_size = self._get_database_index_size()
+        except psycopg2.OperationalError as err:
+            # For basic apps using Aurora serverless, db connections
+            # are closed every day. Handle this db connection error gracefully
+            # and need not proceed collecting db stats for this round.
+            # https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless.html#aurora-serverless.limitations
+            # The database connection will be refreshed in the next round
+            # of stats collection.
+            logging.warn(
+                "Database is currently not reachable. Failed to gather database stats."
+            )
+            return stats
+
         if index_size:
             database_stats["indexes_size"] = index_size
         storage = self._get_database_storage()
@@ -479,7 +494,6 @@ def _inject_database_stats(self, stats):
         mutations_stats = self._get_database_mutations()
         if mutations_stats:
             database_stats.update(mutations_stats)
-        stats["database"] = database_stats
         tcp_latency = self._get_database_tcp_latency()
         if tcp_latency:
             database_stats["tcp_latency"] = tcp_latency

buildpack/util.py

@@ -75,8 +75,13 @@ def _flatten(l):
 DEPENDENCY_VERSION_KEY = "version"
 DEPENDENCY_FILE = "dependencies.yml"
 DO_NOT_RECURSE_FIELDS = [DEPENDENCY_ALIAS_KEY, DEPENDENCY_NAME_KEY]
+CACHED_DEPENDENCIES = []
 
 
+def initialize_globals():
+    global CACHED_DEPENDENCIES
+    CACHED_DEPENDENCIES = []
+
 # Returns whether an object is a "variable" / literal in a dependency definition
 def _is_dependency_literal(o):
     return type(o) in (int, str, float, bool)
@@ -339,6 +344,7 @@ def resolve_dependency(
 
     vendored_location = _find_file_in_directory(file_name, vendor_dir)
     cached_location = os.path.join(cache_dir, file_name)
+    CACHED_DEPENDENCIES.append(cached_location)
     if not is_path_accessible(vendored_location):
         if ignore_cache or not is_path_accessible(cached_location):
             download(url, cached_location)