Monitoring bug fixes and enhancements (#239)

* Add Prometheus metrics and improve configurability

Introduced Prometheus counters for tracking application version and CVE counts. Made metrics write interval configurable via application properties. Applied minor refactorings to enhance maintainability and flexibility.

* Update metrics, add application name, and improve docs

Updated metrics to include application name and version. Enhanced documentation with memory optimization tips and added a new environment variable for metrics configuration. Upgraded Prometheus library to version 1.3.5, and removed redundant code in `CveCommand`.

* Remove TaskSchedulingAutoConfiguration import

This change removes the unnecessary import of TaskSchedulingAutoConfiguration, simplifying the configuration and potentially reducing overhead. The scheduling functionality remains intact due to the @EnableScheduling annotation.

* Clarify units for `METRICS_WRITE_INTERVAL` in README.

Updated the `METRICS_WRITE_INTERVAL` description to specify that the update interval is measured in milliseconds. This improves the clarity of the documentation and avoids potential confusion for users.

* Rename and update CVE metric gauges for clarity.

Renamed the existing CVE counter to `CVE_LOAD_COUNTER` and introduced a new `CVE_COUNTER` to differentiate between loaded and cached CVE counts. Updated logic to set the new `CVE_COUNTER` value based on cached CVEs.

* Refactor metric builder formatting for readability

Consolidate and reformat metric builder initialization to improve code readability and maintain consistency across the Application and CveCommand classes. This change does not modify functionality but simplifies line breaking for better clarity.

vulnz/README.md

@@ -79,6 +79,11 @@ Alternatively, run the CLI using the `-Xmx2g` argument:
 java -Xmx2g -jar ./vulnz-7.1.0.jar
 ```
 
+An option to save memory would be: `-XX:+UseStringDeduplication`:
+```bash
+export JAVA_OPTS="-Xmx2g -XX:+UseStringDeduplication"
+```
+
 ### Creating the Cache
 
 To create a local cache of the NVD CVE Data you can execute the following command
@@ -110,6 +115,7 @@ There are a couple of ENV vars
 - `MAX_RETRY_ARG` Using max retry attempts
 - `MAX_RECORDS_PER_PAGE_ARG` Using max records per page
 - `METRICS_ENABLE` If is set to `true`, OpenMetrics data for the vulnz cli can be retrieved via the endpoint http://.../metrics
+- `METRICS_WRITE_INTERVAL` Sets the update interval for generating metrics, in milliseconds. Default: `5000`
 
 ### Run
 

vulnz/build.gradle

@@ -30,9 +30,9 @@ dependencies {
     implementation 'jakarta.transaction:jakarta.transaction-api:2.0.1'
     implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
     implementation 'com.sun.activation:jakarta.activation:2.0.1'
-    implementation 'io.prometheus:client_java:1.3.4'
-    implementation 'io.prometheus:prometheus-metrics-exposition-formats:1.3.4'
-    implementation 'io.prometheus:prometheus-metrics-instrumentation-jvm:1.3.4'
+    implementation 'io.prometheus:client_java:1.3.5'
+    implementation 'io.prometheus:prometheus-metrics-exposition-formats:1.3.5'
+    implementation 'io.prometheus:prometheus-metrics-instrumentation-jvm:1.3.5'
 
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
 }

vulnz/src/main/java/io/github/jeremylong/vulnz/cli/Application.java

@@ -17,6 +17,9 @@
 package io.github.jeremylong.vulnz.cli;
 
 import io.github.jeremylong.vulnz.cli.commands.MainCommand;
+import io.prometheus.metrics.core.metrics.Counter;
+import io.prometheus.metrics.model.snapshots.Labels;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.ExitCodeGenerator;
 import org.springframework.boot.SpringApplication;
@@ -31,7 +34,6 @@
 import org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration;
 import org.springframework.boot.autoconfigure.sql.init.SqlInitializationAutoConfiguration;
 import org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration;
-import org.springframework.boot.autoconfigure.task.TaskSchedulingAutoConfiguration;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import picocli.CommandLine;
 import picocli.spring.boot.autoconfigure.PicocliAutoConfiguration;
@@ -43,13 +45,18 @@
         ConfigurationPropertiesAutoConfiguration.class, ProjectInfoAutoConfiguration.class,
         PropertyPlaceholderAutoConfiguration.class, LifecycleAutoConfiguration.class,
         ApplicationAvailabilityAutoConfiguration.class, AopAutoConfiguration.class, JacksonAutoConfiguration.class,
-        SqlInitializationAutoConfiguration.class, TaskExecutionAutoConfiguration.class,
-        TaskSchedulingAutoConfiguration.class})
+        SqlInitializationAutoConfiguration.class, TaskExecutionAutoConfiguration.class})
 public class Application implements CommandLineRunner, ExitCodeGenerator {
     private final CommandLine.IFactory factory;
     private final MainCommand command;
     private int exitCode;
 
+    @Value("${application.version:0.0.0}")
+    private String applicationVersion;
+
+    @Value("${spring.application.name:nvd}")
+    private String applicationName;
+
     Application(CommandLine.IFactory factory, MainCommand command) {
         this.factory = factory;
         this.command = command;
@@ -70,6 +77,9 @@ public int getExitCode() {
 
     @Override
     public void run(String... args) {
+        Counter.builder().name("application").help("Information about the current project version and name")
+                .constLabels(Labels.of("version", applicationVersion, "name", applicationName)).register().inc();
+
         // add extra line to make output more readable
         System.err.println();
         exitCode = new CommandLine(command, factory).setCaseInsensitiveEnumValuesAllowed(true).execute(args);

vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java

@@ -35,6 +35,7 @@
 import io.github.jeremylong.vulnz.cli.ui.IProgressMonitor;
 import io.github.jeremylong.vulnz.cli.ui.JlineShutdownHook;
 import io.github.jeremylong.vulnz.cli.ui.ProgressMonitor;
+import io.prometheus.metrics.core.metrics.Gauge;
 import org.apache.commons.io.output.CountingOutputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -78,6 +79,11 @@ public class CveCommand extends AbstractNvdCommand {
      */
     private static final String HEXES = "0123456789abcdef";
 
+    private static final Gauge CVE_LOAD_COUNTER = Gauge.builder().name("cve_load_counter")
+            .help("Total number of loaded cve's").register();
+    private static final Gauge CVE_COUNTER = Gauge.builder().name("cve_counter").help("Total number of cached cve's")
+            .register();
+
     @CommandLine.ArgGroup(exclusive = true)
     ConfigGroup configGroup;
 
@@ -288,12 +294,14 @@ private Integer processRequest(NvdCveClientBuilder builder, CacheProperties prop
                 collectCves(cves, data);
                 lastModified = api.getLastUpdated();
                 count += data.size();
+                CVE_LOAD_COUNTER.set(count);
                 monitor.updateProgress("NVD", count, api.getTotalAvailable());
             }
         } catch (Exception ex) {
             LOG.debug("\nERROR", ex);
             throw new CacheException("Unable to complete NVD cache update due to error: " + ex.getMessage());
         }
+        CVE_COUNTER.set(cves.values().stream().mapToLong(HashMap::size).sum());
         if (lastModified != null) {
             properties.set("lastModifiedDate", lastModified);
         }

vulnz/src/main/java/io/github/jeremylong/vulnz/cli/monitoring/PrometheusFileWriter.java

@@ -46,7 +46,7 @@ public void init() {
         JvmMetrics.builder().register();
     }
 
-    @Scheduled(fixedRate = 5000)
+    @Scheduled(fixedRateString = "${metrics.write.interval:5000}")
     public void writeFile() {
         File directory = command.getCacheDirectory();
         if (directory != null) {

vulnz/src/main/resources/application.properties

@@ -8,3 +8,4 @@ spring.main.web-application-type=none
 spring.main.log-startup-info=false
 application.version=@version@
 metrics.enable=false
+metrics.write.interval=5000