feat: reimplement caching to use a per-year scope fixes #258 (#259)

* Reimplement caching to use a per-year scope
* Add forgiveness if a year fails, continue to the next one
* Reimplement how the lastUpdated date is used and stored per year
* Add lockfile
* fix: preserve modified entries if year fails
* polish docs, add exit code

---------

Co-authored-by: Jeremy Long <[email protected]>

.gitignore

@@ -32,3 +32,6 @@ nb-configuration.xml
 **/nbproject/
 local.properties
 data-source/data/
+
+# IntellIJ run configs
+.run

open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java

@@ -17,6 +17,7 @@
 package io.github.jeremylong.openvulnerability.client.nvd;
 
 import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier;
+import java.util.stream.Collectors;
 import org.apache.hc.core5.http.NameValuePair;
 import org.apache.hc.core5.http.message.BasicNameValuePair;
 import org.slf4j.Logger;
@@ -224,12 +225,22 @@ public NvdCveClientBuilder withFilter(BooleanFilter filter) {
      * @return the builder
      */
     public NvdCveClientBuilder withLastModifiedFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) {
-        DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
+        DateTimeFormatter dtf = DateTimeFormatter.ISO_OFFSET_DATE_TIME;
+        // ensure we have no filters yet
+        removeLastModifiedFilter();
+
         filters.add(new BasicNameValuePair("lastModStartDate", utcStartDate.format(dtf)));
         filters.add(new BasicNameValuePair("lastModEndDate", utcEndDate.format(dtf)));
         return this;
     }
 
+    public NvdCveClientBuilder removeLastModifiedFilter() {
+        // ensure we have no filters yet
+        filters.removeIf((item) -> item.getName().equals("lastModStartDate"));
+        filters.removeIf((item) -> item.getName().equals("lastModEndDate"));
+        return this;
+    }
+
     /**
      * Use an additional identifier as part of the User-Agent when making requests.
      *
@@ -249,12 +260,22 @@ public NvdCveClientBuilder withAdditionalUserAgent(String userAgent) {
      * @return the builder
      */
     public NvdCveClientBuilder withPublishedDateFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) {
-        DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
+        DateTimeFormatter dtf = DateTimeFormatter.ISO_OFFSET_DATE_TIME;
+
+        // ensure we have no filters yet
+        removePublishDateFilter();
+
         filters.add(new BasicNameValuePair("pubStartDate", utcStartDate.format(dtf)));
         filters.add(new BasicNameValuePair("pubEndDate", utcEndDate.format(dtf)));
         return this;
     }
 
+    public NvdCveClientBuilder removePublishDateFilter() {
+        filters.removeIf((item) -> item.getName().equals("pubStartDate"));
+        filters.removeIf((item) -> item.getName().equals("pubEndDate"));
+        return this;
+    }
+
     /**
      * Filter the results for a specific CVSS V2 Severity.
      *

vulnz/Dockerfile

@@ -39,12 +39,11 @@ COPY ["/src/docker/supervisor/supervisord.conf", "/etc/supervisord.conf"]
 COPY ["/src/docker/scripts/mirror.sh", "/mirror.sh"]
 COPY ["/src/docker/scripts/validate.sh", "/validate.sh"]
 COPY ["/src/docker/crontab/mirror", "/etc/crontabs/mirror"]
-COPY ["/src/docker/crontab/validate", "/etc/crontabs/validate"]
 COPY ["/src/docker/apache/mirror.conf", "/usr/local/apache2/conf"]
 COPY ["/build/libs/vulnz-$BUILD_VERSION.jar", "/usr/local/bin/vulnz"]
 
 RUN chmod +x /mirror.sh /validate.sh && \
-    chown root:root /etc/crontabs/mirror /etc/crontabs/validate && \
+    chown root:root /etc/crontabs/mirror && \
     chown mirror:mirror /mirror.sh /validate.sh && \
     chown mirror:mirror /usr/local/bin/vulnz 
 

vulnz/README.md

@@ -132,7 +132,6 @@ docker run --name vulnz -e JAVA_OPT=-Xmx2g jeremylong/open-vulnerability-data-mi
 
 # you can also adjust the delay 
 docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=3000 jeremylong/open-vulnerability-data-mirror:v7.2.1 
-
 ```
 
 If you like, run this to pre-populate the database right away
@@ -148,7 +147,8 @@ Assuming the current version is `7.2.1`
 ```bash
 export TARGET_VERSION=7.2.1
 ./gradlew vulnz:build -Pversion=$TARGET_VERSION
-docker build vulnz/ -t ghcr.io/jeremylong/vulnz:$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION
+docker build vulnz/ -t ghcr.io/jeremylong/vulnz:v$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION
+docker push 
 ```
 
 ### Release

vulnz/src/docker/crontab/mirror

@@ -1 +1,2 @@
 0 0 * * * /mirror.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_mirror.log
+0 4 * * * /validate.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_validate.log

vulnz/src/docker/scripts/mirror.sh

@@ -1,8 +1,17 @@
 #!/bin/sh
+
 set -e
 
 echo "Updating..."
 
+LOCKFILE=/tmp/vulzn.lock
+
+if [ -f $LOCKFILE ]; then
+  echo "Lockfile found - another mirror-sync process already running"
+else
+  touch $LOCKFILE
+fi
+
 DELAY_ARG=""
 if [ -z $NVD_API_KEY ]; then
   DELAY_ARG="--delay=10000"
@@ -33,5 +42,12 @@ if [ -n "${DEBUG}" ]; then
   DEBUG_ARG="--debug"
 fi
 
-exec java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG --cache --directory /usr/local/apache2/htdocs
+function remove_lockfile() {
+  rm -f $LOCKFILE
+  exit 0
+}
+trap remove_lockfile SIGHUP SIGINT SIGQUIT SIGABRT SIGALRM SIGTERM SIGTSTP
+
+java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG $CONTINUE_ARG --cache --directory /usr/local/apache2/htdocs
 
+rm -f $LOCKFILE

vulnz/src/docker/supervisor/supervisord.conf

@@ -59,3 +59,4 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 redirect_stderr=true
 user=mirror
+stopsecs=29