Reimplement caching to use a per-year scope

- Add forgiveness if a year fails, continue to the next one
- add continue feature
- Let the continue process run every 2 hourse, not every 30 min
- Add continue support to docker image mirror.sh - off by default

open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java

@@ -17,6 +17,7 @@
 package io.github.jeremylong.openvulnerability.client.nvd;
 
 import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier;
+import java.util.stream.Collectors;
 import org.apache.hc.core5.http.NameValuePair;
 import org.apache.hc.core5.http.message.BasicNameValuePair;
 import org.slf4j.Logger;
@@ -249,7 +250,12 @@ public NvdCveClientBuilder withAdditionalUserAgent(String userAgent) {
      * @return the builder
      */
     public NvdCveClientBuilder withPublishedDateFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) {
-        DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
+        DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME;
+
+        // ensure we have no filters yet
+        filters.removeIf((item) -> item.getName().equals("pubStartDate"));
+        filters.removeIf((item) -> item.getName().equals("pubEndDate"));
+
         filters.add(new BasicNameValuePair("pubStartDate", utcStartDate.format(dtf)));
         filters.add(new BasicNameValuePair("pubEndDate", utcEndDate.format(dtf)));
         return this;

vulnz/Dockerfile

@@ -37,15 +37,17 @@ RUN apk update && \
 
 COPY ["/src/docker/supervisor/supervisord.conf", "/etc/supervisord.conf"]
 COPY ["/src/docker/scripts/mirror.sh", "/mirror.sh"]
+COPY ["/src/docker/scripts/continue.sh", "/continue.sh"]
 COPY ["/src/docker/scripts/validate.sh", "/validate.sh"]
 COPY ["/src/docker/crontab/mirror", "/etc/crontabs/mirror"]
 COPY ["/src/docker/crontab/validate", "/etc/crontabs/validate"]
+COPY ["/src/docker/crontab/continue", "/etc/crontabs/continue"]
 COPY ["/src/docker/apache/mirror.conf", "/usr/local/apache2/conf"]
 COPY ["/build/libs/vulnz-$BUILD_VERSION.jar", "/usr/local/bin/vulnz"]
 
-RUN chmod +x /mirror.sh /validate.sh && \
-    chown root:root /etc/crontabs/mirror /etc/crontabs/validate && \
-    chown mirror:mirror /mirror.sh /validate.sh && \
+RUN chmod +x /mirror.sh /continue.sh /validate.sh && \
+    chown root:root /etc/crontabs/mirror /etc/crontabs/continue /etc/crontabs/validate && \
+    chown mirror:mirror /mirror.sh /continue.sh /validate.sh && \
     chown mirror:mirror /usr/local/bin/vulnz 
 
 # ensures we can log cron task is into stdout of docker

vulnz/README.md

@@ -133,6 +133,10 @@ docker run --name vulnz -e JAVA_OPT=-Xmx2g jeremylong/open-vulnerability-data-mi
 # you can also adjust the delay 
 docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=3000 jeremylong/open-vulnerability-data-mirror:v7.2.1 
 
+# you can use the continue feature to skip cache files for years not older then 3 days
+# if you enable the continue feature, it will also try to fetch missing years every 2 hours
+docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=3000 -e CONTINUE=1 jeremylong/open-vulnerability-data-mirror:v7.2.1 
+
 ```
 
 If you like, run this to pre-populate the database right away

vulnz/src/docker/crontab/continue

@@ -0,0 +1 @@
+0 */2 * * * /continue.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_continue.log

vulnz/src/docker/scripts/continue.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+set -e
+
+if [ -z "${CONTINUE}" ]; then
+  # continue disabled, do not retry failed years
+  exit 0
+fi
+
+echo "Continuing feature detected - ensuring all years mirrored..."
+
+DELAY_ARG=""
+if [ -z $NVD_API_KEY ]; then
+  DELAY_ARG="--delay=10000"
+else
+  echo "Using NVD API KEY: ${NVD_API_KEY:0:5}****"
+fi
+
+if [ -n "${DELAY}" ]; then
+  echo "Overriding delay with ${DELAY}ms"
+  DELAY_ARG="--delay=$DELAY"
+fi
+
+MAX_RETRY_ARG=""
+if [ -n "${MAX_RETRY}" ]; then
+  echo "Using max retry attempts: $MAX_RETRY"
+  MAX_RETRY_ARG="--maxRetry=$MAX_RETRY"
+fi
+
+MAX_RECORDS_PER_PAGE_ARG=""
+if [ -n "${MAX_RECORDS_PER_PAGE}" ]; then
+  echo "Using max records per page: $MAX_RECORDS_PER_PAGE"
+  MAX_RECORDS_PER_PAGE_ARG="--recordsPerPage=$MAX_RECORDS_PER_PAGE"
+fi
+
+exec java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG --continue --cache --directory /usr/local/apache2/htdocs
+

vulnz/src/docker/scripts/mirror.sh

@@ -33,5 +33,10 @@ if [ -n "${DEBUG}" ]; then
   DEBUG_ARG="--debug"
 fi
 
-exec java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG --cache --directory /usr/local/apache2/htdocs
+CONTINUE_ARG=""
+if [ -n "${CONTINUE}" ]; then
+  echo "Using continue"
+  CONTINUE_ARG="--continue"
+fi
 
+exec java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG $CONTINUE_ARG --cache --directory /usr/local/apache2/htdocs