Reimplement caching to use a per-year scope

- Add forgiveness if a year fails, continue to the next one
- add continue feature
- Let the continue process run every 2 hourse, not every 30 min
- Add continue support to docker image mirror.sh - off by default
- Add lockfile

.gitignore

@@ -32,3 +32,6 @@ nb-configuration.xml
 **/nbproject/
 local.properties
 data-source/data/
+
+# IntellIJ run configs
+.run

open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java

@@ -17,6 +17,7 @@
 package io.github.jeremylong.openvulnerability.client.nvd;
 
 import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier;
+import java.util.stream.Collectors;
 import org.apache.hc.core5.http.NameValuePair;
 import org.apache.hc.core5.http.message.BasicNameValuePair;
 import org.slf4j.Logger;
@@ -249,7 +250,12 @@ public NvdCveClientBuilder withAdditionalUserAgent(String userAgent) {
      * @return the builder
      */
     public NvdCveClientBuilder withPublishedDateFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) {
-        DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
+        DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME;
+
+        // ensure we have no filters yet
+        filters.removeIf((item) -> item.getName().equals("pubStartDate"));
+        filters.removeIf((item) -> item.getName().equals("pubEndDate"));
+
         filters.add(new BasicNameValuePair("pubStartDate", utcStartDate.format(dtf)));
         filters.add(new BasicNameValuePair("pubEndDate", utcEndDate.format(dtf)));
         return this;

vulnz/Dockerfile

@@ -37,15 +37,17 @@ RUN apk update && \
 
 COPY ["/src/docker/supervisor/supervisord.conf", "/etc/supervisord.conf"]
 COPY ["/src/docker/scripts/mirror.sh", "/mirror.sh"]
+COPY ["/src/docker/scripts/continue.sh", "/continue.sh"]
 COPY ["/src/docker/scripts/validate.sh", "/validate.sh"]
 COPY ["/src/docker/crontab/mirror", "/etc/crontabs/mirror"]
 COPY ["/src/docker/crontab/validate", "/etc/crontabs/validate"]
+COPY ["/src/docker/crontab/continue", "/etc/crontabs/continue"]
 COPY ["/src/docker/apache/mirror.conf", "/usr/local/apache2/conf"]
 COPY ["/build/libs/vulnz-$BUILD_VERSION.jar", "/usr/local/bin/vulnz"]
 
-RUN chmod +x /mirror.sh /validate.sh && \
-    chown root:root /etc/crontabs/mirror /etc/crontabs/validate && \
-    chown mirror:mirror /mirror.sh /validate.sh && \
+RUN chmod +x /mirror.sh /continue.sh /validate.sh && \
+    chown root:root /etc/crontabs/mirror /etc/crontabs/continue /etc/crontabs/validate && \
+    chown mirror:mirror /mirror.sh /continue.sh /validate.sh && \
     chown mirror:mirror /usr/local/bin/vulnz 
 
 # ensures we can log cron task is into stdout of docker

vulnz/README.md

@@ -133,6 +133,10 @@ docker run --name vulnz -e JAVA_OPT=-Xmx2g jeremylong/open-vulnerability-data-mi
 # you can also adjust the delay 
 docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=3000 jeremylong/open-vulnerability-data-mirror:v7.2.1 
 
+# you can use the continue feature to skip cache files for years not older then 3 days
+# if you enable the continue feature, it will also try to fetch missing years every 2 hours
+docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=3000 -e CONTINUE=1 jeremylong/open-vulnerability-data-mirror:v7.2.1 
+
 ```
 
 If you like, run this to pre-populate the database right away
@@ -148,7 +152,8 @@ Assuming the current version is `7.2.1`
 ```bash
 export TARGET_VERSION=7.2.1
 ./gradlew vulnz:build -Pversion=$TARGET_VERSION
-docker build vulnz/ -t ghcr.io/jeremylong/vulnz:$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION
+docker build vulnz/ -t ghcr.io/jeremylong/vulnz:v$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION
+docker push 
 ```
 
 ### Release

vulnz/src/docker/crontab/continue

@@ -0,0 +1 @@
+0 */2 * * * /continue.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_continue.log

vulnz/src/docker/scripts/continue.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -e
+
+LOCKFILE=/tmp/vulzn.lock
+if [ -f $LOCKFILE ]; then
+  echo "Lockfile found - another mirror-sync process already running"
+else
+  touch $LOCKFILE
+fi
+
+if [ -z "${CONTINUE}" ]; then
+  # continue disabled, do not retry failed years
+  exit 0
+fi
+
+echo "Continuing feature detected - ensuring all years mirrored..."
+
+DELAY_ARG=""
+if [ -z $NVD_API_KEY ]; then
+  DELAY_ARG="--delay=10000"
+else
+  echo "Using NVD API KEY: ${NVD_API_KEY:0:5}****"
+fi
+
+if [ -n "${DELAY}" ]; then
+  echo "Overriding delay with ${DELAY}ms"
+  DELAY_ARG="--delay=$DELAY"
+fi
+
+MAX_RETRY_ARG=""
+if [ -n "${MAX_RETRY}" ]; then
+  echo "Using max retry attempts: $MAX_RETRY"
+  MAX_RETRY_ARG="--maxRetry=$MAX_RETRY"
+fi
+
+MAX_RECORDS_PER_PAGE_ARG=""
+if [ -n "${MAX_RECORDS_PER_PAGE}" ]; then
+  echo "Using max records per page: $MAX_RECORDS_PER_PAGE"
+  MAX_RECORDS_PER_PAGE_ARG="--recordsPerPage=$MAX_RECORDS_PER_PAGE"
+fi
+
+function remove_lockfile() {
+  rm -f $LOCKFILE
+  exit 0
+}
+trap remove_lockfile SIGHUP SIGINT SIGQUIT SIGABRT SIGALRM SIGTERM SIGTSTP
+
+java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG --continue --cache --directory /usr/local/apache2/htdocs
+
+rm -f $LOCKFILE

vulnz/src/docker/scripts/mirror.sh

@@ -1,8 +1,17 @@
 #!/bin/sh
+
 set -e
 
 echo "Updating..."
 
+LOCKFILE=/tmp/vulzn.lock
+
+if [ -f $LOCKFILE ]; then
+  echo "Lockfile found - another mirror-sync process already running"
+else
+  touch $LOCKFILE
+fi
+
 DELAY_ARG=""
 if [ -z $NVD_API_KEY ]; then
   DELAY_ARG="--delay=10000"
@@ -33,5 +42,18 @@ if [ -n "${DEBUG}" ]; then
   DEBUG_ARG="--debug"
 fi
 
-exec java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG --cache --directory /usr/local/apache2/htdocs
+CONTINUE_ARG=""
+if [ -n "${CONTINUE}" ]; then
+  echo "Using continue"
+  CONTINUE_ARG="--continue"
+fi
+
+function remove_lockfile() {
+  rm -f $LOCKFILE
+  exit 0
+}
+trap remove_lockfile SIGHUP SIGINT SIGQUIT SIGABRT SIGALRM SIGTERM SIGTSTP
+
+java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG $CONTINUE_ARG --cache --directory /usr/local/apache2/htdocs
 
+rm -f $LOCKFILE

vulnz/src/docker/supervisor/supervisord.conf

@@ -59,3 +59,4 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 redirect_stderr=true
 user=mirror
+stopsecs=29