From 271412970116f4932a54fda4f1564e1619b640e7 Mon Sep 17 00:00:00 2001 From: em Date: Mon, 27 Jan 2025 15:39:06 +0100 Subject: [PATCH 01/10] Reimplement caching to use a per-year scope - Add forgiveness if a year fails, continue to the next one - Reimplement how the lastUpdated date is used and stored per year - Add lockfile --- .gitignore | 3 + .../client/nvd/NvdCveClientBuilder.java | 25 +- vulnz/README.md | 4 +- vulnz/src/docker/scripts/mirror.sh | 18 +- vulnz/src/docker/supervisor/supervisord.conf | 1 + .../vulnz/cli/commands/CveCommand.java | 417 +++++++++++------- .../vulnz/cli/model/CvesNvdPojo.java | 31 ++ 7 files changed, 345 insertions(+), 154 deletions(-) create mode 100644 vulnz/src/main/java/io/github/jeremylong/vulnz/cli/model/CvesNvdPojo.java diff --git a/.gitignore b/.gitignore index 27b9660c..4f8ac831 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ nb-configuration.xml **/nbproject/ local.properties data-source/data/ + +# IntellIJ run configs +.run diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java index 51b98e77..9447d3a1 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java @@ -17,6 +17,7 @@ package io.github.jeremylong.openvulnerability.client.nvd; import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier; +import java.util.stream.Collectors; import org.apache.hc.core5.http.NameValuePair; import org.apache.hc.core5.http.message.BasicNameValuePair; import org.slf4j.Logger; @@ -224,12 +225,22 @@ public NvdCveClientBuilder withFilter(BooleanFilter filter) { * @return the builder */ public NvdCveClientBuilder withLastModifiedFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) { - DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + DateTimeFormatter dtf = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + // ensure we have no filters yet + removeLastModifiedFilter(); + filters.add(new BasicNameValuePair("lastModStartDate", utcStartDate.format(dtf))); filters.add(new BasicNameValuePair("lastModEndDate", utcEndDate.format(dtf))); return this; } + public NvdCveClientBuilder removeLastModifiedFilter() { + // ensure we have no filters yet + filters.removeIf((item) -> item.getName().equals("lastModStartDate")); + filters.removeIf((item) -> item.getName().equals("lastModEndDate")); + return this; + } + /** * Use an additional identifier as part of the User-Agent when making requests. * @@ -249,12 +260,22 @@ public NvdCveClientBuilder withAdditionalUserAgent(String userAgent) { * @return the builder */ public NvdCveClientBuilder withPublishedDateFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) { - DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + DateTimeFormatter dtf = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + + // ensure we have no filters yet + removePublishDateFilter(); + filters.add(new BasicNameValuePair("pubStartDate", utcStartDate.format(dtf))); filters.add(new BasicNameValuePair("pubEndDate", utcEndDate.format(dtf))); return this; } + public NvdCveClientBuilder removePublishDateFilter() { + filters.removeIf((item) -> item.getName().equals("pubStartDate")); + filters.removeIf((item) -> item.getName().equals("pubEndDate")); + return this; + } + /** * Filter the results for a specific CVSS V2 Severity. * diff --git a/vulnz/README.md b/vulnz/README.md index 3b9278e0..253a3f4b 100644 --- a/vulnz/README.md +++ b/vulnz/README.md @@ -132,7 +132,6 @@ docker run --name vulnz -e JAVA_OPT=-Xmx2g jeremylong/open-vulnerability-data-mi # you can also adjust the delay docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=3000 jeremylong/open-vulnerability-data-mirror:v7.2.1 - ``` If you like, run this to pre-populate the database right away @@ -148,7 +147,8 @@ Assuming the current version is `7.2.1` ```bash export TARGET_VERSION=7.2.1 ./gradlew vulnz:build -Pversion=$TARGET_VERSION -docker build vulnz/ -t ghcr.io/jeremylong/vulnz:$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION +docker build vulnz/ -t ghcr.io/jeremylong/vulnz:v$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION +docker push ``` ### Release diff --git a/vulnz/src/docker/scripts/mirror.sh b/vulnz/src/docker/scripts/mirror.sh index 574bf0ec..51ee4063 100755 --- a/vulnz/src/docker/scripts/mirror.sh +++ b/vulnz/src/docker/scripts/mirror.sh @@ -1,8 +1,17 @@ #!/bin/sh + set -e echo "Updating..." +LOCKFILE=/tmp/vulzn.lock + +if [ -f $LOCKFILE ]; then + echo "Lockfile found - another mirror-sync process already running" +else + touch $LOCKFILE +fi + DELAY_ARG="" if [ -z $NVD_API_KEY ]; then DELAY_ARG="--delay=10000" @@ -33,5 +42,12 @@ if [ -n "${DEBUG}" ]; then DEBUG_ARG="--debug" fi -exec java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG --cache --directory /usr/local/apache2/htdocs +function remove_lockfile() { + rm -f $LOCKFILE + exit 0 +} +trap remove_lockfile SIGHUP SIGINT SIGQUIT SIGABRT SIGALRM SIGTERM SIGTSTP + +java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG $CONTINUE_ARG --cache --directory /usr/local/apache2/htdocs +rm -f $LOCKFILE diff --git a/vulnz/src/docker/supervisor/supervisord.conf b/vulnz/src/docker/supervisor/supervisord.conf index 00fc65d6..e7cdedfd 100644 --- a/vulnz/src/docker/supervisor/supervisord.conf +++ b/vulnz/src/docker/supervisor/supervisord.conf @@ -59,3 +59,4 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true user=mirror +stopsecs=29 diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index 8d3acd9d..63b72587 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -32,39 +32,29 @@ import io.github.jeremylong.vulnz.cli.cache.CacheException; import io.github.jeremylong.vulnz.cli.cache.CacheProperties; import io.github.jeremylong.vulnz.cli.model.BasicOutput; +import io.github.jeremylong.vulnz.cli.model.CvesNvdPojo; import io.github.jeremylong.vulnz.cli.ui.IProgressMonitor; import io.github.jeremylong.vulnz.cli.ui.JlineShutdownHook; import io.github.jeremylong.vulnz.cli.ui.ProgressMonitor; import io.prometheus.metrics.core.metrics.Gauge; -import it.unimi.dsi.fastutil.objects.Object2ObjectMap; -import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import java.io.*; +import java.nio.file.Path; +import java.time.*; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; import org.apache.commons.io.output.CountingOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import picocli.CommandLine; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.time.Year; -import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import static com.diogonunes.jcolor.Ansi.colorize; @@ -76,6 +66,16 @@ public class CveCommand extends AbstractNvdCommand { * Reference to the logger. */ private static final Logger LOG = LoggerFactory.getLogger(CveCommand.class); + + /** + * Start year (until today) to cache CVEs for. + */ + private static final int START_YEAR = 2002; + + // TODO - get format and version from API + private static final String FORMAT = "NVD_CVE"; + private static final String VERSION = "2.0"; + /** * Hex code characters used in getHex. */ @@ -229,16 +229,6 @@ public Integer timedCall() throws Exception { if (configGroup != null && configGroup.cacheSettings != null) { CacheProperties properties = new CacheProperties(configGroup.cacheSettings.directory); - if (properties.has("lastModifiedDate")) { - ZonedDateTime start = properties.getTimestamp("lastModifiedDate"); - ZonedDateTime end = start.minusDays(-120); - if (end.compareTo(ZonedDateTime.now()) > 0) { - builder.withLastModifiedFilter(start, end); - } else { - LOG.warn( - "Requesting the entire set of NVD CVE data via the api as the cache was last updated over 120 days ago"); - } - } if (configGroup.cacheSettings.prefix != null) { properties.set("prefix", configGroup.cacheSettings.prefix); } @@ -262,130 +252,277 @@ public Integer timedCall() throws Exception { return processRequest(builder); } - private Integer processRequest(NvdCveClientBuilder builder, CacheProperties properties) { + /** + * For all years, fetch the NVD vulnerabilities and save them each into one cache file. + */ + private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties properties) { + // will hold all entries that have been changed within the last 7 days across all years + List recentlyChangedEntries = new ArrayList<>(); + + for (int currentYear = START_YEAR; currentYear <= Year.now().getValue(); currentYear++) { + + // Be forgiving, if we fail to fetch a year, we just continue with the next one + try { + // reset our filters for each year + apiBuilder.removeLastModifiedFilter(); + apiBuilder.removePublishDateFilter(); + + Path cacheFilePath = buildCacheTargetFileForYear(properties, currentYear); + LOG.info("INFO *** Processing year {} ***", currentYear); + CvesNvdPojo existingCacheData = loadExistingCacheAndConfigureApi(apiBuilder, currentYear, + cacheFilePath); + CvesNvdPojo cvesForYear = aggregateCvesForYear(currentYear, apiBuilder); + + // merge old with new data. It is intended to add old items to the new ones, thus we keep newly fetched + // items with the same ID from the API, not overriding from old cache + if (existingCacheData != null) { + LOG.info("INFO Fetched #{} updated entries for already existing local cache with #{} entries", + cvesForYear.vulnerabilities.size(), existingCacheData.vulnerabilities.size()); + cvesForYear.vulnerabilities.addAll(existingCacheData.vulnerabilities); + } else { + LOG.info("INFO Fetched #{} new entries for year {}", cvesForYear.vulnerabilities.size(), + currentYear); + } + + if (cvesForYear.lastUpdated != null) { + properties.set("lastModifiedDate", cvesForYear.lastUpdated); + } + storeEntireYearToCache(currentYear, cvesForYear, properties); + LOG.info("INFO *** Finished year {} with #{} entries ***", currentYear, + cvesForYear.vulnerabilities.size()); + + // calculate recently changed entries, means changed within the last 7 days + recentlyChangedEntries.addAll(extractRecentChangedEntries(cvesForYear)); + } catch (Exception ex) { + LOG.error("\nERROR processing year {}", currentYear, ex); + LOG.info("INFO ... continuing with next year"); + } + } + + createRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); + + return 0; + } + + private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuilder, int currentYear, + Path cacheFilePath) { + ZonedDateTime lastUpdateDate = determineExistingCacheFileLastChanged(cacheFilePath); + if (lastUpdateDate == null) { + // no existing cache exists - nothing to load and nothing to configure. Fetch the entire year. + return null; + } + + // else only fetch entries that have been changed since the last update + apiBuilder.withLastModifiedFilter(lastUpdateDate, ZonedDateTime.now()); + LOG.info("INFO Found existing local cache for year {}. Cache was created/updated on {}", currentYear, + lastUpdateDate.format(DateTimeFormatter.RFC_1123_DATE_TIME)); + LOG.info("INFO Only fetching items that have been changed since then from the API"); + + // Load existing cache data + ObjectMapper objectMapper = getCacheObjectMapper(); + try (FileInputStream fis = new FileInputStream(cacheFilePath.toFile()); + GZIPInputStream gzis = new GZIPInputStream(fis)) { + CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); + return new CvesNvdPojo(data.getVulnerabilities(), data.getTimestamp()); + } catch (IOException exception) { + throw new CacheException("Unable to read cached data: " + cacheFilePath, exception); + } + } + + /** + * Given the year, fetch all vulnerabilities for this year by using a publication date range filter with a maximum + * range of 120 days (due to NVD api limits). Ensures vulnerabilities are unique and sorted by publishing date, ASC + */ + private CvesNvdPojo aggregateCvesForYear(int year, NvdCveClientBuilder apiBuilder) { + // by NVDs API limit, 120 is the max range + final int MAX_DAYS_RANGE = 115; + + int dayOfYearStart = 1; + + ZonedDateTime yearStart = ZonedDateTime.of(year, 1, 1, 0, 0, 0, 0, ZoneId.systemDefault()); + int yearLengthInDays = Year.from(yearStart).length(); + CvesNvdPojo finalResult = new CvesNvdPojo(new ArrayList<>(), ZonedDateTime.now()); + while (dayOfYearStart < yearLengthInDays) { + // first day of the year we start from + ZonedDateTime from = yearStart.plusDays(dayOfYearStart); + + if (from.isAfter(ZonedDateTime.now())) { + // we are fetching for items in the future, stop here + break; + } + // get the end day of the range, but ensure we do not overshoot + int dayOfYearEnd = Math.min(dayOfYearStart + MAX_DAYS_RANGE, yearLengthInDays - 1); + + ZonedDateTime to = yearStart.plusDays(dayOfYearEnd) + // since we need the end of the day, just pick the next one and reset time to 00:00 + .plusDays(1).truncatedTo(ChronoUnit.DAYS); + // apply our range to the API call + apiBuilder.withPublishedDateFilter(from, to); + + // fetch entries for this range + CvesNvdPojo currentResult = fetchFromNVDApi(apiBuilder); + + // update our last updated, since we crawl up the year, this will be the most recent in the end + finalResult.lastUpdated = currentResult.lastUpdated; + // aggregate all results + finalResult.vulnerabilities.addAll(currentResult.vulnerabilities); + + // let the next fetch start on the following day we ended on before (since we fetched up to 'to midnight' + dayOfYearStart = dayOfYearEnd + 1; + } + + return finalResult; + } + + /** + * Given the CVEs for the year, stores all of them in a json cache file and a meta-dat file. creates + * nvdcve-$year.json.gz and nvdcve-$year.meta + */ + private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties properties) { + ZonedDateTime lastChanged = Objects.requireNonNullElseGet(cves.lastUpdated, ZonedDateTime::now); + properties.set("lastModifiedDate." + year, lastChanged); + int size = cves.vulnerabilities.size(); + + CveApiJson20 data = new CveApiJson20(size, 0, size, FORMAT, VERSION, lastChanged, cves.vulnerabilities); + + MessageDigest md; + try { + md = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new CacheException("Unable to calculate sha256 checksum", e); + } + + // save vulnerabilities into cache + final File cacheFile = buildCacheTargetFileForYear(properties, year).toFile(); + long uncompressedSize = saveCacheAsGzip(cacheFile, data, md); + + // save meta data + final File metaDataFile = buildMetaDataTargetFileForYear(properties, year).toFile(); + saveMetaData(metaDataFile, cacheFile.length(), uncompressedSize, lastChanged, md); + } + + /** + * Create cache holding all CVE items that have been changed within the last 7 days. Creates: + * nvdcve-modified.json.gz and nvdcve-modified.meta + */ + private void createRecentlyChangedCacheFile(List recentlyChangedEntries, CacheProperties properties, + ZonedDateTime lastChanged) { + final String prefix = properties.get("prefix", "nvdcve-"); + Path recentChangesCachePath = Path.of(properties.getDirectory().getPath(), prefix + "modified.json.gz"); + CveApiJson20 data = new CveApiJson20(recentlyChangedEntries.size(), 0, recentlyChangedEntries.size(), FORMAT, + VERSION, lastChanged, recentlyChangedEntries); + MessageDigest md; + try { + md = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new CacheException("Unable to calculate sha256 checksum", e); + } + File recentCacheFile = recentChangesCachePath.toFile(); + long uncompressedSize = saveCacheAsGzip(recentCacheFile, data, md); + + Path metaDataFile = Path.of(properties.getDirectory().getPath(), prefix + "modified.meta"); + saveMetaData(metaDataFile.toFile(), recentCacheFile.length(), uncompressedSize, lastChanged, md); + } + + private long saveCacheAsGzip(File targetFile, CveApiJson20 data, MessageDigest md) { + final ObjectMapper objectMapper = getCacheObjectMapper(); + + long uncompressedSize; + try (FileOutputStream fileOutputStream = new FileOutputStream(targetFile); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); + DigestOutputStream digestOutputStream = new DigestOutputStream(gzipOutputStream, md); + CountingOutputStream countingOutputStream = new CountingOutputStream(digestOutputStream)) { + objectMapper.writeValue(countingOutputStream, data); + uncompressedSize = countingOutputStream.getByteCount(); + } catch (IOException ex) { + throw new CacheException("Unable to write cached data: " + targetFile, ex); + } + + return uncompressedSize; + } + + private ObjectMapper getCacheObjectMapper() { final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); if (isPrettyPrint()) { objectMapper.enable(SerializationFeature.INDENT_OUTPUT); } - final Object2ObjectOpenHashMap> cves = new Object2ObjectOpenHashMap<>(); - populateKeys(cves); + return objectMapper; + } - ZonedDateTime lastModified = downloadAllUpdates(builder, cves); - if (lastModified != null) { - properties.set("lastModifiedDate", lastModified); + private void saveMetaData(File targetFile, long compressedSize, long uncompressedSize, ZonedDateTime lastChanged, + MessageDigest md) { + String checksum = getHex(md.digest()); + try (FileOutputStream fileOutputStream = new FileOutputStream(targetFile); + OutputStreamWriter osw = new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8); + PrintWriter writer = new PrintWriter(osw)) { + final String lmd = DateTimeFormatter.ISO_DATE_TIME.format(lastChanged); + writer.println("lastModifiedDate:" + lmd); + writer.println("size:" + uncompressedSize); + writer.println("gzSize:" + compressedSize); + writer.println("sha256:" + checksum); + } catch (IOException ex) { + throw new CacheException("Unable to write cached meta-data: {}" + targetFile.getAbsolutePath(), ex); } - // update cache - // todo - get format and version from API - final String format = "NVD_CVE"; - final String version = "2.0"; - final String prefix = properties.get("prefix", "nvdcve-"); - - for (Object2ObjectMap.Entry> entry : cves - .object2ObjectEntrySet()) { - if (entry.getValue().isEmpty()) { - continue; - } - final File file = new File(properties.getDirectory(), prefix + entry.getKey() + ".json.gz"); - final File meta = new File(properties.getDirectory(), prefix + entry.getKey() + ".meta"); - final Object2ObjectOpenHashMap yearData; - - // Load existing year data if present - if (file.isFile()) { - yearData = new Object2ObjectOpenHashMap<>(); - try (FileInputStream fis = new FileInputStream(file); GZIPInputStream gzis = new GZIPInputStream(fis)) { - CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); - boolean isYearData = !"modified".equals(entry.getKey()); - // filter the "modified" data to load only the last 7 days of data - data.getVulnerabilities().stream().filter(cve -> isYearData - || ChronoUnit.DAYS.between(cve.getCve().getLastModified(), ZonedDateTime.now()) <= 7) - .forEach(cve -> yearData.put(cve.getCve().getId(), cve)); - } catch (IOException exception) { - throw new CacheException("Unable to read cached data: " + file, exception); - } - yearData.putAll(entry.getValue()); - } else { - yearData = entry.getValue(); - } + } - List vulnerabilities = new ArrayList(yearData.values()); - vulnerabilities.sort((v1, v2) -> { - return v1.getCve().getId().compareTo(v2.getCve().getId()); - }); - ZonedDateTime timestamp; - Optional maxDate = vulnerabilities.stream().map(v -> v.getCve().getLastModified()) - .max(ZonedDateTime::compareTo); - if (maxDate.isPresent()) { - timestamp = maxDate.get(); - } else if (lastModified != null) { - timestamp = lastModified; - } else { - timestamp = ZonedDateTime.now(); - } - properties.set("lastModifiedDate." + entry.getKey(), timestamp); - CveApiJson20 data = new CveApiJson20(vulnerabilities.size(), 0, vulnerabilities.size(), format, version, - timestamp, vulnerabilities); - MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new CacheException("Unable to calculate sha256 checksum", e); - } - long byteCount = 0; - try (FileOutputStream fileOutputStream = new FileOutputStream(file); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); - DigestOutputStream digestOutputStream = new DigestOutputStream(gzipOutputStream, md); - CountingOutputStream countingOutputStream = new CountingOutputStream(digestOutputStream)) { - objectMapper.writeValue(countingOutputStream, data); - byteCount = countingOutputStream.getByteCount(); - } catch (IOException ex) { - throw new CacheException("Unable to write cached data: " + file, ex); - } - String checksum = getHex(md.digest()); - try (FileOutputStream fileOutputStream = new FileOutputStream(meta); - OutputStreamWriter osw = new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8); - PrintWriter writer = new PrintWriter(osw)) { - final String lmd = DateTimeFormatter.ISO_DATE_TIME.format(timestamp); - writer.println("lastModifiedDate:" + lmd); - writer.println("size:" + byteCount); - writer.println("gzSize:" + file.length()); - writer.println("sha256:" + checksum); - } catch (IOException ex) { - throw new CacheException("Unable to write cached meta-data: " + file, ex); - } + private ZonedDateTime determineExistingCacheFileLastChanged(Path cacheFilePath) { + File cacheFile = cacheFilePath.toFile(); + if (!cacheFile.exists() || cacheFile.length() == 0) { + return null; } - return 0; + + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(cacheFile.getAbsoluteFile().lastModified()), + ZoneId.systemDefault()); } - private static void populateKeys( - Object2ObjectOpenHashMap> cves) { - cves.put("modified", new Object2ObjectOpenHashMap<>()); - for (int i = 2002; i <= Year.now().getValue(); i++) { - cves.put(Integer.toString(i), new Object2ObjectOpenHashMap<>()); - } + private Path buildCacheTargetFileForYear(CacheProperties properties, int year) { + final String prefix = properties.get("prefix", "nvdcve-"); + return Path.of(properties.getDirectory().getPath(), prefix + year + ".json.gz"); } - private ZonedDateTime downloadAllUpdates(NvdCveClientBuilder builder, - Object2ObjectOpenHashMap> cves) { - ZonedDateTime lastModified = null; - int count = 0; + private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year) { + final String prefix = properties.get("prefix", "nvdcve-"); + return Path.of(properties.getDirectory().getPath(), prefix + year + ".meta"); + } + + private List extractRecentChangedEntries(CvesNvdPojo cvesForYear) { + return cvesForYear.vulnerabilities.stream() + .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= 7) + .collect(Collectors.toList()); + } + + /** + * Fetching from the NVD api, paginated (max entries per page). Aggregates all entries of all pages, sorts them by + * publishing date (ASC) and ensures those are unique. + */ + private CvesNvdPojo fetchFromNVDApi(NvdCveClientBuilder apiBuilder) { // retrieve from NVD API - try (NvdCveClient api = builder.build(); IProgressMonitor monitor = new ProgressMonitor(interactive, "NVD")) { + try (NvdCveClient api = apiBuilder.build(); + IProgressMonitor monitor = new ProgressMonitor(interactive, "NVD")) { Runtime.getRuntime().addShutdownHook(new JlineShutdownHook()); + + // we use a set for de-duplication + Set vulnerabilities = new HashSet<>(); + + // crawl the api page by page while (api.hasNext()) { Collection data = api.next(); - collectCves(cves, data); - lastModified = api.getLastUpdated(); - count += data.size(); - CVE_LOAD_COUNTER.set(count); - monitor.updateProgress("NVD", count, api.getTotalAvailable()); + vulnerabilities.addAll(data); + if (!data.isEmpty()) { + CVE_LOAD_COUNTER.set(data.size()); + monitor.updateProgress("NVD", (int) CVE_LOAD_COUNTER.get(), api.getTotalAvailable()); + } } + CVE_COUNTER.set(vulnerabilities.size()); + + // convert to sorted list by id + List sorted = vulnerabilities.stream().sorted(Comparator.comparing(v -> v.getCve().getId())) + .collect(Collectors.toList()); + return new CvesNvdPojo(sorted, api.getLastUpdated()); } catch (Exception ex) { - LOG.debug("\nERROR", ex); throw new CacheException("Unable to complete NVD cache update due to error: " + ex.getMessage()); } - CVE_COUNTER.set(cves.values().stream().mapToLong(Object2ObjectOpenHashMap::size).sum()); - return lastModified; } /** @@ -412,24 +549,6 @@ public static String getHex(byte[] raw) { return hex.toString(); } - private void collectCves(Object2ObjectOpenHashMap> cves, - Collection vulnerabilities) { - for (DefCveItem item : vulnerabilities) { - cves.get(getNvdYear(item)).put(item.getCve().getId(), item); - if (ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= 7) { - cves.get("modified").put(item.getCve().getId(), item); - } - } - } - - private String getNvdYear(DefCveItem item) { - int year = item.getCve().getPublished().getYear(); - if (year < 2002) { - year = 2002; - } - return Integer.toString(year); - } - private int processRequest(NvdCveClientBuilder builder) throws IOException { JsonGenerator jsonOut = getJsonGenerator(); int status = 1; diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/model/CvesNvdPojo.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/model/CvesNvdPojo.java new file mode 100644 index 00000000..8b3812dd --- /dev/null +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/model/CvesNvdPojo.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2025 Jeremy Long. All Rights Reserved. + */ +package io.github.jeremylong.vulnz.cli.model; + +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import java.time.ZonedDateTime; +import java.util.List; + +public class CvesNvdPojo { + public List vulnerabilities; + public ZonedDateTime lastUpdated; + + public CvesNvdPojo(List vulnerabilities, ZonedDateTime lastUpdated) { + this.vulnerabilities = vulnerabilities; + this.lastUpdated = lastUpdated; + } +} From fc1121b0831e9d5f0b3a30c30708bb6c2413abe6 Mon Sep 17 00:00:00 2001 From: em Date: Tue, 28 Jan 2025 07:14:37 +0100 Subject: [PATCH 02/10] Move validate cron task to mirror, remove dedicated file (since it is a user file) --- vulnz/Dockerfile | 3 +-- vulnz/src/docker/crontab/mirror | 1 + vulnz/src/docker/crontab/validate | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 vulnz/src/docker/crontab/validate diff --git a/vulnz/Dockerfile b/vulnz/Dockerfile index 181ee72d..8f108a14 100644 --- a/vulnz/Dockerfile +++ b/vulnz/Dockerfile @@ -39,12 +39,11 @@ COPY ["/src/docker/supervisor/supervisord.conf", "/etc/supervisord.conf"] COPY ["/src/docker/scripts/mirror.sh", "/mirror.sh"] COPY ["/src/docker/scripts/validate.sh", "/validate.sh"] COPY ["/src/docker/crontab/mirror", "/etc/crontabs/mirror"] -COPY ["/src/docker/crontab/validate", "/etc/crontabs/validate"] COPY ["/src/docker/apache/mirror.conf", "/usr/local/apache2/conf"] COPY ["/build/libs/vulnz-$BUILD_VERSION.jar", "/usr/local/bin/vulnz"] RUN chmod +x /mirror.sh /validate.sh && \ - chown root:root /etc/crontabs/mirror /etc/crontabs/validate && \ + chown root:root /etc/crontabs/mirror && \ chown mirror:mirror /mirror.sh /validate.sh && \ chown mirror:mirror /usr/local/bin/vulnz diff --git a/vulnz/src/docker/crontab/mirror b/vulnz/src/docker/crontab/mirror index e0a95541..93111d7a 100755 --- a/vulnz/src/docker/crontab/mirror +++ b/vulnz/src/docker/crontab/mirror @@ -1 +1,2 @@ 0 0 * * * /mirror.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_mirror.log +0 4 * * * /validate.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_validate.log diff --git a/vulnz/src/docker/crontab/validate b/vulnz/src/docker/crontab/validate deleted file mode 100644 index 6dbbb593..00000000 --- a/vulnz/src/docker/crontab/validate +++ /dev/null @@ -1 +0,0 @@ -0 4 * * * /validate.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_validate.log From f2777b304244db1433d00efe654f298b73e41ab1 Mon Sep 17 00:00:00 2001 From: em Date: Tue, 28 Jan 2025 08:09:34 +0100 Subject: [PATCH 03/10] Move validate cron task to mirror, remove dedicated file (since it is a user file) --- .../vulnz/cli/commands/CveCommand.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index 63b72587..e1b1e15d 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -299,7 +299,7 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p } } - createRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); + createCacheRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); return 0; } @@ -385,12 +385,7 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties CveApiJson20 data = new CveApiJson20(size, 0, size, FORMAT, VERSION, lastChanged, cves.vulnerabilities); - MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new CacheException("Unable to calculate sha256 checksum", e); - } + MessageDigest md = getDigestAlg(); // save vulnerabilities into cache final File cacheFile = buildCacheTargetFileForYear(properties, year).toFile(); @@ -405,21 +400,23 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties * Create cache holding all CVE items that have been changed within the last 7 days. Creates: * nvdcve-modified.json.gz and nvdcve-modified.meta */ - private void createRecentlyChangedCacheFile(List recentlyChangedEntries, CacheProperties properties, + private void createCacheRecentlyChangedCacheFile(List recentlyChanged, CacheProperties properties, ZonedDateTime lastChanged) { final String prefix = properties.get("prefix", "nvdcve-"); Path recentChangesCachePath = Path.of(properties.getDirectory().getPath(), prefix + "modified.json.gz"); - CveApiJson20 data = new CveApiJson20(recentlyChangedEntries.size(), 0, recentlyChangedEntries.size(), FORMAT, - VERSION, lastChanged, recentlyChangedEntries); - MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new CacheException("Unable to calculate sha256 checksum", e); - } + int recentSize = recentlyChanged.size(); + + CveApiJson20 data = new CveApiJson20(recentSize, 0, recentSize, FORMAT, VERSION, lastChanged, recentlyChanged); + + MessageDigest md = getDigestAlg(); + + // create cache file including the CVE entries that recently changed File recentCacheFile = recentChangesCachePath.toFile(); long uncompressedSize = saveCacheAsGzip(recentCacheFile, data, md); + LOG.info("INFO Stored {} entries in {} as recent changed items across all years", + data.getVulnerabilities().size(), recentCacheFile.getName()); + // Create meta-file Path metaDataFile = Path.of(properties.getDirectory().getPath(), prefix + "modified.meta"); saveMetaData(metaDataFile.toFile(), recentCacheFile.length(), uncompressedSize, lastChanged, md); } @@ -549,6 +546,14 @@ public static String getHex(byte[] raw) { return hex.toString(); } + private MessageDigest getDigestAlg() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new CacheException("Unable to calculate sha256 checksum", e); + } + } + private int processRequest(NvdCveClientBuilder builder) throws IOException { JsonGenerator jsonOut = getJsonGenerator(); int status = 1; From d898893ca536ca58d18d35dd20186f90324e8ba3 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 6 Feb 2025 06:13:13 -0500 Subject: [PATCH 04/10] fix: preserve modified entries if year fails If there is an exception on year 2017 and there were modified entries in the existing 2017 data file - the old data file would still be on disk for 2017 containing the modified entries - but when we write out the actual modified file, because 2017 was skipped we will not preserve the 2017 modified entries in the "modified" file. This PR corrects this. --- .../vulnz/cli/commands/CveCommand.java | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index e1b1e15d..d2cbb5c1 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -38,6 +38,7 @@ import io.github.jeremylong.vulnz.cli.ui.ProgressMonitor; import io.prometheus.metrics.core.metrics.Gauge; import java.io.*; +import java.nio.file.Files; import java.nio.file.Path; import java.time.*; import java.time.temporal.ChronoUnit; @@ -85,6 +86,7 @@ public class CveCommand extends AbstractNvdCommand { .help("Total number of loaded cve's").register(); private static final Gauge CVE_COUNTER = Gauge.builder().name("cve_counter").help("Total number of cached cve's") .register(); + public static final int EIGHT_DAYS = 8; @CommandLine.ArgGroup(exclusive = true) ConfigGroup configGroup; @@ -256,7 +258,7 @@ public Integer timedCall() throws Exception { * For all years, fetch the NVD vulnerabilities and save them each into one cache file. */ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties properties) { - // will hold all entries that have been changed within the last 7 days across all years + // will hold all entries that have been changed within the last 8 days across all years List recentlyChangedEntries = new ArrayList<>(); for (int currentYear = START_YEAR; currentYear <= Year.now().getValue(); currentYear++) { @@ -269,8 +271,7 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p Path cacheFilePath = buildCacheTargetFileForYear(properties, currentYear); LOG.info("INFO *** Processing year {} ***", currentYear); - CvesNvdPojo existingCacheData = loadExistingCacheAndConfigureApi(apiBuilder, currentYear, - cacheFilePath); + CvesNvdPojo existingCacheData = loadExistingCacheAndConfigureApi(apiBuilder, cacheFilePath); CvesNvdPojo cvesForYear = aggregateCvesForYear(currentYear, apiBuilder); // merge old with new data. It is intended to add old items to the new ones, thus we keep newly fetched @@ -291,7 +292,7 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p LOG.info("INFO *** Finished year {} with #{} entries ***", currentYear, cvesForYear.vulnerabilities.size()); - // calculate recently changed entries, means changed within the last 7 days + // calculate recently changed entries, means changed within the last 8 days recentlyChangedEntries.addAll(extractRecentChangedEntries(cvesForYear)); } catch (Exception ex) { LOG.error("\nERROR processing year {}", currentYear, ex); @@ -299,13 +300,32 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p } } + recentlyChangedEntries = loadExistingModifiedCache(apiBuilder, properties, recentlyChangedEntries); + createCacheRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); return 0; } - private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuilder, int currentYear, - Path cacheFilePath) { + private List loadExistingModifiedCache(NvdCveClientBuilder apiBuilder, CacheProperties properties, List recentlyChangedEntries) { + try { + Path cacheFilePath = buildCacheTargetFileForYear(properties, "modified"); + if (Files.isRegularFile(cacheFilePath)) { + List modified = new ArrayList<>(); + CvesNvdPojo existing = loadExistingCacheAndConfigureApi(apiBuilder, cacheFilePath); + existing.vulnerabilities.stream() + .filter(cve -> ChronoUnit.DAYS.between(cve.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) + .forEach(cve -> modified.add(cve)); + modified.addAll(recentlyChangedEntries); + recentlyChangedEntries = modified; + } + } catch (Exception ex) { + LOG.error("ERROR processing modified changed entries", ex); + } + return recentlyChangedEntries; + } + + private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuilder, Path cacheFilePath) { ZonedDateTime lastUpdateDate = determineExistingCacheFileLastChanged(cacheFilePath); if (lastUpdateDate == null) { // no existing cache exists - nothing to load and nothing to configure. Fetch the entire year. @@ -314,7 +334,7 @@ private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuil // else only fetch entries that have been changed since the last update apiBuilder.withLastModifiedFilter(lastUpdateDate, ZonedDateTime.now()); - LOG.info("INFO Found existing local cache for year {}. Cache was created/updated on {}", currentYear, + LOG.info("INFO Found existing local cache '{}'. Cache was created/updated on {}", cacheFilePath.getFileName(), lastUpdateDate.format(DateTimeFormatter.RFC_1123_DATE_TIME)); LOG.info("INFO Only fetching items that have been changed since then from the API"); @@ -397,7 +417,7 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties } /** - * Create cache holding all CVE items that have been changed within the last 7 days. Creates: + * Create cache holding all CVE items that have been changed within the last 8 days. Creates: * nvdcve-modified.json.gz and nvdcve-modified.meta */ private void createCacheRecentlyChangedCacheFile(List recentlyChanged, CacheProperties properties, @@ -473,6 +493,10 @@ private ZonedDateTime determineExistingCacheFileLastChanged(Path cacheFilePath) ZoneId.systemDefault()); } + private Path buildCacheTargetFileForYear(CacheProperties properties, String year) { + return buildCacheTargetFileForYear(properties, Integer.parseInt(year)); + } + private Path buildCacheTargetFileForYear(CacheProperties properties, int year) { final String prefix = properties.get("prefix", "nvdcve-"); return Path.of(properties.getDirectory().getPath(), prefix + year + ".json.gz"); @@ -485,7 +509,7 @@ private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year private List extractRecentChangedEntries(CvesNvdPojo cvesForYear) { return cvesForYear.vulnerabilities.stream() - .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= 7) + .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) .collect(Collectors.toList()); } From da874289682d0e83d0af098a53c0ab5e59cb2542 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 8 Feb 2025 07:45:44 -0500 Subject: [PATCH 05/10] chore: spotlessApply --- .../jeremylong/vulnz/cli/commands/CveCommand.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index d2cbb5c1..c44699b7 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -307,15 +307,15 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p return 0; } - private List loadExistingModifiedCache(NvdCveClientBuilder apiBuilder, CacheProperties properties, List recentlyChangedEntries) { + private List loadExistingModifiedCache(NvdCveClientBuilder apiBuilder, CacheProperties properties, + List recentlyChangedEntries) { try { Path cacheFilePath = buildCacheTargetFileForYear(properties, "modified"); if (Files.isRegularFile(cacheFilePath)) { List modified = new ArrayList<>(); CvesNvdPojo existing = loadExistingCacheAndConfigureApi(apiBuilder, cacheFilePath); - existing.vulnerabilities.stream() - .filter(cve -> ChronoUnit.DAYS.between(cve.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) - .forEach(cve -> modified.add(cve)); + existing.vulnerabilities.stream().filter(cve -> ChronoUnit.DAYS.between(cve.getCve().getLastModified(), + ZonedDateTime.now()) <= EIGHT_DAYS).forEach(cve -> modified.add(cve)); modified.addAll(recentlyChangedEntries); recentlyChangedEntries = modified; } @@ -508,8 +508,8 @@ private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year } private List extractRecentChangedEntries(CvesNvdPojo cvesForYear) { - return cvesForYear.vulnerabilities.stream() - .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) + return cvesForYear.vulnerabilities.stream().filter( + item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) .collect(Collectors.toList()); } From f880f3990ee17c5d6976c30844c698ea321b969c Mon Sep 17 00:00:00 2001 From: em Date: Sun, 9 Feb 2025 10:11:24 +0100 Subject: [PATCH 06/10] Revert "chore: spotlessApply" This reverts commit da874289682d0e83d0af098a53c0ab5e59cb2542. --- .../jeremylong/vulnz/cli/commands/CveCommand.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index c44699b7..d2cbb5c1 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -307,15 +307,15 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p return 0; } - private List loadExistingModifiedCache(NvdCveClientBuilder apiBuilder, CacheProperties properties, - List recentlyChangedEntries) { + private List loadExistingModifiedCache(NvdCveClientBuilder apiBuilder, CacheProperties properties, List recentlyChangedEntries) { try { Path cacheFilePath = buildCacheTargetFileForYear(properties, "modified"); if (Files.isRegularFile(cacheFilePath)) { List modified = new ArrayList<>(); CvesNvdPojo existing = loadExistingCacheAndConfigureApi(apiBuilder, cacheFilePath); - existing.vulnerabilities.stream().filter(cve -> ChronoUnit.DAYS.between(cve.getCve().getLastModified(), - ZonedDateTime.now()) <= EIGHT_DAYS).forEach(cve -> modified.add(cve)); + existing.vulnerabilities.stream() + .filter(cve -> ChronoUnit.DAYS.between(cve.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) + .forEach(cve -> modified.add(cve)); modified.addAll(recentlyChangedEntries); recentlyChangedEntries = modified; } @@ -508,8 +508,8 @@ private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year } private List extractRecentChangedEntries(CvesNvdPojo cvesForYear) { - return cvesForYear.vulnerabilities.stream().filter( - item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) + return cvesForYear.vulnerabilities.stream() + .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) .collect(Collectors.toList()); } From f4560d6908b7e786411e336e6a42a0dd1bc3fe82 Mon Sep 17 00:00:00 2001 From: em Date: Sun, 9 Feb 2025 10:11:39 +0100 Subject: [PATCH 07/10] Revert "fix: preserve modified entries if year fails" This reverts commit d898893ca536ca58d18d35dd20186f90324e8ba3. --- .../vulnz/cli/commands/CveCommand.java | 42 ++++--------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index d2cbb5c1..e1b1e15d 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -38,7 +38,6 @@ import io.github.jeremylong.vulnz.cli.ui.ProgressMonitor; import io.prometheus.metrics.core.metrics.Gauge; import java.io.*; -import java.nio.file.Files; import java.nio.file.Path; import java.time.*; import java.time.temporal.ChronoUnit; @@ -86,7 +85,6 @@ public class CveCommand extends AbstractNvdCommand { .help("Total number of loaded cve's").register(); private static final Gauge CVE_COUNTER = Gauge.builder().name("cve_counter").help("Total number of cached cve's") .register(); - public static final int EIGHT_DAYS = 8; @CommandLine.ArgGroup(exclusive = true) ConfigGroup configGroup; @@ -258,7 +256,7 @@ public Integer timedCall() throws Exception { * For all years, fetch the NVD vulnerabilities and save them each into one cache file. */ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties properties) { - // will hold all entries that have been changed within the last 8 days across all years + // will hold all entries that have been changed within the last 7 days across all years List recentlyChangedEntries = new ArrayList<>(); for (int currentYear = START_YEAR; currentYear <= Year.now().getValue(); currentYear++) { @@ -271,7 +269,8 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p Path cacheFilePath = buildCacheTargetFileForYear(properties, currentYear); LOG.info("INFO *** Processing year {} ***", currentYear); - CvesNvdPojo existingCacheData = loadExistingCacheAndConfigureApi(apiBuilder, cacheFilePath); + CvesNvdPojo existingCacheData = loadExistingCacheAndConfigureApi(apiBuilder, currentYear, + cacheFilePath); CvesNvdPojo cvesForYear = aggregateCvesForYear(currentYear, apiBuilder); // merge old with new data. It is intended to add old items to the new ones, thus we keep newly fetched @@ -292,7 +291,7 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p LOG.info("INFO *** Finished year {} with #{} entries ***", currentYear, cvesForYear.vulnerabilities.size()); - // calculate recently changed entries, means changed within the last 8 days + // calculate recently changed entries, means changed within the last 7 days recentlyChangedEntries.addAll(extractRecentChangedEntries(cvesForYear)); } catch (Exception ex) { LOG.error("\nERROR processing year {}", currentYear, ex); @@ -300,32 +299,13 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p } } - recentlyChangedEntries = loadExistingModifiedCache(apiBuilder, properties, recentlyChangedEntries); - createCacheRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); return 0; } - private List loadExistingModifiedCache(NvdCveClientBuilder apiBuilder, CacheProperties properties, List recentlyChangedEntries) { - try { - Path cacheFilePath = buildCacheTargetFileForYear(properties, "modified"); - if (Files.isRegularFile(cacheFilePath)) { - List modified = new ArrayList<>(); - CvesNvdPojo existing = loadExistingCacheAndConfigureApi(apiBuilder, cacheFilePath); - existing.vulnerabilities.stream() - .filter(cve -> ChronoUnit.DAYS.between(cve.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) - .forEach(cve -> modified.add(cve)); - modified.addAll(recentlyChangedEntries); - recentlyChangedEntries = modified; - } - } catch (Exception ex) { - LOG.error("ERROR processing modified changed entries", ex); - } - return recentlyChangedEntries; - } - - private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuilder, Path cacheFilePath) { + private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuilder, int currentYear, + Path cacheFilePath) { ZonedDateTime lastUpdateDate = determineExistingCacheFileLastChanged(cacheFilePath); if (lastUpdateDate == null) { // no existing cache exists - nothing to load and nothing to configure. Fetch the entire year. @@ -334,7 +314,7 @@ private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuil // else only fetch entries that have been changed since the last update apiBuilder.withLastModifiedFilter(lastUpdateDate, ZonedDateTime.now()); - LOG.info("INFO Found existing local cache '{}'. Cache was created/updated on {}", cacheFilePath.getFileName(), + LOG.info("INFO Found existing local cache for year {}. Cache was created/updated on {}", currentYear, lastUpdateDate.format(DateTimeFormatter.RFC_1123_DATE_TIME)); LOG.info("INFO Only fetching items that have been changed since then from the API"); @@ -417,7 +397,7 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties } /** - * Create cache holding all CVE items that have been changed within the last 8 days. Creates: + * Create cache holding all CVE items that have been changed within the last 7 days. Creates: * nvdcve-modified.json.gz and nvdcve-modified.meta */ private void createCacheRecentlyChangedCacheFile(List recentlyChanged, CacheProperties properties, @@ -493,10 +473,6 @@ private ZonedDateTime determineExistingCacheFileLastChanged(Path cacheFilePath) ZoneId.systemDefault()); } - private Path buildCacheTargetFileForYear(CacheProperties properties, String year) { - return buildCacheTargetFileForYear(properties, Integer.parseInt(year)); - } - private Path buildCacheTargetFileForYear(CacheProperties properties, int year) { final String prefix = properties.get("prefix", "nvdcve-"); return Path.of(properties.getDirectory().getPath(), prefix + year + ".json.gz"); @@ -509,7 +485,7 @@ private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year private List extractRecentChangedEntries(CvesNvdPojo cvesForYear) { return cvesForYear.vulnerabilities.stream() - .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= EIGHT_DAYS) + .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= 7) .collect(Collectors.toList()); } From 8b4d1e896c4cd354cd01d0898a0ce0b774297dc9 Mon Sep 17 00:00:00 2001 From: em Date: Sun, 9 Feb 2025 12:41:42 +0100 Subject: [PATCH 08/10] Ensure we use old modified entries when rebuilding the modified cache --- .../vulnz/cli/commands/CveCommand.java | 66 ++++++++++++------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index e1b1e15d..ec9e23eb 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -80,6 +80,7 @@ public class CveCommand extends AbstractNvdCommand { * Hex code characters used in getHex. */ private static final String HEXES = "0123456789abcdef"; + private static final long MAX_AGE_OF_MODIFIED_DIFF = 8; private static final Gauge CVE_LOAD_COUNTER = Gauge.builder().name("cve_load_counter") .help("Total number of loaded cve's").register(); @@ -256,7 +257,7 @@ public Integer timedCall() throws Exception { * For all years, fetch the NVD vulnerabilities and save them each into one cache file. */ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties properties) { - // will hold all entries that have been changed within the last 7 days across all years + // will hold all entries that have been changed within the last8 days across all years List recentlyChangedEntries = new ArrayList<>(); for (int currentYear = START_YEAR; currentYear <= Year.now().getValue(); currentYear++) { @@ -291,15 +292,16 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p LOG.info("INFO *** Finished year {} with #{} entries ***", currentYear, cvesForYear.vulnerabilities.size()); - // calculate recently changed entries, means changed within the last 7 days - recentlyChangedEntries.addAll(extractRecentChangedEntries(cvesForYear)); + // remove all entries older than MAX_AGE_OF_MODIFIED_DIFF days + removeOutdatedCves(cvesForYear.vulnerabilities); + recentlyChangedEntries.addAll(cvesForYear.vulnerabilities); } catch (Exception ex) { LOG.error("\nERROR processing year {}", currentYear, ex); LOG.info("INFO ... continuing with next year"); } } - createCacheRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); + saveCacheRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); return 0; } @@ -319,14 +321,7 @@ private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuil LOG.info("INFO Only fetching items that have been changed since then from the API"); // Load existing cache data - ObjectMapper objectMapper = getCacheObjectMapper(); - try (FileInputStream fis = new FileInputStream(cacheFilePath.toFile()); - GZIPInputStream gzis = new GZIPInputStream(fis)) { - CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); - return new CvesNvdPojo(data.getVulnerabilities(), data.getTimestamp()); - } catch (IOException exception) { - throw new CacheException("Unable to read cached data: " + cacheFilePath, exception); - } + return loadCvesFromCache(cacheFilePath); } /** @@ -389,7 +384,7 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties // save vulnerabilities into cache final File cacheFile = buildCacheTargetFileForYear(properties, year).toFile(); - long uncompressedSize = saveCacheAsGzip(cacheFile, data, md); + long uncompressedSize = saveCvestoCache(cacheFile, data, md); // save meta data final File metaDataFile = buildMetaDataTargetFileForYear(properties, year).toFile(); @@ -397,22 +392,31 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties } /** - * Create cache holding all CVE items that have been changed within the last 7 days. Creates: + * Create cache holding all CVE items that have been changed within the last 8 days. Creates: * nvdcve-modified.json.gz and nvdcve-modified.meta */ - private void createCacheRecentlyChangedCacheFile(List recentlyChanged, CacheProperties properties, + private void saveCacheRecentlyChangedCacheFile(List recentlyChanged, CacheProperties properties, ZonedDateTime lastChanged) { final String prefix = properties.get("prefix", "nvdcve-"); Path recentChangesCachePath = Path.of(properties.getDirectory().getPath(), prefix + "modified.json.gz"); - int recentSize = recentlyChanged.size(); - CveApiJson20 data = new CveApiJson20(recentSize, 0, recentSize, FORMAT, VERSION, lastChanged, recentlyChanged); + + // pre-load the old modified + var oldModifiedCacheData = loadCvesFromCache(recentChangesCachePath); + + // merge newly entries with the previously stored ones + oldModifiedCacheData.vulnerabilities.addAll(recentlyChanged); + // ensure we do not include a slice more the + removeOutdatedCves(oldModifiedCacheData.vulnerabilities); + + int recentSize = oldModifiedCacheData.vulnerabilities.size(); + CveApiJson20 data = new CveApiJson20(recentSize, 0, recentSize, FORMAT, VERSION, lastChanged, oldModifiedCacheData.vulnerabilities); MessageDigest md = getDigestAlg(); // create cache file including the CVE entries that recently changed File recentCacheFile = recentChangesCachePath.toFile(); - long uncompressedSize = saveCacheAsGzip(recentCacheFile, data, md); + long uncompressedSize = saveCvestoCache(recentCacheFile, data, md); LOG.info("INFO Stored {} entries in {} as recent changed items across all years", data.getVulnerabilities().size(), recentCacheFile.getName()); @@ -421,7 +425,19 @@ private void createCacheRecentlyChangedCacheFile(List recentlyChange saveMetaData(metaDataFile.toFile(), recentCacheFile.length(), uncompressedSize, lastChanged, md); } - private long saveCacheAsGzip(File targetFile, CveApiJson20 data, MessageDigest md) { + private CvesNvdPojo loadCvesFromCache(Path cacheFilePath) { + // Load existing cache data + ObjectMapper objectMapper = getCacheObjectMapper(); + try (FileInputStream fis = new FileInputStream(cacheFilePath.toFile()); + GZIPInputStream gzis = new GZIPInputStream(fis)) { + CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); + return new CvesNvdPojo(data.getVulnerabilities(), data.getTimestamp()); + } catch (IOException exception) { + throw new CacheException("Unable to read cached data: " + cacheFilePath, exception); + } + } + + private long saveCvestoCache(File targetFile, CveApiJson20 data, MessageDigest md) { final ObjectMapper objectMapper = getCacheObjectMapper(); long uncompressedSize; @@ -483,10 +499,14 @@ private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year return Path.of(properties.getDirectory().getPath(), prefix + year + ".meta"); } - private List extractRecentChangedEntries(CvesNvdPojo cvesForYear) { - return cvesForYear.vulnerabilities.stream() - .filter(item -> ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= 7) - .collect(Collectors.toList()); + /** + * removes all entries that are older then MAX_AGE_OF_MODIFIED_DIFF - operators by ref to reduce copies + */ + private void removeOutdatedCves(List cves) { + cves.removeIf((item) -> + // remove all items that are older than 8 days + !(ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= MAX_AGE_OF_MODIFIED_DIFF) + ); } /** From b256bd86e06c8d282c5e4a707cc6c51701cb3802 Mon Sep 17 00:00:00 2001 From: em Date: Sun, 9 Feb 2025 16:48:41 +0100 Subject: [PATCH 09/10] run spotless --- .../vulnz/cli/commands/CveCommand.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index ec9e23eb..c5acff34 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -400,7 +400,6 @@ private void saveCacheRecentlyChangedCacheFile(List recentlyChanged, final String prefix = properties.get("prefix", "nvdcve-"); Path recentChangesCachePath = Path.of(properties.getDirectory().getPath(), prefix + "modified.json.gz"); - // pre-load the old modified var oldModifiedCacheData = loadCvesFromCache(recentChangesCachePath); @@ -410,7 +409,8 @@ private void saveCacheRecentlyChangedCacheFile(List recentlyChanged, removeOutdatedCves(oldModifiedCacheData.vulnerabilities); int recentSize = oldModifiedCacheData.vulnerabilities.size(); - CveApiJson20 data = new CveApiJson20(recentSize, 0, recentSize, FORMAT, VERSION, lastChanged, oldModifiedCacheData.vulnerabilities); + CveApiJson20 data = new CveApiJson20(recentSize, 0, recentSize, FORMAT, VERSION, lastChanged, + oldModifiedCacheData.vulnerabilities); MessageDigest md = getDigestAlg(); @@ -426,15 +426,15 @@ private void saveCacheRecentlyChangedCacheFile(List recentlyChanged, } private CvesNvdPojo loadCvesFromCache(Path cacheFilePath) { - // Load existing cache data - ObjectMapper objectMapper = getCacheObjectMapper(); - try (FileInputStream fis = new FileInputStream(cacheFilePath.toFile()); - GZIPInputStream gzis = new GZIPInputStream(fis)) { - CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); - return new CvesNvdPojo(data.getVulnerabilities(), data.getTimestamp()); - } catch (IOException exception) { - throw new CacheException("Unable to read cached data: " + cacheFilePath, exception); - } + // Load existing cache data + ObjectMapper objectMapper = getCacheObjectMapper(); + try (FileInputStream fis = new FileInputStream(cacheFilePath.toFile()); + GZIPInputStream gzis = new GZIPInputStream(fis)) { + CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); + return new CvesNvdPojo(data.getVulnerabilities(), data.getTimestamp()); + } catch (IOException exception) { + throw new CacheException("Unable to read cached data: " + cacheFilePath, exception); + } } private long saveCvestoCache(File targetFile, CveApiJson20 data, MessageDigest md) { @@ -504,9 +504,8 @@ private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year */ private void removeOutdatedCves(List cves) { cves.removeIf((item) -> - // remove all items that are older than 8 days - !(ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= MAX_AGE_OF_MODIFIED_DIFF) - ); + // remove all items that are older than 8 days + !(ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= MAX_AGE_OF_MODIFIED_DIFF)); } /** From 55a4bcd25e8dd5d3b459351455a47a494a6b831e Mon Sep 17 00:00:00 2001 From: em Date: Sun, 9 Feb 2025 17:06:31 +0100 Subject: [PATCH 10/10] polish docs, add exit code --- .../vulnz/cli/commands/CveCommand.java | 75 ++++++++++++------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index c5acff34..650025fa 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -72,14 +72,33 @@ public class CveCommand extends AbstractNvdCommand { */ private static final int START_YEAR = 2002; - // TODO - get format and version from API + // FIXME - get format and version from API private static final String FORMAT = "NVD_CVE"; private static final String VERSION = "2.0"; + /** + * by NVDs API limit, 120 is the max range + */ + private static final int NVD_API_MAX_DAYS_RANGE = 115; + + private static final int EXIT_CODE_SUCCESS = 0; + /** + * At least one year failed to fetch + */ + private static final int EXIT_CODE_PARTIAL_ERROR_YEAR = 100; + + /** + * At least one year failed to fetch + */ + private static final int EXIT_CODE_FATAL_ERROR_RECENT_MODIFIED = 101; /** * Hex code characters used in getHex. */ private static final String HEXES = "0123456789abcdef"; + + /** + * Defines how many days we look into the past when aggregating the recent-modified cache + */ private static final long MAX_AGE_OF_MODIFIED_DIFF = 8; private static final Gauge CVE_LOAD_COUNTER = Gauge.builder().name("cve_load_counter") @@ -257,6 +276,7 @@ public Integer timedCall() throws Exception { * For all years, fetch the NVD vulnerabilities and save them each into one cache file. */ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties properties) { + int exitCode = EXIT_CODE_SUCCESS; // will hold all entries that have been changed within the last8 days across all years List recentlyChangedEntries = new ArrayList<>(); @@ -296,14 +316,20 @@ private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties p removeOutdatedCves(cvesForYear.vulnerabilities); recentlyChangedEntries.addAll(cvesForYear.vulnerabilities); } catch (Exception ex) { - LOG.error("\nERROR processing year {}", currentYear, ex); + LOG.error("\nPARTIAL-ERROR processing year {}", currentYear, ex); LOG.info("INFO ... continuing with next year"); + exitCode = EXIT_CODE_PARTIAL_ERROR_YEAR; } } - saveCacheRecentlyChangedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); + try { + saveRecentlyModifiedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); + } catch (Exception ex) { + LOG.error("\nERROR saving recent-modified file", ex); + exitCode = EXIT_CODE_FATAL_ERROR_RECENT_MODIFIED; + } - return 0; + return exitCode; } private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuilder, int currentYear, @@ -329,29 +355,28 @@ private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuil * range of 120 days (due to NVD api limits). Ensures vulnerabilities are unique and sorted by publishing date, ASC */ private CvesNvdPojo aggregateCvesForYear(int year, NvdCveClientBuilder apiBuilder) { - // by NVDs API limit, 120 is the max range - final int MAX_DAYS_RANGE = 115; - int dayOfYearStart = 1; + int sliceStartDay = 1; - ZonedDateTime yearStart = ZonedDateTime.of(year, 1, 1, 0, 0, 0, 0, ZoneId.systemDefault()); - int yearLengthInDays = Year.from(yearStart).length(); + ZonedDateTime yearFirstDay = ZonedDateTime.of(year, 1, 1, 0, 0, 0, 0, ZoneId.systemDefault()); + int yearLengthInDays = Year.from(yearFirstDay).length(); CvesNvdPojo finalResult = new CvesNvdPojo(new ArrayList<>(), ZonedDateTime.now()); - while (dayOfYearStart < yearLengthInDays) { + while (sliceStartDay < yearLengthInDays) { // first day of the year we start from - ZonedDateTime from = yearStart.plusDays(dayOfYearStart); + ZonedDateTime from = yearFirstDay.plusDays(sliceStartDay); if (from.isAfter(ZonedDateTime.now())) { // we are fetching for items in the future, stop here break; } - // get the end day of the range, but ensure we do not overshoot - int dayOfYearEnd = Math.min(dayOfYearStart + MAX_DAYS_RANGE, yearLengthInDays - 1); - ZonedDateTime to = yearStart.plusDays(dayOfYearEnd) + // get the end day of the range, but ensure we do not overshoot + int sliceEndDay = Math.min(sliceStartDay + NVD_API_MAX_DAYS_RANGE, yearLengthInDays - 1); + ZonedDateTime to = yearFirstDay.plusDays(sliceEndDay) // since we need the end of the day, just pick the next one and reset time to 00:00 .plusDays(1).truncatedTo(ChronoUnit.DAYS); - // apply our range to the API call + + // apply our "year slice" range to the API call apiBuilder.withPublishedDateFilter(from, to); // fetch entries for this range @@ -362,8 +387,8 @@ private CvesNvdPojo aggregateCvesForYear(int year, NvdCveClientBuilder apiBuilde // aggregate all results finalResult.vulnerabilities.addAll(currentResult.vulnerabilities); - // let the next fetch start on the following day we ended on before (since we fetched up to 'to midnight' - dayOfYearStart = dayOfYearEnd + 1; + // let the next fetch start on the following day we ended on before (since we fetched up to 'to midnight') + sliceStartDay = sliceEndDay + 1; } return finalResult; @@ -377,14 +402,12 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties ZonedDateTime lastChanged = Objects.requireNonNullElseGet(cves.lastUpdated, ZonedDateTime::now); properties.set("lastModifiedDate." + year, lastChanged); int size = cves.vulnerabilities.size(); - - CveApiJson20 data = new CveApiJson20(size, 0, size, FORMAT, VERSION, lastChanged, cves.vulnerabilities); - MessageDigest md = getDigestAlg(); // save vulnerabilities into cache + CveApiJson20 data = new CveApiJson20(size, 0, size, FORMAT, VERSION, lastChanged, cves.vulnerabilities); final File cacheFile = buildCacheTargetFileForYear(properties, year).toFile(); - long uncompressedSize = saveCvestoCache(cacheFile, data, md); + long uncompressedSize = saveCvesToCache(cacheFile, data, md); // save meta data final File metaDataFile = buildMetaDataTargetFileForYear(properties, year).toFile(); @@ -395,12 +418,12 @@ private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties * Create cache holding all CVE items that have been changed within the last 8 days. Creates: * nvdcve-modified.json.gz and nvdcve-modified.meta */ - private void saveCacheRecentlyChangedCacheFile(List recentlyChanged, CacheProperties properties, + private void saveRecentlyModifiedCacheFile(List recentlyChanged, CacheProperties properties, ZonedDateTime lastChanged) { final String prefix = properties.get("prefix", "nvdcve-"); Path recentChangesCachePath = Path.of(properties.getDirectory().getPath(), prefix + "modified.json.gz"); - // pre-load the old modified + // preload the old modified var oldModifiedCacheData = loadCvesFromCache(recentChangesCachePath); // merge newly entries with the previously stored ones @@ -416,7 +439,7 @@ private void saveCacheRecentlyChangedCacheFile(List recentlyChanged, // create cache file including the CVE entries that recently changed File recentCacheFile = recentChangesCachePath.toFile(); - long uncompressedSize = saveCvestoCache(recentCacheFile, data, md); + long uncompressedSize = saveCvesToCache(recentCacheFile, data, md); LOG.info("INFO Stored {} entries in {} as recent changed items across all years", data.getVulnerabilities().size(), recentCacheFile.getName()); @@ -437,7 +460,7 @@ private CvesNvdPojo loadCvesFromCache(Path cacheFilePath) { } } - private long saveCvestoCache(File targetFile, CveApiJson20 data, MessageDigest md) { + private long saveCvesToCache(File targetFile, CveApiJson20 data, MessageDigest md) { final ObjectMapper objectMapper = getCacheObjectMapper(); long uncompressedSize; @@ -500,7 +523,7 @@ private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year } /** - * removes all entries that are older then MAX_AGE_OF_MODIFIED_DIFF - operators by ref to reduce copies + * removes all entries that are older than MAX_AGE_OF_MODIFIED_DIFF - operates on the reference to reduce copies */ private void removeOutdatedCves(List cves) { cves.removeIf((item) ->