diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/HttpAsyncClientSupplier.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/HttpAsyncClientSupplier.java new file mode 100644 index 00000000..a6643928 --- /dev/null +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/HttpAsyncClientSupplier.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Jeremy Long. All Rights Reserved. + */ +package io.github.jeremylong.openvulnerability.client; + +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.async.HttpAsyncClients; +import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; + +import java.net.ProxySelector; +import java.util.function.Supplier; + +/** + * Supplier for {@link CloseableHttpAsyncClient}s. + *

+ * May be used to provide customized HTTP clients to data source clients. + *

+ * Closing of the supplied {@link CloseableHttpAsyncClient} instances is a responsibility of the caller. + */ +@FunctionalInterface +public interface HttpAsyncClientSupplier extends Supplier { + + static HttpAsyncClientSupplier getDefault() { + return () -> { + SystemDefaultRoutePlanner planner = new SystemDefaultRoutePlanner(ProxySelector.getDefault()); + return HttpAsyncClients.custom() + .setRoutePlanner(planner) + .useSystemProperties() + .build(); + }; + } + +} diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/HttpClientSupplier.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/HttpClientSupplier.java new file mode 100644 index 00000000..714595b3 --- /dev/null +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/HttpClientSupplier.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2023 Jeremy Long. All Rights Reserved. + */ +package io.github.jeremylong.openvulnerability.client; + +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; + +import java.net.ProxySelector; +import java.util.function.Supplier; + +/** + * Supplier for {@link CloseableHttpClient}s. + *

+ * May be used to provide customized HTTP clients to data source clients. + *

+ * Closing of the supplied {@link CloseableHttpClient} instances is a responsibility of the caller. + */ +@FunctionalInterface +public interface HttpClientSupplier extends Supplier { + + static HttpClientSupplier getDefault() { + return () -> { + SystemDefaultRoutePlanner planner = new SystemDefaultRoutePlanner(ProxySelector.getDefault()); + return HttpClientBuilder.create() + .setRoutePlanner(planner) + .useSystemProperties() + .build(); + }; + } + +} diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/epss/EpssDataFeed.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/epss/EpssDataFeed.java index 21d16e2e..a9307ef9 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/epss/EpssDataFeed.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/epss/EpssDataFeed.java @@ -17,13 +17,11 @@ package io.github.jeremylong.openvulnerability.client.epss; import io.github.jeremylong.openvulnerability.client.DataFeed; +import io.github.jeremylong.openvulnerability.client.HttpClientSupplier; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; import java.io.IOException; -import java.net.ProxySelector; import java.util.List; /** @@ -35,23 +33,27 @@ public class EpssDataFeed implements DataFeed> { private final static String DEFAULT_LOCATION = "https://epss.cyentia.com/epss_scores-current.csv.gz"; + private final HttpClientSupplier httpClientSupplier; private final String downloadUrl; public EpssDataFeed() { - this.downloadUrl = DEFAULT_LOCATION; + this(DEFAULT_LOCATION); } public EpssDataFeed(String downloadUrl) { + this(downloadUrl, null); + } + + public EpssDataFeed(String downloadUrl, HttpClientSupplier httpClientSupplier) { this.downloadUrl = downloadUrl; + this.httpClientSupplier = httpClientSupplier != null ? httpClientSupplier : HttpClientSupplier.getDefault(); } @Override public List download() { List list = null; HttpGet request = new HttpGet(downloadUrl); - SystemDefaultRoutePlanner planner = new SystemDefaultRoutePlanner(ProxySelector.getDefault()); - try (CloseableHttpClient client = HttpClientBuilder.create().setRoutePlanner(planner).useSystemProperties() - .build()) { + try (CloseableHttpClient client = httpClientSupplier.get()) { list = client.execute(request, new EpssResponseHandler()); } catch (IOException e) { e.printStackTrace(); diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/ghsa/GitHubSecurityAdvisoryClient.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/ghsa/GitHubSecurityAdvisoryClient.java index 47c5eb32..c395b6aa 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/ghsa/GitHubSecurityAdvisoryClient.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/ghsa/GitHubSecurityAdvisoryClient.java @@ -22,13 +22,12 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.samskivert.mustache.Mustache; import com.samskivert.mustache.Template; +import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier; import io.github.jeremylong.openvulnerability.client.PagedDataSource; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; -import org.apache.hc.client5.http.impl.async.HttpAsyncClients; -import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; import org.apache.hc.core5.http.ContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +36,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.ProxySelector; import java.nio.charset.StandardCharsets; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -141,7 +139,7 @@ public class GitHubSecurityAdvisoryClient implements PagedDataSource { * Jackson object mapper. */ private final ObjectMapper objectMapper; + private final HttpClientSupplier httpClientSupplier; private final String downloadUrl; public KevDataFeed() { @@ -48,7 +47,12 @@ public KevDataFeed() { } public KevDataFeed(String downloadUrl) { + this(downloadUrl, null); + } + + public KevDataFeed(String downloadUrl, HttpClientSupplier httpClientSupplier) { this.downloadUrl = downloadUrl; + this.httpClientSupplier = httpClientSupplier != null ? httpClientSupplier : HttpClientSupplier.getDefault(); objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); } @@ -56,10 +60,8 @@ public KevDataFeed(String downloadUrl) { @Override public KevCatalog download() { HttpGet request = new HttpGet(downloadUrl); - SystemDefaultRoutePlanner planner = new SystemDefaultRoutePlanner(ProxySelector.getDefault()); String json; - try (CloseableHttpClient client = HttpClientBuilder.create().setRoutePlanner(planner).useSystemProperties() - .build()) { + try (CloseableHttpClient client = httpClientSupplier.get()) { json = client.execute(request, new BasicHttpClientResponseHandler()); } catch (IOException e) { throw new KevException("Unable to download the Known Exploitable Vulnerability Catalog", e); diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java index 9616974f..5b1fa87e 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier; import io.github.jeremylong.openvulnerability.client.PagedDataSource; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; @@ -133,7 +134,7 @@ public class NvdCveClient implements PagedDataSource { * @param maxPageCount the maximum number of pages to retrieve from the NVD API. */ NvdCveClient(String apiKey, String endpoint, int threadCount, int maxPageCount) { - this(apiKey, endpoint, apiKey == null ? 6500 : 600, threadCount, maxPageCount, 10); + this(apiKey, endpoint, 0, threadCount, maxPageCount, 10, null); } /** @@ -146,7 +147,7 @@ public class NvdCveClient implements PagedDataSource { * @param maxRetryCount the maximum number of retries for 503 and 429 status code responses. */ NvdCveClient(String apiKey, String endpoint, int threadCount, int maxPageCount, int maxRetryCount) { - this(apiKey, endpoint, apiKey == null ? 6500 : 600, threadCount, maxPageCount, maxRetryCount); + this(apiKey, endpoint, 0, threadCount, maxPageCount, maxRetryCount, null); } /** @@ -158,8 +159,10 @@ public class NvdCveClient implements PagedDataSource { * @param threadCount the number of threads to use when calling the NVD API. * @param maxPageCount the maximum number of pages to retrieve from the NVD API. * @param maxRetryCount the maximum number of retries for 503 and 429 status code responses. + * @param httpClientSupplier supplier for custom HTTP clients; if {@code null} a default client will be used */ - NvdCveClient(String apiKey, String endpoint, long delay, int threadCount, int maxPageCount, int maxRetryCount) { + NvdCveClient(String apiKey, String endpoint, long delay, int threadCount, int maxPageCount, int maxRetryCount, + HttpAsyncClientSupplier httpClientSupplier) { this.apiKey = apiKey; if (endpoint == null) { this.endpoint = DEFAULT_ENDPOINT; @@ -186,8 +189,11 @@ public class NvdCveClient implements PagedDataSource { meter = new RateMeter(50, 32500); } clients = new ArrayList<>(threadCount); + if (delay == 0) { + delay = apiKey == null ? 6500 : 600; + } for (int i = 0; i < threadCount; i++) { - clients.add(new RateLimitedClient(maxRetryCount, delay, meter)); + clients.add(new RateLimitedClient(maxRetryCount, delay, meter, httpClientSupplier)); } objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java index 91732003..bb6dbdda 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java @@ -16,6 +16,7 @@ */ package io.github.jeremylong.openvulnerability.client.nvd; +import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier; import org.apache.hc.core5.http.NameValuePair; import org.apache.hc.core5.http.message.BasicNameValuePair; import org.slf4j.Logger; @@ -76,6 +77,7 @@ public final class NvdCveClientBuilder { * The maximum number of pages to retrieve from the NVD API. */ private int maxPageCount = 0; + private HttpAsyncClientSupplier httpClientSupplier; /** * Private constructor for a builder. @@ -329,18 +331,29 @@ public NvdCveClientBuilder withVersionEnd(String versionEnd, VersionType endType return this; } + /** + * Provide a supplier for custom HTTP clients. + *

+ * Note that {@link #withDelay(long)} and {@link #withMaxRetryCount(int)} have no effect when + * a custom {@link HttpAsyncClientSupplier} is provided. Instead, clients created by the supplier + * should be configured to use {@link NvdApiRetryStrategy} with the desired delay and retry count values. + * + * @param httpClientSupplier supplier for custom HTTP clients; if {@code null} a default client will be used + * @return the builder + */ + public NvdCveClientBuilder withHttpClientSupplier(final HttpAsyncClientSupplier httpClientSupplier) { + this.httpClientSupplier = httpClientSupplier; + return this; + } + /** * Build the NVD CVE API client. * * @return the NVD CVE API client */ public NvdCveClient build() { - NvdCveClient client; - if (delay > 0) { - client = new NvdCveClient(apiKey, endpoint, delay, threadCount, maxPageCount, maxRetryCount); - } else { - client = new NvdCveClient(apiKey, endpoint, threadCount, maxPageCount, maxRetryCount); - } + NvdCveClient client = new NvdCveClient(apiKey, endpoint, delay, threadCount, maxPageCount, maxRetryCount, + httpClientSupplier); if (!filters.isEmpty()) { client.setFilters(filters); } diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/RateLimitedClient.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/RateLimitedClient.java index a9591678..cc349bf2 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/RateLimitedClient.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/RateLimitedClient.java @@ -16,6 +16,7 @@ */ package io.github.jeremylong.openvulnerability.client.nvd; +import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; @@ -107,7 +108,7 @@ class RateLimitedClient implements AutoCloseable { * @param meter the rate meter to limit the request rate */ RateLimitedClient(long minimumDelay, RateMeter meter) { - this(10, minimumDelay, meter); + this(10, minimumDelay, meter, null); } /** @@ -118,15 +119,19 @@ class RateLimitedClient implements AutoCloseable { * @param maxRetries the maximum number of retry attemps * @param minimumDelay the number of milliseconds to wait between API calls * @param meter the rate meter to limit the request rate + * @param httpClientSupplier supplier for custom HTTP clients; if {@code null} a default client will be used */ - RateLimitedClient(int maxRetries, long minimumDelay, RateMeter meter) { + RateLimitedClient(int maxRetries, long minimumDelay, RateMeter meter, HttpAsyncClientSupplier httpClientSupplier) { this.meter = meter; this.delay = minimumDelay; LOG.debug("rate limited call delay: {}", delay); - NvdApiRetryStrategy retryStrategy = new NvdApiRetryStrategy(maxRetries, minimumDelay); - SystemDefaultRoutePlanner planner = new SystemDefaultRoutePlanner(ProxySelector.getDefault()); - client = HttpAsyncClients.custom().setRoutePlanner(planner).setRetryStrategy(retryStrategy) - .useSystemProperties().build(); + if (httpClientSupplier == null) { + NvdApiRetryStrategy retryStrategy = new NvdApiRetryStrategy(maxRetries, minimumDelay); + SystemDefaultRoutePlanner planner = new SystemDefaultRoutePlanner(ProxySelector.getDefault()); + client = HttpAsyncClients.custom().setRoutePlanner(planner).setRetryStrategy(retryStrategy).build(); + } else { + client = httpClientSupplier.get(); + } client.start(); }