Downloader.java
/*
* This file is part of dependency-check-utils.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2024 Hans Aikema. All Rights Reserved.
*/
package org.owasp.dependencycheck.utils;
import org.apache.hc.client5.http.HttpResponseException;
import org.apache.hc.client5.http.auth.AuthCache;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.Credentials;
import org.apache.hc.client5.http.auth.CredentialsStore;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
import org.apache.hc.client5.http.impl.auth.BasicScheme;
import org.apache.hc.client5.http.impl.auth.SystemDefaultCredentialsProvider;
import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.ClassicHttpResponse;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpException;
import org.apache.hc.core5.http.HttpHeaders;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.Method;
import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.BasicHttpEntity;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicClassicHttpRequest;
import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLHandshakeException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.ProxySelector;
import java.net.SocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static java.lang.String.format;
/**
* A Utility class to centralize download logic like HTTP(S) proxy configuration and proxy- and server credential handling.
* @author Jeremy Long, Hans Aikema
*/
public final class Downloader {
/**
* The builder to use for a HTTP Client that uses the configured proxy-settings
*/
private final HttpClientBuilder httpClientBuilder;
/**
* The builder to use for a HTTP Client that explicitly opts out of proxy-usage
*/
private final HttpClientBuilder httpClientBuilderExplicitNoproxy;
/**
* The Authentication cache for pre-emptive authentication.
* This gets filled with credentials from the settings in {@link #configure(Settings)}.
*/
private final AuthCache authCache = new BasicAuthCache();
/**
* The credentialsProvider for pre-emptive authentication.
* This gets filled with credentials from the settings in {@link #configure(Settings)}.
*/
private final SystemDefaultCredentialsProvider credentialsProvider = new SystemDefaultCredentialsProvider();
/**
* The settings
*/
private Settings settings;
/**
* The Logger for use throughout the class.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class);
/**
* The singleton instance of the downloader
*/
private static final Downloader INSTANCE = new Downloader();
/**
* The Credentials for the proxy when proxy authentication is configured in the Settings.
*/
private Credentials proxyCreds = null;
/**
* A BasicScheme initialized with the proxy-credentials when proxy authentication is configured in the Settings.
*/
private BasicScheme proxyPreEmptAuth = null;
/**
* The AuthScope for the proxy when proxy authentication is configured in the Settings.
*/
private AuthScope proxyAuthScope = null;
/**
* The HttpHost for the proxy when proxy authentication is configured in the Settings.
*/
private HttpHost proxyHttpHost = null;
private Downloader() {
// Singleton class
final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
//TODO: ensure proper closure and eviction policy
httpClientBuilder = HttpClientBuilder.create()
.useSystemProperties()
.setConnectionManager(connectionManager)
.setConnectionManagerShared(true);
httpClientBuilderExplicitNoproxy = HttpClientBuilder.create()
.useSystemProperties()
.setConnectionManager(connectionManager)
.setConnectionManagerShared(true)
.setProxySelector(new ProxySelector() {
@Override
public List<Proxy> select(URI uri) {
return Collections.singletonList(Proxy.NO_PROXY);
}
@Override
public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
}
});
}
/**
* The singleton instance for downloading file resources.
*
* @return The singleton instance managing download credentials and proxy configuration
*/
public static Downloader getInstance() {
return INSTANCE;
}
/**
* Initialize the Downloader from the settings.
* Extracts the configured proxy- and credential information from the settings and system properties and
* caches those for future use by the Downloader.
*
* @param settings The settings to configure from
* @throws InvalidSettingException When improper configurations are found.
*/
public void configure(Settings settings) throws InvalidSettingException {
this.settings = settings;
if (settings.getString(Settings.KEYS.PROXY_SERVER) != null) {
// Legacy proxy configuration present
// So don't rely on the system properties for proxy; use the legacy settings configuration
final String proxyHost = settings.getString(Settings.KEYS.PROXY_SERVER);
final int proxyPort = settings.getInt(Settings.KEYS.PROXY_PORT, -1);
final String nonProxyHosts = settings.getString(Settings.KEYS.PROXY_NON_PROXY_HOSTS);
if (nonProxyHosts != null && !nonProxyHosts.isEmpty()) {
final ProxySelector selector = new SelectiveProxySelector(
new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)),
nonProxyHosts.split("\\|")
);
httpClientBuilder.setProxySelector(selector);
} else {
httpClientBuilder.setProxy(new HttpHost(proxyHost, proxyPort));
}
if (settings.getString(Settings.KEYS.PROXY_USERNAME) != null) {
final String proxyuser = settings.getString(Settings.KEYS.PROXY_USERNAME);
final char[] proxypass = settings.getString(Settings.KEYS.PROXY_PASSWORD).toCharArray();
this.proxyHttpHost = new HttpHost(null, proxyHost, proxyPort);
this.proxyCreds = new UsernamePasswordCredentials(proxyuser, proxypass);
this.proxyAuthScope = new AuthScope(proxyHttpHost);
this.proxyPreEmptAuth = new BasicScheme();
this.proxyPreEmptAuth.initPreemptive(proxyCreds);
tryConfigureProxyCredentials(credentialsProvider, authCache);
}
}
tryAddRetireJSCredentials();
tryAddHostedSuppressionCredentials();
tryAddKEVCredentials();
tryAddNexusAnalyzerCredentials();
tryAddCentralAnalyzerCredentials();
tryAddCentralContentCredentials();
tryAddNVDApiDatafeed();
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
httpClientBuilderExplicitNoproxy.setDefaultCredentialsProvider(credentialsProvider);
}
private void tryAddRetireJSCredentials() throws InvalidSettingException {
if (!settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, "").isBlank()) {
configureCredentials(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, "RetireJS repo.js",
Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD,
Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_BEARER_TOKEN
);
}
}
private void tryAddHostedSuppressionCredentials() throws InvalidSettingException {
if (!settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, "").isBlank()) {
configureCredentials(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, "Hosted suppressions",
Settings.KEYS.HOSTED_SUPPRESSIONS_USER, Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD,
Settings.KEYS.HOSTED_SUPPRESSIONS_BEARER_TOKEN
);
}
}
private void tryAddKEVCredentials() throws InvalidSettingException {
if (!settings.getString(Settings.KEYS.KEV_URL, "").isBlank()) {
configureCredentials(Settings.KEYS.KEV_URL, "Known Exploited Vulnerabilities",
Settings.KEYS.KEV_USER, Settings.KEYS.KEV_PASSWORD,
Settings.KEYS.KEV_BEARER_TOKEN
);
}
}
private void tryAddNexusAnalyzerCredentials() throws InvalidSettingException {
if (!settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL, "").isBlank()) {
configureCredentials(Settings.KEYS.ANALYZER_NEXUS_URL, "Nexus Analyzer",
Settings.KEYS.ANALYZER_NEXUS_USER, Settings.KEYS.ANALYZER_NEXUS_PASSWORD,
null
);
}
}
private void tryAddCentralAnalyzerCredentials() throws InvalidSettingException {
if (!settings.getString(Settings.KEYS.ANALYZER_CENTRAL_URL, "").isBlank()) {
configureCredentials(Settings.KEYS.ANALYZER_CENTRAL_URL, "Central Analyzer",
Settings.KEYS.ANALYZER_CENTRAL_USER, Settings.KEYS.ANALYZER_CENTRAL_PASSWORD,
Settings.KEYS.ANALYZER_CENTRAL_BEARER_TOKEN
);
}
}
private void tryAddCentralContentCredentials() throws InvalidSettingException {
if (!settings.getString(Settings.KEYS.CENTRAL_CONTENT_URL, "").isBlank()) {
configureCredentials(Settings.KEYS.CENTRAL_CONTENT_URL, "Central Content",
Settings.KEYS.CENTRAL_CONTENT_USER, Settings.KEYS.CENTRAL_CONTENT_PASSWORD,
Settings.KEYS.CENTRAL_CONTENT_BEARER_TOKEN
);
}
}
private void tryAddNVDApiDatafeed() throws InvalidSettingException {
if (!settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL, "").isBlank()) {
configureCredentials(Settings.KEYS.NVD_API_DATAFEED_URL, "NVD API Datafeed",
Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD,
Settings.KEYS.NVD_API_DATAFEED_BEARER_TOKEN
);
}
}
/**
* Configure pre-emptive credentials for the host/port of the URL when configured in settings for the default credential-store and
* authentication-cache.
*
* @param urlKey The settings property key for a configured url for which the credentials should hold
* @param scopeDescription A descriptive text for use in error messages for this credential
* @param userKey The settings property key for a potentially configured configured Basic-auth username
* @param passwordKey The settings property key for a potentially configured configured Basic-auth password
* @param tokenKey The settings property key for a potentially configured Bearer-auth token
* @throws InvalidSettingException When the password is empty or one of the other keys are not found in the settings.
*/
private void configureCredentials(String urlKey, String scopeDescription, String userKey, String passwordKey, String tokenKey)
throws InvalidSettingException {
final URL theURL;
try {
theURL = new URL(settings.getString(urlKey, ""));
} catch (MalformedURLException e) {
throw new InvalidSettingException(scopeDescription + " URL must be a valid URL (was: " + settings.getString(urlKey, "") + ")", e);
}
configureCredentials(theURL, scopeDescription, userKey, passwordKey, tokenKey, credentialsProvider, authCache);
}
/**
* Configure pre-emptive credentials for the host/port of the URL when configured in settings for a specific credential-store and
* authentication-cache.
*
* @param theURL The url for which the credentials should hold
* @param scopeDescription A descriptive text for use in error messages for this credential
* @param userKey The settings property key for a potentially configured configured Basic-auth username
* @param passwordKey The settings property key for a potentially configured configured Basic-auth password
* @param tokenKey The settings property key for a potentially configured Bearer-auth token
* @param theCredentialsStore The credential store that will be set in the HTTP clients context
* @param theAuthCache The authentication cache that will be set in the HTTP clients context
* @throws InvalidSettingException When the password is empty or one of the other keys are not found in the settings.
*/
private void configureCredentials(URL theURL, String scopeDescription, String userKey, String passwordKey, String tokenKey,
CredentialsStore theCredentialsStore, AuthCache theAuthCache)
throws InvalidSettingException {
final String theUser = settings.getString(userKey);
final String thePass = settings.getString(passwordKey);
final String theToken = tokenKey != null ? settings.getString(tokenKey) : null;
if (theUser == null && thePass == null && theToken == null) {
// no credentials configured
return;
}
final String theProtocol = theURL.getProtocol();
if ("file".equals(theProtocol)) {
// no credentials support for file protocol
return;
} else if ("http".equals(theProtocol) && (theUser != null && thePass != null)) {
LOGGER.warn("Insecure configuration: Basic Credentials are configured to be used over a plain http connection for {}. "
+ "Consider migrating to https to guard the credentials.", scopeDescription);
} else if ("http".equals(theProtocol) && (theToken != null)) {
LOGGER.warn("Insecure configuration: Bearer Credentials are configured to be used over a plain http connection for {}. "
+ "Consider migrating to https to guard the credentials.", scopeDescription);
} else if (!"https".equals(theProtocol)) {
throw new InvalidSettingException("Unsupported protocol in the " + scopeDescription
+ " URL; only file, http and https are supported");
}
if (theToken != null) {
HC5CredentialHelper.configurePreEmptiveBearerAuth(theURL, theToken, theCredentialsStore, theAuthCache);
} else if (theUser != null && thePass != null) {
HC5CredentialHelper.configurePreEmptiveBasicAuth(theURL, theUser, thePass, theCredentialsStore, theAuthCache);
}
}
/**
* Retrieves a file from a given URL and saves it to the outputPath.
*
* @param url the URL of the file to download
* @param outputPath the path to the save the file to
* @throws DownloadFailedException is thrown if there is an error downloading the file
* @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
* @throws TooManyRequestsException thrown when a 429 is received
* @throws ResourceNotFoundException thrown when a 404 is received
*/
public void fetchFile(URL url, File outputPath)
throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
fetchFile(url, outputPath, true);
}
/**
* Retrieves a file from a given URL and saves it to the outputPath.
*
* @param url the URL of the file to download
* @param outputPath the path to the save the file to
* @param useProxy whether to use the configured proxy when downloading
* files
* @throws DownloadFailedException is thrown if there is an error downloading the file
* @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
* @throws TooManyRequestsException thrown when a 429 is received
* @throws ResourceNotFoundException thrown when a 404 is received
*/
public void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException,
TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
try {
if ("file".equals(url.getProtocol())) {
final Path p = Paths.get(url.toURI());
Files.copy(p, outputPath.toPath(), StandardCopyOption.REPLACE_EXISTING);
} else {
final BasicClassicHttpRequest req;
req = new BasicClassicHttpRequest(Method.GET, url.toURI());
try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath);
hc.execute(req, getPreEmptiveAuthContext(), responseHandler);
}
}
} catch (HttpResponseException hre) {
wrapAndThrowHttpResponseException(url.toString(), hre);
} catch (SSLHandshakeException ex) {
if (ex.getMessage().contains("unable to find valid certification path to requested target")) {
final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. "
+ "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url);
throw new URLConnectionFailureException(msg, ex);
}
final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
throw new DownloadFailedException(msg, ex);
} catch (RuntimeException | URISyntaxException | IOException ex) {
final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
throw new DownloadFailedException(msg, ex);
}
}
private static void wrapAndThrowHttpResponseException(String url, HttpResponseException hre)
throws ResourceNotFoundException, TooManyRequestsException, DownloadFailedException {
final String messageFormat = "%s - Server status: %d - Server reason: %s";
switch (hre.getStatusCode()) {
case 404:
throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre);
case 429:
throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre);
default:
throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre);
}
}
/**
* Retrieves a file from a given URL using an ad-hoc created CredentialsProvider if needed
* and saves it to the outputPath.
*
* @param url the URL of the file to download
* @param outputPath the path to the save the file to
* @param useProxy whether to use the configured proxy when downloading files
* @param userKey The settings property key for a potentially configured configured Basic-auth username
* @param passwordKey The settings property key for a potentially configured configured Basic-auth password
* @param tokenKey The settings property key for a potentially configured Bearer-auth token
* @throws DownloadFailedException is thrown if there is an error downloading the file
* @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
* @throws TooManyRequestsException thrown when a 429 is received
* @throws ResourceNotFoundException thrown when a 404 is received
* @implNote This method should only be used in cases where the target host cannot be determined beforehand from settings, so that ad-hoc
* Credentials needs to be constructed for the target URL when the user/password keys point to configured credentials. The method delegates to
* {@link #fetchFile(URL, File, boolean)} when credentials are not configured for the given keys or the resource points to a file.
*/
public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey, String passwordKey, String tokenKey)
throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
final boolean basicConfigured = userKey != null && settings.getString(userKey) != null
&& passwordKey != null && settings.getString(passwordKey) != null;
final boolean tokenConfigured = tokenKey != null && settings.getString(tokenKey) != null;
if ("file".equals(url.getProtocol()) || (!basicConfigured && !tokenConfigured)) {
// no credentials configured, so use the default fetchFile
fetchFile(url, outputPath, useProxy);
return;
}
final String theProtocol = url.getProtocol();
if (!("http".equals(theProtocol) || "https".equals(theProtocol))) {
throw new DownloadFailedException("Unsupported protocol in the URL; only file, http and https are supported");
}
try {
final HttpClientContext dedicatedAuthContext = HttpClientContext.create();
final CredentialsStore dedicatedCredentialStore = new SystemDefaultCredentialsProvider();
final AuthCache dedicatedAuthCache = new BasicAuthCache();
configureCredentials(url, url.toString(), userKey, passwordKey, tokenKey, dedicatedCredentialStore, dedicatedAuthCache);
if (useProxy && proxyAuthScope != null) {
tryConfigureProxyCredentials(dedicatedCredentialStore, dedicatedAuthCache);
}
dedicatedAuthContext.setCredentialsProvider(dedicatedCredentialStore);
dedicatedAuthContext.setAuthCache(dedicatedAuthCache);
try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI());
final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath);
hc.execute(req, dedicatedAuthContext, responseHandler);
}
} catch (HttpResponseException hre) {
wrapAndThrowHttpResponseException(url.toString(), hre);
} catch (SSLHandshakeException ex) {
if (ex.getMessage().contains("unable to find valid certification path to requested target")) {
final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. "
+ "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url);
throw new URLConnectionFailureException(msg, ex);
}
final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
throw new DownloadFailedException(msg, ex);
} catch (RuntimeException | URISyntaxException | IOException ex) {
final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
throw new DownloadFailedException(msg, ex);
}
}
/**
* Add the proxy credentials to the CredentialsProvider and AuthCache instances when proxy-authentication is configured in the settings.
* @param credentialsProvider The credentialStore to configure the credentials in
* @param authCache The AuthCache to cache the pre-empted credentials in
*/
private void tryConfigureProxyCredentials(@NotNull CredentialsStore credentialsProvider, @NotNull AuthCache authCache) {
if (proxyPreEmptAuth != null) {
credentialsProvider.setCredentials(proxyAuthScope, proxyCreds);
authCache.put(proxyHttpHost, proxyPreEmptAuth);
}
}
/**
* Posts a payload to the URL and returns the response as a string.
*
* @param url the URL to POST to
* @param payload the Payload to post
* @param payloadType the string describing the payload's mime-type
* @param hdr Additional headers to add to the HTTP request
* @return the content of the response
* @throws DownloadFailedException is thrown if there is an error downloading the file
* @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
* @throws TooManyRequestsException thrown when a 429 is received
* @throws ResourceNotFoundException thrown when a 404 is received
*/
public String postBasedFetchContent(URI url, String payload, ContentType payloadType, List<Header> hdr)
throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
try {
if (url.getScheme() == null || !url.getScheme().toLowerCase(Locale.ROOT).matches("^https?")) {
throw new IllegalArgumentException("Unsupported protocol in the URL; only http and https are supported");
} else {
final BasicClassicHttpRequest req;
req = new BasicClassicHttpRequest(Method.POST, url);
req.setEntity(new StringEntity(payload, payloadType));
for (Header h : hdr) {
req.addHeader(h);
}
final String result;
try (CloseableHttpClient hc = httpClientBuilder.build()) {
result = hc.execute(req, getPreEmptiveAuthContext(), new BasicHttpClientResponseHandler());
}
return result;
}
} catch (HttpResponseException hre) {
wrapAndThrowHttpResponseException(url.toString(), hre);
throw new InternalError("wrapAndThrowHttpResponseException will always throw an exception but Java compiler fails to spot it");
} catch (SSLHandshakeException ex) {
if (ex.getMessage().contains("unable to find valid certification path to requested target")) {
final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. "
+ "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url);
throw new URLConnectionFailureException(msg, ex);
}
final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage());
throw new DownloadFailedException(msg, ex);
} catch (IOException | RuntimeException ex) {
final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage());
throw new DownloadFailedException(msg, ex);
}
}
/**
* Retrieves a file from a given URL and returns the contents.
*
* @param url the URL of the file to download
* @param charset The characterset to use to interpret the binary content of the file
* @return the content of the file
* @throws DownloadFailedException is thrown if there is an error
* downloading the file
* @throws TooManyRequestsException thrown when a 429 is received
* @throws ResourceNotFoundException thrown when a 404 is received
*/
public String fetchContent(URL url, Charset charset) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
return fetchContent(url, true, charset);
}
/**
* Retrieves a file from a given URL and returns the contents.
*
* @param url the URL of the file to download
* @param useProxy whether to use the configured proxy when downloading
* files
* @param charset The characterset to use to interpret the binary content of the file
* @return the content of the file
* @throws DownloadFailedException is thrown if there is an error
* downloading the file
* @throws TooManyRequestsException thrown when a 429 is received
* @throws ResourceNotFoundException thrown when a 404 is received
*/
public String fetchContent(URL url, boolean useProxy, Charset charset)
throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
try {
final String result;
if ("file".equals(url.getProtocol())) {
final Path p = Paths.get(url.toURI());
result = Files.readString(p, charset);
} else {
final BasicClassicHttpRequest req;
req = new BasicClassicHttpRequest(Method.GET, url.toURI());
try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
req.addHeader(HttpHeaders.ACCEPT_CHARSET, charset.name());
final ExplicitCharsetToStringResponseHandler responseHandler = new ExplicitCharsetToStringResponseHandler(charset);
result = hc.execute(req, getPreEmptiveAuthContext(), responseHandler);
}
}
return result;
} catch (HttpResponseException hre) {
wrapAndThrowHttpResponseException(url.toString(), hre);
throw new InternalError("wrapAndThrowHttpResponseException will always throw an exception but Java compiler fails to spot it");
} catch (RuntimeException | URISyntaxException | IOException ex) {
final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage());
throw new DownloadFailedException(msg, ex);
}
}
/**
* Gets a HttpClientContext that supports pre-emptive authentication.
* @return A HttpClientContext pre-configured with the authentication cache build from the settings.
*/
public HttpClientContext getPreEmptiveAuthContext() {
final HttpClientContext context = HttpClientContext.create();
context.setCredentialsProvider(credentialsProvider);
context.setAuthCache(authCache);
return context;
}
/**
* Gets a pre-configured HttpClient.
* Mainly targeted for use in paged resultset scenarios with multiple roundtrips.
* @param useProxy Whether to use the configuration that includes proxy-settings
* @return A HttpClient pre-configured with the settings.
*/
public CloseableHttpClient getHttpClient(boolean useProxy) {
return useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build();
}
/**
* Download a resource from the given URL and have its content handled by the given ResponseHandler.
*
* @param url The url of the resource
* @param handler The responsehandler to handle the response
* @param <T> The return-type for the responseHandler
* @return The response handler result
* @throws IOException on I/O Exceptions
* @throws TooManyRequestsException When HTTP status 429 is encountered
* @throws ResourceNotFoundException When HTTP status 404 is encountered
*/
public <T> T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler<T> handler)
throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException {
return fetchAndHandle(url, handler, Collections.emptyList(), true);
}
/**
* Download a resource from the given URL and have its content handled by the given ResponseHandler.
*
* @param url The url of the resource
* @param handler The responsehandler to handle the response
* @param hdr Additional headers to add to the HTTP request
* @param <T> The return-type for the responseHandler
* @return The response handler result
* @throws IOException on I/O Exceptions
* @throws TooManyRequestsException When HTTP status 429 is encountered
* @throws ResourceNotFoundException When HTTP status 404 is encountered
*/
public <T> T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler<T> handler, @NotNull List<Header> hdr)
throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException {
return fetchAndHandle(url, handler, hdr, true);
}
/**
* Download a resource from the given URL and have its content handled by the given ResponseHandler.
*
* @param url The url of the resource
* @param handler The responsehandler to handle the response
* @param hdr Additional headers to add to the HTTP request
* @param useProxy Whether to use the configured proxy for the connection
* @param <T> The return-type for the responseHandler
* @return The response handler result
* @throws IOException on I/O Exceptions
* @throws TooManyRequestsException When HTTP status 429 is encountered
* @throws ResourceNotFoundException When HTTP status 404 is encountered
*/
public <T> T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler<T> handler, @NotNull List<Header> hdr, boolean useProxy)
throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException {
final T data;
if ("file".equals(url.getProtocol())) {
final Path p = Paths.get(url.toURI());
try (InputStream is = Files.newInputStream(p)) {
final HttpEntity dummyEntity = new BasicHttpEntity(is, ContentType.APPLICATION_JSON);
final ClassicHttpResponse dummyResponse = new BasicClassicHttpResponse(200);
dummyResponse.setEntity(dummyEntity);
data = handler.handleResponse(dummyResponse);
} catch (HttpException e) {
throw new IllegalStateException("HttpException encountered emulating a HTTP response from a file", e);
}
} else {
try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
return fetchAndHandle(hc, url, handler, hdr);
}
}
return data;
}
/**
* Download a resource from the given URL and have its content handled by the given ResponseHandler.
*
* @param client The HTTP Client to reuse for the request
* @param url The url of the resource
* @param handler The responsehandler to handle the response
* @param hdr Additional headers to add to the HTTP request
* @param <T> The return-type for the responseHandler
* @return The response handler result
* @throws IOException on I/O Exceptions
* @throws TooManyRequestsException When HTTP status 429 is encountered
* @throws ResourceNotFoundException When HTTP status 404 is encountered
*/
public <T> T fetchAndHandle(@NotNull CloseableHttpClient client, @NotNull URL url, @NotNull HttpClientResponseHandler<T> handler,
@NotNull List<Header> hdr) throws IOException, TooManyRequestsException, ResourceNotFoundException {
try {
final String theProtocol = url.getProtocol();
if (!("http".equals(theProtocol) || "https".equals(theProtocol))) {
throw new DownloadFailedException("Unsupported protocol in the URL; only http and https are supported");
}
final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI());
for (Header h : hdr) {
req.addHeader(h);
}
final HttpClientContext context = getPreEmptiveAuthContext();
return client.execute(req, context, handler);
} catch (HttpResponseException hre) {
final String messageFormat = "%s - Server status: %d - Server reason: %s";
switch (hre.getStatusCode()) {
case 404:
throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()));
case 429:
throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()));
default:
throw new IOException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()));
}
} catch (RuntimeException | URISyntaxException ex) {
final String msg = format("Download failed, unable to retrieve and parse '%s'; %s", url, ex.getMessage());
throw new IOException(msg, ex);
}
}
private static class SelectiveProxySelector extends ProxySelector {
/**
* The suffix-match entries from the nonProxyHosts (those starting with a {@code *}).
*/
private final List<String> suffixMatch = new ArrayList<>();
/**
* The full host entries from the nonProxyHosts (those <em>not</em> starting with a {@code *}).
*/
private final List<String> fullmatch = new ArrayList<>();
/**
* The proxy use when no proxy-exception is found.
*/
private final Proxy configuredProxy;
SelectiveProxySelector(Proxy httpHost, String[] nonProxyHostsPatterns) {
for (String nonProxyHostPattern : nonProxyHostsPatterns) {
if (nonProxyHostPattern.startsWith("*")) {
suffixMatch.add(nonProxyHostPattern.substring(1));
} else {
fullmatch.add(nonProxyHostPattern);
}
}
this.configuredProxy = httpHost;
}
@Override
public List<Proxy> select(URI uri) {
final String theHost = uri.getHost();
if (fullmatch.contains(theHost)) {
return Collections.singletonList(Proxy.NO_PROXY);
} else {
for (String suffix : suffixMatch) {
if (theHost.endsWith(suffix)) {
return Collections.singletonList(Proxy.NO_PROXY);
}
}
}
return List.of(configuredProxy);
}
@Override
public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
// nothing to be done for this single proxy proxy-selector
}
}
}