/*
    * This file is part of dependency-check-utils.
    *
    * Licensed under the Apache License, Version 2.0 (the "License");
    * you may not use this file except in compliance with the License.
    * You may obtain a copy of the License at
    *
    *     http://www.apache.org/licenses/LICENSE-2.0
    *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   *
   * Copyright (c) 2024 Hans Aikema. All Rights Reserved.
   */
  package org.owasp.dependencycheck.utils;
  
  import org.apache.hc.client5.http.HttpResponseException;
  import org.apache.hc.client5.http.auth.AuthCache;
  import org.apache.hc.client5.http.auth.AuthScope;
  import org.apache.hc.client5.http.auth.Credentials;
  import org.apache.hc.client5.http.auth.CredentialsStore;
  import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
  import org.apache.hc.client5.http.impl.auth.BasicAuthCache;
  import org.apache.hc.client5.http.impl.auth.BasicScheme;
  import org.apache.hc.client5.http.impl.auth.SystemDefaultCredentialsProvider;
  import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler;
  import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
  import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
  import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
  import org.apache.hc.client5.http.protocol.HttpClientContext;
  import org.apache.hc.core5.http.ClassicHttpResponse;
  import org.apache.hc.core5.http.ContentType;
  import org.apache.hc.core5.http.Header;
  import org.apache.hc.core5.http.HttpEntity;
  import org.apache.hc.core5.http.HttpException;
  import org.apache.hc.core5.http.HttpHeaders;
  import org.apache.hc.core5.http.HttpHost;
  import org.apache.hc.core5.http.Method;
  import org.apache.hc.core5.http.io.HttpClientResponseHandler;
  import org.apache.hc.core5.http.io.entity.BasicHttpEntity;
  import org.apache.hc.core5.http.io.entity.StringEntity;
  import org.apache.hc.core5.http.message.BasicClassicHttpRequest;
  import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
  import org.jetbrains.annotations.NotNull;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import javax.net.ssl.SSLHandshakeException;
  import java.io.File;
  import java.io.IOException;
  import java.io.InputStream;
  import java.net.InetSocketAddress;
  import java.net.MalformedURLException;
  import java.net.Proxy;
  import java.net.ProxySelector;
  import java.net.SocketAddress;
  import java.net.URI;
  import java.net.URISyntaxException;
  import java.net.URL;
  import java.nio.charset.Charset;
  import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.Paths;
  import java.nio.file.StandardCopyOption;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.List;
  import java.util.Locale;
  
  import static java.lang.String.format;
  
  /**
   * A Utility class to centralize download logic like HTTP(S) proxy configuration and proxy- and server credential handling.
   * @author Jeremy Long, Hans Aikema
   */
  public final class Downloader {
  
      /**
       * The builder to use for a HTTP Client that uses the configured proxy-settings
       */
      private final HttpClientBuilder httpClientBuilder;
  
      /**
       * The builder to use for a HTTP Client that explicitly opts out of proxy-usage
       */
      private final HttpClientBuilder httpClientBuilderExplicitNoproxy;
  
      /**
       * The Authentication cache for pre-emptive authentication.
       * This gets filled with credentials from the settings in {@link #configure(Settings)}.
       */
      private final AuthCache authCache = new BasicAuthCache();
  
      /**
       * The credentialsProvider for pre-emptive authentication.
       * This gets filled with credentials from the settings in {@link #configure(Settings)}.
      */
     private final SystemDefaultCredentialsProvider credentialsProvider = new SystemDefaultCredentialsProvider();
 
     /**
      * The settings
      */
     private Settings settings;
 
     /**
      * The Logger for use throughout the class.
      */
     private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class);
 
     /**
      * The singleton instance of the downloader
      */
     private static final Downloader INSTANCE = new Downloader();
     /**
      * The Credentials for the proxy when proxy authentication is configured in the Settings.
      */
     private Credentials proxyCreds = null;
     /**
      * A BasicScheme initialized with the proxy-credentials when proxy authentication is configured in the Settings.
      */
     private BasicScheme proxyPreEmptAuth = null;
     /**
      * The AuthScope for the proxy when proxy authentication is configured in the Settings.
      */
     private AuthScope proxyAuthScope = null;
     /**
      * The HttpHost for the proxy when proxy authentication is configured in the Settings.
      */
     private HttpHost proxyHttpHost = null;
 
     private Downloader() {
         // Singleton class
         final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
         //TODO: ensure proper closure and eviction policy
         httpClientBuilder = HttpClientBuilder.create()
                 .useSystemProperties()
                 .setConnectionManager(connectionManager)
                 .setConnectionManagerShared(true);
         httpClientBuilderExplicitNoproxy = HttpClientBuilder.create()
                 .useSystemProperties()
                 .setConnectionManager(connectionManager)
                 .setConnectionManagerShared(true)
                 .setProxySelector(new ProxySelector() {
                     @Override
                     public List<Proxy> select(URI uri) {
                         return Collections.singletonList(Proxy.NO_PROXY);
                     }
 
                     @Override
                     public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
 
                     }
                 });
     }
 
     /**
      * The singleton instance for downloading file resources.
      *
      * @return The singleton instance managing download credentials and proxy configuration
      */
     public static Downloader getInstance() {
         return INSTANCE;
     }
 
     /**
      * Initialize the Downloader from the settings.
      * Extracts the configured proxy- and credential information from the settings and system properties and
      * caches those for future use by the Downloader.
      *
      * @param settings The settings to configure from
      * @throws InvalidSettingException When improper configurations are found.
      */
     public void configure(Settings settings) throws InvalidSettingException {
         this.settings = settings;
 
         if (settings.getString(Settings.KEYS.PROXY_SERVER) != null) {
             // Legacy proxy configuration present
             // So don't rely on the system properties for proxy; use the legacy settings configuration
             final String proxyHost = settings.getString(Settings.KEYS.PROXY_SERVER);
             final int proxyPort = settings.getInt(Settings.KEYS.PROXY_PORT, -1);
             final String nonProxyHosts = settings.getString(Settings.KEYS.PROXY_NON_PROXY_HOSTS);
             if (nonProxyHosts != null && !nonProxyHosts.isEmpty()) {
                 final ProxySelector selector = new SelectiveProxySelector(
                         new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)),
                         nonProxyHosts.split("\\|")
                 );
                 httpClientBuilder.setProxySelector(selector);
             } else {
                 httpClientBuilder.setProxy(new HttpHost(proxyHost, proxyPort));
             }
             if (settings.getString(Settings.KEYS.PROXY_USERNAME) != null) {
                 final String proxyuser = settings.getString(Settings.KEYS.PROXY_USERNAME);
                 final char[] proxypass = settings.getString(Settings.KEYS.PROXY_PASSWORD).toCharArray();
                 this.proxyHttpHost = new HttpHost(null, proxyHost, proxyPort);
                 this.proxyCreds = new UsernamePasswordCredentials(proxyuser, proxypass);
                 this.proxyAuthScope = new AuthScope(proxyHttpHost);
                 this.proxyPreEmptAuth = new BasicScheme();
                 this.proxyPreEmptAuth.initPreemptive(proxyCreds);
                 tryConfigureProxyCredentials(credentialsProvider, authCache);
             }
         }
         tryAddRetireJSCredentials();
         tryAddHostedSuppressionCredentials();
         tryAddKEVCredentials();
         tryAddNexusAnalyzerCredentials();
         tryAddCentralAnalyzerCredentials();
         tryAddCentralContentCredentials();
         tryAddNVDApiDatafeed();
         httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
         httpClientBuilderExplicitNoproxy.setDefaultCredentialsProvider(credentialsProvider);
     }
 
     private void tryAddRetireJSCredentials() throws InvalidSettingException {
         if (!settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, "").isBlank()) {
             configureCredentials(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, "RetireJS repo.js",
                     Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD,
                     Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_BEARER_TOKEN
                     );
         }
     }
 
     private void tryAddHostedSuppressionCredentials() throws InvalidSettingException {
         if (!settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, "").isBlank()) {
             configureCredentials(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, "Hosted suppressions",
                     Settings.KEYS.HOSTED_SUPPRESSIONS_USER, Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD,
                     Settings.KEYS.HOSTED_SUPPRESSIONS_BEARER_TOKEN
             );
         }
     }
 
     private void tryAddKEVCredentials() throws InvalidSettingException {
         if (!settings.getString(Settings.KEYS.KEV_URL, "").isBlank()) {
             configureCredentials(Settings.KEYS.KEV_URL, "Known Exploited Vulnerabilities",
                     Settings.KEYS.KEV_USER, Settings.KEYS.KEV_PASSWORD,
                     Settings.KEYS.KEV_BEARER_TOKEN
             );
         }
     }
 
     private void tryAddNexusAnalyzerCredentials() throws InvalidSettingException {
         if (!settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL, "").isBlank()) {
             configureCredentials(Settings.KEYS.ANALYZER_NEXUS_URL, "Nexus Analyzer",
                     Settings.KEYS.ANALYZER_NEXUS_USER, Settings.KEYS.ANALYZER_NEXUS_PASSWORD,
                     null
             );
         }
     }
 
     private void tryAddCentralAnalyzerCredentials() throws InvalidSettingException {
         if (!settings.getString(Settings.KEYS.ANALYZER_CENTRAL_URL, "").isBlank()) {
             configureCredentials(Settings.KEYS.ANALYZER_CENTRAL_URL, "Central Analyzer",
                     Settings.KEYS.ANALYZER_CENTRAL_USER, Settings.KEYS.ANALYZER_CENTRAL_PASSWORD,
                     Settings.KEYS.ANALYZER_CENTRAL_BEARER_TOKEN
             );
         }
     }
 
     private void tryAddCentralContentCredentials() throws InvalidSettingException {
         if (!settings.getString(Settings.KEYS.CENTRAL_CONTENT_URL, "").isBlank()) {
             configureCredentials(Settings.KEYS.CENTRAL_CONTENT_URL, "Central Content",
                     Settings.KEYS.CENTRAL_CONTENT_USER, Settings.KEYS.CENTRAL_CONTENT_PASSWORD,
                     Settings.KEYS.CENTRAL_CONTENT_BEARER_TOKEN
 
             );
         }
     }
 
     private void tryAddNVDApiDatafeed() throws InvalidSettingException {
         if (!settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL, "").isBlank()) {
             configureCredentials(Settings.KEYS.NVD_API_DATAFEED_URL, "NVD API Datafeed",
                     Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD,
                     Settings.KEYS.NVD_API_DATAFEED_BEARER_TOKEN
             );
         }
     }
 
     /**
      * Configure pre-emptive credentials for the host/port of the URL when configured in settings for the default credential-store and
      * authentication-cache.
      *
      * @param urlKey           The settings property key for a configured url for which the credentials should hold
      * @param scopeDescription A descriptive text for use in error messages for this credential
      * @param userKey          The settings property key for a potentially configured configured Basic-auth username
      * @param passwordKey      The settings property key for a potentially configured configured Basic-auth password
      * @param tokenKey         The settings property key for a potentially configured Bearer-auth token
      * @throws InvalidSettingException When the password is empty or one of the other keys are not found in the settings.
      */
     private void configureCredentials(String urlKey, String scopeDescription, String userKey, String passwordKey, String tokenKey)
             throws InvalidSettingException {
         final URL theURL;
         try {
             theURL = new URL(settings.getString(urlKey, ""));
         } catch (MalformedURLException e) {
             throw new InvalidSettingException(scopeDescription + " URL must be a valid URL (was: " + settings.getString(urlKey, "") + ")", e);
         }
         configureCredentials(theURL, scopeDescription, userKey, passwordKey, tokenKey, credentialsProvider, authCache);
     }
 
     /**
      * Configure pre-emptive credentials for the host/port of the URL when configured in settings for a specific credential-store and
      * authentication-cache.
      *
      * @param theURL      The url for which the credentials should hold
      * @param scopeDescription        A descriptive text for use in error messages for this credential
      * @param userKey     The settings property key for a potentially configured configured Basic-auth username
      * @param passwordKey The settings property key for a potentially configured configured Basic-auth password
      * @param tokenKey The settings property key for a potentially configured Bearer-auth token
      * @param theCredentialsStore The credential store that will be set in the HTTP clients context
      * @param theAuthCache        The authentication cache that will be set in the HTTP clients context
      * @throws InvalidSettingException When the password is empty or one of the other keys are not found in the settings.
      */
     private void configureCredentials(URL theURL, String scopeDescription, String userKey, String passwordKey, String tokenKey,
                                       CredentialsStore theCredentialsStore, AuthCache theAuthCache)
             throws InvalidSettingException {
         final String theUser = settings.getString(userKey);
         final String thePass = settings.getString(passwordKey);
         final String theToken = tokenKey != null ? settings.getString(tokenKey) : null;
         if (theUser == null && thePass == null && theToken == null) {
             // no credentials configured
             return;
         }
         final String theProtocol = theURL.getProtocol();
         if ("file".equals(theProtocol)) {
             // no credentials support for file protocol
             return;
         } else if ("http".equals(theProtocol) && (theUser != null && thePass != null)) {
             LOGGER.warn("Insecure configuration: Basic Credentials are configured to be used over a plain http connection for {}. "
                     + "Consider migrating to https to guard the credentials.", scopeDescription);
         } else if ("http".equals(theProtocol) && (theToken != null)) {
             LOGGER.warn("Insecure configuration: Bearer Credentials are configured to be used over a plain http connection for {}. "
                     + "Consider migrating to https to guard the credentials.", scopeDescription);
         } else if (!"https".equals(theProtocol)) {
             throw new InvalidSettingException("Unsupported protocol in the " + scopeDescription
                     + " URL; only file, http and https are supported");
         }
         if (theToken != null) {
             HC5CredentialHelper.configurePreEmptiveBearerAuth(theURL, theToken, theCredentialsStore, theAuthCache);
         } else if (theUser != null && thePass != null) {
             HC5CredentialHelper.configurePreEmptiveBasicAuth(theURL, theUser, thePass, theCredentialsStore, theAuthCache);
         }
     }
 
     /**
      * Retrieves a file from a given URL and saves it to the outputPath.
      *
      * @param url        the URL of the file to download
      * @param outputPath the path to the save the file to
      * @throws DownloadFailedException       is thrown if there is an error downloading the file
      * @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
      * @throws TooManyRequestsException      thrown when a 429 is received
      * @throws ResourceNotFoundException     thrown when a 404 is received
      */
     public void fetchFile(URL url, File outputPath)
             throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
         fetchFile(url, outputPath, true);
     }
 
     /**
      * Retrieves a file from a given URL and saves it to the outputPath.
      *
      * @param url        the URL of the file to download
      * @param outputPath the path to the save the file to
      * @param useProxy   whether to use the configured proxy when downloading
      *                   files
      * @throws DownloadFailedException       is thrown if there is an error downloading the file
      * @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
      * @throws TooManyRequestsException      thrown when a 429 is received
      * @throws ResourceNotFoundException     thrown when a 404 is received
      */
     public void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException,
             TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
         try {
             if ("file".equals(url.getProtocol())) {
                 final Path p = Paths.get(url.toURI());
                 Files.copy(p, outputPath.toPath(), StandardCopyOption.REPLACE_EXISTING);
             } else {
                 final BasicClassicHttpRequest req;
                 req = new BasicClassicHttpRequest(Method.GET, url.toURI());
                 try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
                     final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath);
                     hc.execute(req, getPreEmptiveAuthContext(), responseHandler);
                 }
             }
         } catch (HttpResponseException hre) {
             wrapAndThrowHttpResponseException(url.toString(), hre);
         } catch (SSLHandshakeException ex) {
             if (ex.getMessage().contains("unable to find valid certification path to requested target")) {
                 final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. "
                         + "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url);
                 throw new URLConnectionFailureException(msg, ex);
             }
             final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
             throw new DownloadFailedException(msg, ex);
         } catch (RuntimeException | URISyntaxException | IOException ex) {
             final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
             throw new DownloadFailedException(msg, ex);
         }
     }
 
     private static void wrapAndThrowHttpResponseException(String url, HttpResponseException hre)
             throws ResourceNotFoundException, TooManyRequestsException, DownloadFailedException {
         final String messageFormat = "%s - Server status: %d - Server reason: %s";
         switch (hre.getStatusCode()) {
             case 404:
                 throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre);
             case 429:
                 throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre);
             default:
                 throw new DownloadFailedException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()), hre);
         }
     }
 
     /**
      * Retrieves a file from a given URL using an ad-hoc created CredentialsProvider if needed
      * and saves it to the outputPath.
      *
      * @param url         the URL of the file to download
      * @param outputPath  the path to the save the file to
      * @param useProxy    whether to use the configured proxy when downloading files
      * @param userKey     The settings property key for a potentially configured configured Basic-auth username
      * @param passwordKey The settings property key for a potentially configured configured Basic-auth password
      * @param tokenKey    The settings property key for a potentially configured Bearer-auth token
      * @throws DownloadFailedException       is thrown if there is an error downloading the file
      * @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
      * @throws TooManyRequestsException      thrown when a 429 is received
      * @throws ResourceNotFoundException     thrown when a 404 is received
      * @implNote This method should only be used in cases where the target host cannot be determined beforehand from settings, so that ad-hoc
      * Credentials needs to be constructed for the target URL when the user/password keys point to configured credentials. The method delegates to
      * {@link #fetchFile(URL, File, boolean)} when credentials are not configured for the given keys or the resource points to a file.
      */
     public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey, String passwordKey, String tokenKey)
             throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
         final boolean basicConfigured = userKey != null && settings.getString(userKey) != null
                 && passwordKey != null && settings.getString(passwordKey) != null;
         final boolean tokenConfigured = tokenKey != null && settings.getString(tokenKey) != null;
         if ("file".equals(url.getProtocol()) || (!basicConfigured && !tokenConfigured)) {
             // no credentials configured, so use the default fetchFile
             fetchFile(url, outputPath, useProxy);
             return;
         }
         final String theProtocol = url.getProtocol();
         if (!("http".equals(theProtocol) || "https".equals(theProtocol))) {
             throw new DownloadFailedException("Unsupported protocol in the URL; only file, http and https are supported");
         }
         try {
             final HttpClientContext dedicatedAuthContext = HttpClientContext.create();
             final CredentialsStore dedicatedCredentialStore = new SystemDefaultCredentialsProvider();
             final AuthCache dedicatedAuthCache = new BasicAuthCache();
             configureCredentials(url, url.toString(), userKey, passwordKey, tokenKey, dedicatedCredentialStore, dedicatedAuthCache);
             if (useProxy && proxyAuthScope != null) {
                 tryConfigureProxyCredentials(dedicatedCredentialStore, dedicatedAuthCache);
             }
             dedicatedAuthContext.setCredentialsProvider(dedicatedCredentialStore);
             dedicatedAuthContext.setAuthCache(dedicatedAuthCache);
             try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
                 final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI());
                 final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath);
                 hc.execute(req, dedicatedAuthContext, responseHandler);
             }
         } catch (HttpResponseException hre) {
             wrapAndThrowHttpResponseException(url.toString(), hre);
         } catch (SSLHandshakeException ex) {
             if (ex.getMessage().contains("unable to find valid certification path to requested target")) {
                 final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. "
                         + "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url);
                 throw new URLConnectionFailureException(msg, ex);
             }
             final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
             throw new DownloadFailedException(msg, ex);
         } catch (RuntimeException | URISyntaxException | IOException ex) {
             final String msg = format("Download failed, unable to copy '%s' to '%s'; %s", url, outputPath.getAbsolutePath(), ex.getMessage());
             throw new DownloadFailedException(msg, ex);
         }
     }
 
     /**
      * Add the proxy credentials to the CredentialsProvider and AuthCache instances when proxy-authentication is configured in the settings.
      * @param credentialsProvider The credentialStore to configure the credentials in
      * @param authCache The AuthCache to cache the pre-empted credentials in
      */
     private void tryConfigureProxyCredentials(@NotNull CredentialsStore credentialsProvider, @NotNull AuthCache authCache) {
         if (proxyPreEmptAuth != null) {
             credentialsProvider.setCredentials(proxyAuthScope, proxyCreds);
             authCache.put(proxyHttpHost, proxyPreEmptAuth);
         }
     }
 
     /**
      * Posts a payload to the URL and returns the response as a string.
      *
      * @param url         the URL to POST to
      * @param payload     the Payload to post
      * @param payloadType the string describing the payload's mime-type
      * @param hdr         Additional headers to add to the HTTP request
      * @return the content of the response
      * @throws DownloadFailedException       is thrown if there is an error downloading the file
      * @throws URLConnectionFailureException is thrown when certificate-chain trust errors occur downloading the file
      * @throws TooManyRequestsException      thrown when a 429 is received
      * @throws ResourceNotFoundException     thrown when a 404 is received
      */
     public String postBasedFetchContent(URI url, String payload, ContentType payloadType, List<Header> hdr)
             throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException, URLConnectionFailureException {
         try {
             if (url.getScheme() == null || !url.getScheme().toLowerCase(Locale.ROOT).matches("^https?")) {
                 throw new IllegalArgumentException("Unsupported protocol in the URL; only http and https are supported");
             } else {
                 final BasicClassicHttpRequest req;
                 req = new BasicClassicHttpRequest(Method.POST, url);
                 req.setEntity(new StringEntity(payload, payloadType));
                 for (Header h : hdr) {
                     req.addHeader(h);
                 }
                 final String result;
                 try (CloseableHttpClient hc = httpClientBuilder.build()) {
                     result = hc.execute(req, getPreEmptiveAuthContext(), new BasicHttpClientResponseHandler());
                 }
                 return result;
             }
         } catch (HttpResponseException hre) {
             wrapAndThrowHttpResponseException(url.toString(), hre);
             throw new InternalError("wrapAndThrowHttpResponseException will always throw an exception but Java compiler fails to spot it");
         } catch (SSLHandshakeException ex) {
             if (ex.getMessage().contains("unable to find valid certification path to requested target")) {
                 final String msg = String.format("Unable to connect to '%s' - the Java trust store does not contain a trusted root for the cert. "
                         + "Please see https://github.com/jeremylong/InstallCert for one method of updating the trusted certificates.", url);
                 throw new URLConnectionFailureException(msg, ex);
             }
             final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage());
             throw new DownloadFailedException(msg, ex);
         } catch (IOException | RuntimeException ex) {
             final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage());
             throw new DownloadFailedException(msg, ex);
         }
     }
 
     /**
      * Retrieves a file from a given URL and returns the contents.
      *
      * @param url     the URL of the file to download
      * @param charset The characterset to use to interpret the binary content of the file
      * @return the content of the file
      * @throws DownloadFailedException   is thrown if there is an error
      *                                   downloading the file
      * @throws TooManyRequestsException  thrown when a 429 is received
      * @throws ResourceNotFoundException thrown when a 404 is received
      */
     public String fetchContent(URL url, Charset charset) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
         return fetchContent(url, true, charset);
     }
 
     /**
      * Retrieves a file from a given URL and returns the contents.
      *
      * @param url      the URL of the file to download
      * @param useProxy whether to use the configured proxy when downloading
      *                 files
      * @param charset  The characterset to use to interpret the binary content of the file
      * @return the content of the file
      * @throws DownloadFailedException   is thrown if there is an error
      *                                   downloading the file
      * @throws TooManyRequestsException  thrown when a 429 is received
      * @throws ResourceNotFoundException thrown when a 404 is received
      */
     public String fetchContent(URL url, boolean useProxy, Charset charset)
             throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
         try {
             final String result;
             if ("file".equals(url.getProtocol())) {
                 final Path p = Paths.get(url.toURI());
                 result = Files.readString(p, charset);
             } else {
                 final BasicClassicHttpRequest req;
                 req = new BasicClassicHttpRequest(Method.GET, url.toURI());
                 try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
                     req.addHeader(HttpHeaders.ACCEPT_CHARSET, charset.name());
                     final ExplicitCharsetToStringResponseHandler responseHandler = new ExplicitCharsetToStringResponseHandler(charset);
                     result = hc.execute(req, getPreEmptiveAuthContext(), responseHandler);
                 }
             }
             return result;
         } catch (HttpResponseException hre) {
             wrapAndThrowHttpResponseException(url.toString(), hre);
             throw new InternalError("wrapAndThrowHttpResponseException will always throw an exception but Java compiler fails to spot it");
         } catch (RuntimeException | URISyntaxException | IOException ex) {
             final String msg = format("Download failed, error downloading '%s'; %s", url, ex.getMessage());
             throw new DownloadFailedException(msg, ex);
         }
     }
 
     /**
      * Gets a HttpClientContext that supports pre-emptive authentication.
      * @return A HttpClientContext pre-configured with the authentication cache build from the settings.
      */
     public HttpClientContext getPreEmptiveAuthContext() {
         final HttpClientContext context = HttpClientContext.create();
         context.setCredentialsProvider(credentialsProvider);
         context.setAuthCache(authCache);
         return context;
     }
 
     /**
      * Gets a pre-configured HttpClient.
      * Mainly targeted for use in paged resultset scenarios with multiple roundtrips.
      * @param useProxy Whether to use the configuration that includes proxy-settings
      * @return A HttpClient pre-configured with the settings.
      */
     public CloseableHttpClient getHttpClient(boolean useProxy) {
         return useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build();
     }
 
     /**
      * Download a resource from the given URL and have its content handled by the given ResponseHandler.
      *
      * @param url             The url of the resource
      * @param handler   The responsehandler to handle the response
      * @param <T>             The return-type for the responseHandler
      * @return The response handler result
      * @throws IOException               on I/O Exceptions
      * @throws TooManyRequestsException  When HTTP status 429 is encountered
      * @throws ResourceNotFoundException When HTTP status 404 is encountered
      */
     public <T> T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler<T> handler)
             throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException {
         return fetchAndHandle(url, handler, Collections.emptyList(), true);
     }
 
     /**
      * Download a resource from the given URL and have its content handled by the given ResponseHandler.
      *
      * @param url               The url of the resource
      * @param handler   The responsehandler to handle the response
      * @param hdr Additional headers to add to the HTTP request
      * @param <T>               The return-type for the responseHandler
      * @return The response handler result
      * @throws IOException               on I/O Exceptions
      * @throws TooManyRequestsException  When HTTP status 429 is encountered
      * @throws ResourceNotFoundException When HTTP status 404 is encountered
      */
     public <T> T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler<T> handler, @NotNull List<Header> hdr)
             throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException {
         return fetchAndHandle(url, handler, hdr, true);
     }
 
     /**
      * Download a resource from the given URL and have its content handled by the given ResponseHandler.
      *
      * @param url               The url of the resource
      * @param handler   The responsehandler to handle the response
      * @param hdr Additional headers to add to the HTTP request
      * @param useProxy          Whether to use the configured proxy for the connection
      * @param <T>               The return-type for the responseHandler
      * @return The response handler result
      * @throws IOException               on I/O Exceptions
      * @throws TooManyRequestsException  When HTTP status 429 is encountered
      * @throws ResourceNotFoundException When HTTP status 404 is encountered
      */
     public <T> T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler<T> handler, @NotNull List<Header> hdr, boolean useProxy)
             throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException {
         final T data;
         if ("file".equals(url.getProtocol())) {
             final Path p = Paths.get(url.toURI());
             try (InputStream is = Files.newInputStream(p)) {
                 final HttpEntity dummyEntity = new BasicHttpEntity(is, ContentType.APPLICATION_JSON);
                 final ClassicHttpResponse dummyResponse = new BasicClassicHttpResponse(200);
                 dummyResponse.setEntity(dummyEntity);
                 data = handler.handleResponse(dummyResponse);
             } catch (HttpException e) {
                 throw new IllegalStateException("HttpException encountered emulating a HTTP response from a file", e);
             }
         } else {
             try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) {
                 return fetchAndHandle(hc, url, handler, hdr);
             }
         }
         return data;
     }
     /**
      * Download a resource from the given URL and have its content handled by the given ResponseHandler.
      *
      * @param client            The HTTP Client to reuse for the request
      * @param url               The url of the resource
      * @param handler   The responsehandler to handle the response
      * @param hdr Additional headers to add to the HTTP request
      * @param <T>               The return-type for the responseHandler
      * @return The response handler result
      * @throws IOException               on I/O Exceptions
      * @throws TooManyRequestsException  When HTTP status 429 is encountered
      * @throws ResourceNotFoundException When HTTP status 404 is encountered
      */
     public <T> T fetchAndHandle(@NotNull CloseableHttpClient client, @NotNull URL url, @NotNull HttpClientResponseHandler<T> handler,
                                 @NotNull List<Header> hdr) throws IOException, TooManyRequestsException, ResourceNotFoundException {
         try {
             final String theProtocol = url.getProtocol();
             if (!("http".equals(theProtocol) || "https".equals(theProtocol))) {
                 throw new DownloadFailedException("Unsupported protocol in the URL; only http and https are supported");
             }
             final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI());
             for (Header h : hdr) {
                 req.addHeader(h);
             }
             final HttpClientContext context = getPreEmptiveAuthContext();
             return client.execute(req, context, handler);
         } catch (HttpResponseException hre) {
             final String messageFormat = "%s - Server status: %d - Server reason: %s";
             switch (hre.getStatusCode()) {
                 case 404:
                     throw new ResourceNotFoundException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()));
                 case 429:
                     throw new TooManyRequestsException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()));
                 default:
                     throw new IOException(String.format(messageFormat, url, hre.getStatusCode(), hre.getReasonPhrase()));
             }
         } catch (RuntimeException | URISyntaxException ex) {
             final String msg = format("Download failed, unable to retrieve and parse '%s'; %s", url, ex.getMessage());
             throw new IOException(msg, ex);
         }
     }
 
     private static class SelectiveProxySelector extends ProxySelector {
 
         /**
          * The suffix-match entries from the nonProxyHosts (those starting with a {@code *}).
          */
         private final List<String> suffixMatch = new ArrayList<>();
         /**
          * The full host entries from the nonProxyHosts (those <em>not</em> starting with a {@code *}).
          */
         private final List<String> fullmatch = new ArrayList<>();
         /**
          * The proxy use when no proxy-exception is found.
          */
         private final Proxy configuredProxy;
 
         SelectiveProxySelector(Proxy httpHost, String[] nonProxyHostsPatterns) {
             for (String nonProxyHostPattern : nonProxyHostsPatterns) {
                 if (nonProxyHostPattern.startsWith("*")) {
                     suffixMatch.add(nonProxyHostPattern.substring(1));
                 } else {
                     fullmatch.add(nonProxyHostPattern);
                 }
             }
             this.configuredProxy = httpHost;
         }
 
         @Override
         public List<Proxy> select(URI uri) {
             final String theHost = uri.getHost();
             if (fullmatch.contains(theHost)) {
                 return Collections.singletonList(Proxy.NO_PROXY);
             } else {
                 for (String suffix : suffixMatch) {
                     if (theHost.endsWith(suffix)) {
                         return Collections.singletonList(Proxy.NO_PROXY);
                     }
                 }
             }
             return List.of(configuredProxy);
         }
 
         @Override
         public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
             // nothing to be done for this single proxy proxy-selector
         }
     }
 }