View Javadoc
1   /*
2    * This file is part of dependency-check-utils.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   *
16   * Copyright (c) 2018 Jeremy Long. All Rights Reserved.
17   */
18  package org.owasp.dependencycheck.utils;
19  
20  import org.slf4j.Logger;
21  import org.slf4j.LoggerFactory;
22  
23  import java.io.File;
24  import java.io.FileInputStream;
25  import java.io.IOException;
26  import java.io.InputStream;
27  
28  import static java.lang.String.format;
29  
30  import java.net.HttpURLConnection;
31  import java.net.URISyntaxException;
32  import java.net.URL;
33  import java.security.InvalidAlgorithmParameterException;
34  import java.util.zip.GZIPInputStream;
35  import java.util.zip.InflaterInputStream;
36  
37  /**
38   * A utility to download files from the Internet.
39   *
40   * @author Jeremy Long
41   * @version $Id: $Id
42   */
43  public class HttpResourceConnection implements AutoCloseable {
44  
45      /**
46       * The logger.
47       */
48      private static final Logger LOGGER = LoggerFactory.getLogger(HttpResourceConnection.class);
49      /**
50       * The maximum number of redirects that will be followed when attempting to
51       * download a file.
52       */
53      private static final int MAX_REDIRECT_ATTEMPTS = 5;
54      /**
55       * The default HTTP request method for query timestamp
56       */
57      private static final String HEAD = "HEAD";
58  
59      /**
60       * The HTTP request method which can be used by query timestamp
61       */
62      private static final String GET = "GET";
63      /**
64       * The configured settings.
65       */
66      private final Settings settings;
67      /**
68       * The URL conn factory.
69       */
70      private final URLConnectionFactory connFactory;
71      /**
72       * The current conn.
73       */
74      private HttpURLConnection connection = null;
75      /**
76       * Whether or not the conn will use the defined proxy.
77       */
78      private final boolean usesProxy;
79  
80      /**
81       * The settings key for the username to be used.
82       */
83      private String userKey = null;
84      /**
85       * The settings key for the password to be used.
86       */
87      private String passwordKey = null;
88  
89      /**
90       * Constructs a new HttpResourceConnection object.
91       *
92       * @param settings the configured settings
93       */
94      public HttpResourceConnection(Settings settings) {
95          this(settings, true);
96      }
97  
98      /**
99       * Constructs a new HttpResourceConnection object.
100      *
101      * @param settings the configured settings
102      * @param usesProxy control whether this conn will use the defined proxy.
103      */
104     public HttpResourceConnection(Settings settings, boolean usesProxy) {
105         this.settings = settings;
106         this.connFactory = new URLConnectionFactory(settings);
107         this.usesProxy = usesProxy;
108     }
109 
110     /**
111      * Constructs a new HttpResourceConnection object.
112      *
113      * @param settings the configured settings
114      * @param usesProxy control whether this conn will use the defined proxy
115      * @param userKey the settings key for the username to be used
116      * @param passwordKey the settings key for the password to be used
117      */
118     public HttpResourceConnection(Settings settings, boolean usesProxy, String userKey, String passwordKey) {
119         this.settings = settings;
120         this.connFactory = new URLConnectionFactory(settings);
121         this.usesProxy = usesProxy;
122         this.userKey = userKey;
123         this.passwordKey = passwordKey;
124     }
125 
126     /**
127      * Retrieves the resource identified by the given URL and returns the
128      * InputStream.
129      *
130      * @param url the URL of the resource to download
131      * @return the stream to read the retrieved content from
132      * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown
133      * if there is an error downloading the resource
134      * @throws TooManyRequestsException thrown when a 429 is received
135      * @throws ResourceNotFoundException thrown when a 404 is received
136      */
137     public InputStream fetch(URL url) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
138         if ("file".equalsIgnoreCase(url.getProtocol())) {
139             final File file;
140             try {
141                 file = new File(url.toURI());
142             } catch (URISyntaxException ex) {
143                 final String msg = format("Download failed, unable to locate '%s'", url);
144                 throw new DownloadFailedException(msg);
145             }
146             if (file.exists()) {
147                 try {
148                     return new FileInputStream(file);
149                 } catch (IOException ex) {
150                     final String msg = format("Download failed, unable to rerieve '%s'", url);
151                     throw new DownloadFailedException(msg, ex);
152                 }
153             } else {
154                 final String msg = format("Download failed, file ('%s') does not exist", url);
155                 throw new DownloadFailedException(msg);
156             }
157         } else {
158             if (connection != null) {
159                 LOGGER.warn("HTTP URL Connection was not properly closed");
160                 connection.disconnect();
161                 connection = null;
162             }
163             connection = obtainConnection(url);
164 
165             final String encoding = connection.getContentEncoding();
166             try {
167                 if ("gzip".equalsIgnoreCase(encoding)) {
168                     return new GZIPInputStream(connection.getInputStream());
169                 } else if ("deflate".equalsIgnoreCase(encoding)) {
170                     return new InflaterInputStream(connection.getInputStream());
171                 } else {
172                     return connection.getInputStream();
173                 }
174             } catch (IOException ex) {
175                 checkForCommonExceptionTypes(ex);
176                 final String msg = format("Error retrieving '%s'%nConnection Timeout: %d%nEncoding: %s%n",
177                         url, connection.getConnectTimeout(), encoding);
178                 throw new DownloadFailedException(msg, ex);
179             } catch (Exception ex) {
180                 final String msg = format("Unexpected exception retrieving '%s'%nConnection Timeout: %d%nEncoding: %s%n",
181                         url, connection.getConnectTimeout(), encoding);
182                 throw new DownloadFailedException(msg, ex);
183             }
184         }
185     }
186 
187     /**
188      * Obtains the HTTP URL Connection.
189      *
190      * @param url the URL
191      * @return the HTTP URL Connection
192      * @throws DownloadFailedException thrown if there is an error creating the
193      * HTTP URL Connection
194      * @throws TooManyRequestsException thrown when a 429 is received
195      * @throws ResourceNotFoundException thrown when a 404 is received
196      */
197     private HttpURLConnection obtainConnection(URL url) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
198         HttpURLConnection conn = null;
199         try {
200             LOGGER.debug("Attempting retrieval of {}", url.toString());
201             conn = connFactory.createHttpURLConnection(url, this.usesProxy);
202             if (userKey != null && passwordKey != null) {
203                 connFactory.addBasicAuthentication(conn, userKey, passwordKey);
204             }
205             conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
206             conn.connect();
207             int status = conn.getResponseCode();
208             final String message = conn.getResponseMessage();
209             int redirectCount = 0;
210             // TODO - should this get replaced by using the conn.setInstanceFollowRedirects(true);
211             while ((status == HttpURLConnection.HTTP_MOVED_TEMP
212                     || status == HttpURLConnection.HTTP_MOVED_PERM
213                     || status == HttpURLConnection.HTTP_SEE_OTHER)
214                     && MAX_REDIRECT_ATTEMPTS > redirectCount++) {
215                 final String location = conn.getHeaderField("Location");
216                 try {
217                     conn.disconnect();
218                 } finally {
219                     conn = null;
220                 }
221                 LOGGER.debug("Download is being redirected from {} to {}", url, location);
222                 conn = connFactory.createHttpURLConnection(new URL(location), this.usesProxy);
223                 conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
224                 conn.connect();
225                 status = conn.getResponseCode();
226             }
227             if (status == 404) {
228                 try {
229                     conn.disconnect();
230                 } finally {
231                     conn = null;
232                 }
233                 throw new ResourceNotFoundException("Requested resource does not exist - received a 404");
234             } else if (status == 429) {
235                 try {
236                     conn.disconnect();
237                 } finally {
238                     conn = null;
239                 }
240                 throw new TooManyRequestsException("Download failed - too many connection requests");
241             } else if (status != 200) {
242                 try {
243                     conn.disconnect();
244                 } finally {
245                     conn = null;
246                 }
247                 final String msg = format("Error retrieving %s; received response code %s; %s", url, status, message);
248                 LOGGER.error(msg);
249                 throw new DownloadFailedException(msg);
250             }
251         } catch (IOException ex) {
252             try {
253                 if (conn != null) {
254                     conn.disconnect();
255                 }
256             } finally {
257                 conn = null;
258             }
259             if ("Connection reset".equalsIgnoreCase(ex.getMessage())) {
260                 final String msg = format("TLS Connection Reset%nPlease see "
261                         + "http://jeremylong.github.io/DependencyCheck/data/tlsfailure.html "
262                         + "for more information regarding how to resolve the issue.");
263                 LOGGER.error(msg);
264                 throw new DownloadFailedException(msg, ex);
265             }
266             final String msg = format("Error downloading file %s; unable to connect.", url);
267             throw new DownloadFailedException(msg, ex);
268         }
269         return conn;
270     }
271 
272     /**
273      * {@inheritDoc}
274      * <p>
275      * Releases the underlying HTTP URL Connection.
276      */
277     @Override
278     public void close() {
279         if (connection != null) {
280             try {
281                 connection.disconnect();
282             } finally {
283                 connection = null;
284             }
285         }
286     }
287 
288     /**
289      * Returns whether or not the connection has been closed.
290      *
291      * @return whether or not the connection has been closed
292      */
293     public boolean isClosed() {
294         return connection == null;
295     }
296 
297     /**
298      * Returns the HEAD or GET HTTP method. HEAD is the default.
299      *
300      * @return the HTTP method to use
301      */
302     private String determineHttpMethod() {
303         return isQuickQuery() ? HEAD : GET;
304     }
305 
306     /**
307      * Determines if the HTTP method GET or HEAD should be used to check the
308      * timestamp on external resources.
309      *
310      * @return true if configured to use HEAD requests
311      */
312     private boolean isQuickQuery() {
313         return settings.getBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, true);
314     }
315 
316     /**
317      * Analyzes the IOException, logs the appropriate information for debugging
318      * purposes, and then throws a DownloadFailedException that wraps the IO
319      * Exception for common IO Exceptions. This is to provide additional details
320      * to assist in resolution of the exception.
321      *
322      * @param ex the original exception
323      * @throws org.owasp.dependencycheck.utils.DownloadFailedException a wrapper
324      * exception that contains the original exception as the cause
325      */
326     public void checkForCommonExceptionTypes(IOException ex) throws DownloadFailedException {
327         Throwable cause = ex;
328         while (cause != null) {
329             if (cause instanceof java.net.UnknownHostException) {
330                 final String msg = format("Unable to resolve domain '%s'", cause.getMessage());
331                 LOGGER.error(msg);
332                 throw new DownloadFailedException(msg);
333             }
334             if (cause instanceof InvalidAlgorithmParameterException) {
335                 final String keystore = System.getProperty("javax.net.ssl.keyStore");
336                 final String version = System.getProperty("java.version");
337                 final String vendor = System.getProperty("java.vendor");
338                 LOGGER.info("Error making HTTPS request - InvalidAlgorithmParameterException");
339                 LOGGER.info("There appears to be an issue with the installation of Java and the cacerts."
340                         + "See closed issue #177 here: https://github.com/jeremylong/DependencyCheck/issues/177");
341                 LOGGER.info("Java Info:\njavax.net.ssl.keyStore='{}'\njava.version='{}'\njava.vendor='{}'",
342                         keystore, version, vendor);
343                 throw new DownloadFailedException("Error making HTTPS request. Please see the log for more details.");
344             }
345             cause = cause.getCause();
346         }
347     }
348 }