1 /*
2 * This file is part of dependency-check-utils.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *
16 * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
17 */
18 package org.owasp.dependencycheck.utils;
19
20 import java.io.ByteArrayOutputStream;
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.OutputStream;
26 import java.net.URL;
27
28 import static java.lang.String.format;
29
30 import java.nio.Buffer;
31 import java.nio.ByteBuffer;
32 import java.nio.channels.Channels;
33 import java.nio.channels.FileChannel;
34 import java.nio.channels.ReadableByteChannel;
35 import java.nio.charset.StandardCharsets;
36 import java.util.zip.GZIPInputStream;
37
38 import org.apache.commons.io.IOUtils;
39 import org.slf4j.Logger;
40 import org.slf4j.LoggerFactory;
41
42 /**
43 * A utility to download files from the Internet.
44 *
45 * @author Jeremy Long
46 * @version $Id: $Id
47 */
48 public final class Downloader {
49
50 /**
51 * UTF-8 character set name.
52 */
53 private static final String UTF8 = StandardCharsets.UTF_8.name();
54 /**
55 * The Logger for use throughout the class.
56 */
57 private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class);
58 /**
59 * The configured settings.
60 */
61 private final Settings settings;
62
63 /**
64 * Constructs a new Downloader object.
65 *
66 * @param settings the configured settings
67 */
68 public Downloader(Settings settings) {
69 this.settings = settings;
70 }
71
72 /**
73 * Retrieves a file from a given URL and saves it to the outputPath.
74 *
75 * @param url the URL of the file to download
76 * @param outputPath the path to the save the file to
77 * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown
78 * if there is an error downloading the file
79 * @throws TooManyRequestsException thrown when a 429 is received
80 * @throws ResourceNotFoundException thrown when a 404 is received
81 */
82 public void fetchFile(URL url, File outputPath) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
83 fetchFile(url, outputPath, true, null, null);
84 }
85
86 /**
87 * Retrieves a file from a given URL and saves it to the outputPath.
88 *
89 * @param url the URL of the file to download
90 * @param outputPath the path to the save the file to
91 * @param userKey the settings key for the username to be used
92 * @param passwordKey the settings key for the password to be used
93 * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown
94 * if there is an error downloading the file
95 * @throws TooManyRequestsException thrown when a 429 is received
96 * @throws ResourceNotFoundException thrown when a 404 is received
97 */
98 public void fetchFile(URL url, File outputPath, String userKey, String passwordKey)
99 throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
100 fetchFile(url, outputPath, true, userKey, passwordKey);
101 }
102
103 /**
104 * Retrieves a file from a given URL and saves it to the outputPath.
105 *
106 * @param url the URL of the file to download
107 * @param outputPath the path to the save the file to
108 * @param useProxy whether to use the configured proxy when downloading
109 * files
110 * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown
111 * if there is an error downloading the file
112 * @throws TooManyRequestsException thrown when a 429 is received
113 * @throws ResourceNotFoundException thrown when a 404 is received
114 */
115 public void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException,
116 TooManyRequestsException, ResourceNotFoundException {
117 fetchFile(url, outputPath, useProxy, null, null);
118 }
119
120 /**
121 * Retrieves a file from a given URL and saves it to the outputPath.
122 *
123 * @param url the URL of the file to download
124 * @param outputPath the path to the save the file to
125 * @param useProxy whether to use the configured proxy when downloading
126 * files
127 * @param userKey the settings key for the username to be used
128 * @param passwordKey the settings key for the password to be used
129 * @throws org.owasp.dependencycheck.utils.DownloadFailedException is thrown
130 * if there is an error downloading the file
131 * @throws TooManyRequestsException thrown when a 429 is received
132 * @throws ResourceNotFoundException thrown when a 404 is received
133 */
134 public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey, String passwordKey) throws DownloadFailedException,
135 TooManyRequestsException, ResourceNotFoundException {
136 InputStream in = null;
137 try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey)) {
138 in = conn.fetch(url);
139 try (ReadableByteChannel sourceChannel = Channels.newChannel(in);
140 FileChannel destChannel = new FileOutputStream(outputPath).getChannel()) {
141 ByteBuffer buffer = ByteBuffer.allocateDirect(8192);
142 while (sourceChannel.read(buffer) != -1) {
143 // cast is a workaround, see https://github.com/plasma-umass/doppio/issues/497#issuecomment-334740243
144 ((Buffer)buffer).flip();
145 destChannel.write(buffer);
146 buffer.compact();
147 }
148 }
149 } catch (IOException ex) {
150 final String msg = format("Download failed, unable to copy '%s' to '%s'; %s",
151 url.toString(), outputPath.getAbsolutePath(), ex.getMessage());
152 throw new DownloadFailedException(msg, ex);
153 } finally {
154 if (in != null) {
155 try {
156 in.close();
157 } catch (IOException ex) {
158 LOGGER.trace("Ignorable error", ex);
159 }
160 }
161 }
162 }
163
164 /**
165 * Retrieves a file from a given URL and returns the contents.
166 *
167 * @param url the URL of the file to download
168 * @param useProxy whether to use the configured proxy when downloading
169 * files
170 * @return the content of the file
171 * @throws DownloadFailedException is thrown if there is an error
172 * downloading the file
173 * @throws TooManyRequestsException thrown when a 429 is received
174 * @throws ResourceNotFoundException thrown when a 404 is received
175 */
176 public String fetchContent(URL url, boolean useProxy) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
177 return fetchContent(url, useProxy, null, null);
178 }
179
180 /**
181 * Retrieves a file from a given URL and returns the contents.
182 *
183 * @param url the URL of the file to download
184 * @param useProxy whether to use the configured proxy when downloading
185 * files
186 * @param userKey the settings key for the username to be used
187 * @param passwordKey the settings key for the password to be used
188 * @return the content of the file
189 * @throws DownloadFailedException is thrown if there is an error
190 * downloading the file
191 * @throws TooManyRequestsException thrown when a 429 is received
192 * @throws ResourceNotFoundException thrown when a 404 is received
193 */
194 public String fetchContent(URL url, boolean useProxy, String userKey, String passwordKey)
195 throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
196 InputStream in = null;
197 try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey);
198 ByteArrayOutputStream out = new ByteArrayOutputStream()) {
199 in = conn.fetch(url);
200 IOUtils.copy(in, out);
201 return out.toString(UTF8);
202 } catch (IOException ex) {
203 final String msg = format("Download failed, unable to retrieve '%s'; %s", url, ex.getMessage());
204 throw new DownloadFailedException(msg, ex);
205 } finally {
206 if (in != null) {
207 try {
208 in.close();
209 } catch (IOException ex) {
210 LOGGER.trace("Ignorable error", ex);
211 }
212 }
213 }
214 }
215 }