1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.central;
19
20 import org.owasp.dependencycheck.utils.TooManyRequestsException;
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.net.HttpURLConnection;
24 import java.net.MalformedURLException;
25 import java.net.URISyntaxException;
26 import java.net.URL;
27 import java.util.ArrayList;
28 import java.util.List;
29 import javax.annotation.concurrent.ThreadSafe;
30 import javax.xml.parsers.DocumentBuilder;
31 import javax.xml.parsers.ParserConfigurationException;
32 import javax.xml.xpath.XPath;
33 import javax.xml.xpath.XPathConstants;
34 import javax.xml.xpath.XPathExpressionException;
35 import javax.xml.xpath.XPathFactory;
36 import org.apache.commons.jcs3.access.exception.CacheException;
37 import org.owasp.dependencycheck.data.cache.DataCache;
38 import org.owasp.dependencycheck.data.cache.DataCacheFactory;
39 import org.owasp.dependencycheck.data.nexus.MavenArtifact;
40 import org.owasp.dependencycheck.utils.Settings;
41 import org.owasp.dependencycheck.utils.URLConnectionFactory;
42 import org.owasp.dependencycheck.utils.XmlUtils;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45 import org.w3c.dom.Document;
46 import org.w3c.dom.NodeList;
47 import org.xml.sax.SAXException;
48
49
50
51
52
53
54 @ThreadSafe
55 public class CentralSearch {
56
57
58
59
60 private final String rootURL;
61
62
63
64
65 private final String query;
66
67
68
69
70 private final boolean useProxy;
71
72
73
74
75 private static final Logger LOGGER = LoggerFactory.getLogger(CentralSearch.class);
76
77
78
79 private final Settings settings;
80
81
82
83 private DataCache<List<MavenArtifact>> cache;
84
85
86
87
88
89
90
91 public CentralSearch(Settings settings) throws MalformedURLException {
92 this.settings = settings;
93
94 final String searchUrl = settings.getString(Settings.KEYS.ANALYZER_CENTRAL_URL);
95 LOGGER.debug("Central Search URL: {}", searchUrl);
96 if (isInvalidURL(searchUrl)) {
97 throw new MalformedURLException(String.format("The configured central analyzer URL is invalid: %s", searchUrl));
98 }
99 this.rootURL = searchUrl;
100 final String queryStr = settings.getString(Settings.KEYS.ANALYZER_CENTRAL_QUERY);
101 LOGGER.debug("Central Search Query: {}", queryStr);
102 if (!queryStr.matches("^%s.*%s.*$")) {
103 final String msg = String.format("The configured central analyzer query parameter is invalid (it must have two %%s): %s", queryStr);
104 throw new MalformedURLException(msg);
105 }
106 this.query = queryStr;
107 LOGGER.debug("Central Search Full URL: {}", String.format(query, rootURL, "[SHA1]"));
108 if (null != settings.getString(Settings.KEYS.PROXY_SERVER)) {
109 useProxy = true;
110 LOGGER.debug("Using proxy");
111 } else {
112 useProxy = false;
113 LOGGER.debug("Not using proxy");
114 }
115 if (settings.getBoolean(Settings.KEYS.ANALYZER_CENTRAL_USE_CACHE, true)) {
116 try {
117 final DataCacheFactory factory = new DataCacheFactory(settings);
118 cache = factory.getCentralCache();
119 } catch (CacheException ex) {
120 settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_USE_CACHE, false);
121 LOGGER.debug("Error creating cache, disabling caching", ex);
122 }
123 }
124 }
125
126
127
128
129
130
131
132
133
134
135
136
137
138 public List<MavenArtifact> searchSha1(String sha1) throws IOException, TooManyRequestsException {
139 if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) {
140 throw new IllegalArgumentException("Invalid SHA1 format");
141 }
142 if (cache != null) {
143 final List<MavenArtifact> cached = cache.get(sha1);
144 if (cached != null) {
145 LOGGER.debug("cache hit for Central: " + sha1);
146 if (cached.isEmpty()) {
147 throw new FileNotFoundException("Artifact not found in Central");
148 }
149 return cached;
150 }
151 }
152 final List<MavenArtifact> result = new ArrayList<>();
153 final URL url = new URL(String.format(query, rootURL, sha1));
154
155 LOGGER.trace("Searching Central url {}", url);
156
157
158
159
160
161 final URLConnectionFactory factory = new URLConnectionFactory(settings);
162 final HttpURLConnection conn = factory.createHttpURLConnection(url, useProxy);
163
164 conn.setDoOutput(true);
165
166
167
168 conn.addRequestProperty("Accept", "application/xml");
169 conn.connect();
170
171 if (conn.getResponseCode() == 200) {
172 boolean missing = false;
173 try {
174 final DocumentBuilder builder = XmlUtils.buildSecureDocumentBuilder();
175 final Document doc = builder.parse(conn.getInputStream());
176 final XPath xpath = XPathFactory.newInstance().newXPath();
177 final String numFound = xpath.evaluate("/response/result/@numFound", doc);
178 if ("0".equals(numFound)) {
179 missing = true;
180 } else {
181 final NodeList docs = (NodeList) xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET);
182 for (int i = 0; i < docs.getLength(); i++) {
183 final String g = xpath.evaluate("./str[@name='g']", docs.item(i));
184 LOGGER.trace("GroupId: {}", g);
185 final String a = xpath.evaluate("./str[@name='a']", docs.item(i));
186 LOGGER.trace("ArtifactId: {}", a);
187 final String v = xpath.evaluate("./str[@name='v']", docs.item(i));
188 final NodeList attributes = (NodeList) xpath.evaluate("./arr[@name='ec']/str", docs.item(i), XPathConstants.NODESET);
189 boolean pomAvailable = false;
190 boolean jarAvailable = false;
191 for (int x = 0; x < attributes.getLength(); x++) {
192 final String tmp = xpath.evaluate(".", attributes.item(x));
193 if (".pom".equals(tmp)) {
194 pomAvailable = true;
195 } else if (".jar".equals(tmp)) {
196 jarAvailable = true;
197 }
198 }
199 final String centralContentUrl = settings.getString(Settings.KEYS.CENTRAL_CONTENT_URL);
200 String artifactUrl = null;
201 String pomUrl = null;
202 if (jarAvailable) {
203
204 artifactUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/'
205 + v + '/' + a + '-' + v + ".jar";
206 }
207 if (pomAvailable) {
208
209 pomUrl = centralContentUrl + g.replace('.', '/') + '/' + a + '/'
210 + v + '/' + a + '-' + v + ".pom";
211 }
212 result.add(new MavenArtifact(g, a, v, artifactUrl, pomUrl));
213 }
214 }
215 } catch (ParserConfigurationException | IOException | SAXException | XPathExpressionException e) {
216
217 final String errorMessage = "Failed to parse MavenCentral XML Response: " + e.getMessage();
218 throw new IOException(errorMessage, e);
219 }
220
221 if (missing) {
222 if (cache != null) {
223 cache.put(sha1, result);
224 }
225 throw new FileNotFoundException("Artifact not found in Central");
226 }
227 } else if (conn.getResponseCode() == 429) {
228 final String errorMessage = "Too many requests sent to MavenCentral; additional requests are being rejected.";
229 throw new TooManyRequestsException(errorMessage);
230 } else {
231 final String errorMessage = "Could not connect to MavenCentral (" + conn.getResponseCode() + "): " + conn.getResponseMessage();
232 throw new IOException(errorMessage);
233 }
234 if (cache != null) {
235 cache.put(sha1, result);
236 }
237 return result;
238 }
239
240
241
242
243
244
245
246 private boolean isInvalidURL(String url) {
247 try {
248 final URL u = new URL(url);
249 u.toURI();
250 } catch (MalformedURLException | URISyntaxException e) {
251 LOGGER.trace("URL is invalid: {}", url);
252 return true;
253 }
254 return false;
255 }
256 }