1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.update;
19
20 import com.fasterxml.jackson.databind.ObjectMapper;
21 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
22 import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
23 import io.github.jeremylong.openvulnerability.client.nvd.NvdApiException;
24 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient;
25 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder;
26 import java.io.File;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.StringReader;
30 import java.net.URI;
31 import java.net.URISyntaxException;
32 import java.net.URL;
33 import java.nio.charset.StandardCharsets;
34 import java.text.MessageFormat;
35 import java.time.Duration;
36 import java.time.ZoneId;
37 import java.time.ZonedDateTime;
38 import java.util.ArrayList;
39 import java.util.Collection;
40 import java.util.HashMap;
41 import java.util.HashSet;
42 import java.util.List;
43 import java.util.Map;
44 import java.util.Properties;
45 import java.util.Set;
46 import java.util.concurrent.ExecutionException;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.zip.GZIPOutputStream;
51 import org.owasp.dependencycheck.Engine;
52 import org.owasp.dependencycheck.data.nvdcve.CveDB;
53 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
54 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
55 import org.owasp.dependencycheck.data.update.exception.UpdateException;
56 import org.owasp.dependencycheck.data.update.nvd.api.DownloadTask;
57 import org.owasp.dependencycheck.data.update.nvd.api.NvdApiProcessor;
58 import org.owasp.dependencycheck.utils.DateUtil;
59 import org.owasp.dependencycheck.utils.DownloadFailedException;
60 import org.owasp.dependencycheck.utils.Downloader;
61 import org.owasp.dependencycheck.utils.InvalidSettingException;
62 import org.owasp.dependencycheck.utils.ResourceNotFoundException;
63 import org.owasp.dependencycheck.utils.Settings;
64 import org.owasp.dependencycheck.utils.TooManyRequestsException;
65 import org.slf4j.Logger;
66 import org.slf4j.LoggerFactory;
67
68
69
70
71
72 public class NvdApiDataSource implements CachedWebDataSource {
73
74
75
76
77 private static final Logger LOGGER = LoggerFactory.getLogger(NvdApiDataSource.class);
78
79
80
81 private static final int PROCESSING_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors();
82
83
84
85 private Settings settings;
86
87
88
89 private CveDB cveDb = null;
90
91
92
93 private DatabaseProperties dbProperties = null;
94
95
96
97 private static final String NVD_API_CACHE_MODIFIED_DATE = "lastModifiedDate";
98
99
100
101
102 private static final int RESULTS_PER_PAGE = 2000;
103
104 @Override
105 public boolean update(Engine engine) throws UpdateException {
106 this.settings = engine.getSettings();
107 this.cveDb = engine.getDatabase();
108 if (isUpdateConfiguredFalse()) {
109 return false;
110 }
111 dbProperties = cveDb.getDatabaseProperties();
112
113 final String nvdDataFeedUrl = settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL);
114 if (nvdDataFeedUrl != null) {
115 return processDatafeed(nvdDataFeedUrl);
116 }
117 return processApi();
118 }
119
120 protected UrlData extractUrlData(String nvdDataFeedUrl) {
121 String url;
122 String pattern = null;
123 if (nvdDataFeedUrl.endsWith(".json.gz")) {
124 final int lio = nvdDataFeedUrl.lastIndexOf("/");
125 pattern = nvdDataFeedUrl.substring(lio + 1);
126 url = nvdDataFeedUrl.substring(0, lio);
127 } else {
128 url = nvdDataFeedUrl;
129 }
130 if (!url.endsWith("/")) {
131 url += "/";
132 }
133 return new UrlData(url, pattern);
134 }
135
136 private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException {
137 boolean updatesMade = false;
138 try {
139 dbProperties = cveDb.getDatabaseProperties();
140 if (checkUpdate()) {
141 final UrlData data = extractUrlData(nvdDataFeedUrl);
142 final String url = data.getUrl();
143 String pattern = data.getPattern();
144 final Properties cacheProperties = getRemoteCacheProperties(url, pattern);
145 if (pattern == null) {
146 final String prefix = cacheProperties.getProperty("prefix", "nvdcve-");
147 pattern = prefix + "{0}.json.gz";
148 }
149
150 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
151 final Map<String, String> updateable = getUpdatesNeeded(url, pattern, cacheProperties, now);
152 if (!updateable.isEmpty()) {
153 final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1);
154 final int downloadPoolSize = Math.min(Runtime.getRuntime().availableProcessors(), max);
155
156 final int maxExec = PROCESSING_THREAD_POOL_SIZE;
157 final int execPoolSize = Math.min(maxExec, 2);
158
159 ExecutorService processingExecutorService = null;
160 ExecutorService downloadExecutorService = null;
161 try {
162 downloadExecutorService = Executors.newFixedThreadPool(downloadPoolSize);
163 processingExecutorService = Executors.newFixedThreadPool(execPoolSize);
164
165 DownloadTask runLast = null;
166 final Set<Future<Future<NvdApiProcessor>>> downloadFutures = new HashSet<>(updateable.size());
167 runLast = startDownloads(updateable, processingExecutorService, runLast, downloadFutures, downloadExecutorService);
168
169
170 final Set<Future<NvdApiProcessor>> processFutures = new HashSet<>(updateable.size());
171 for (Future<Future<NvdApiProcessor>> future : downloadFutures) {
172 processDownload(future, processFutures);
173 }
174
175 processFuture(processFutures);
176 processFutures.clear();
177
178
179 if (runLast != null) {
180 final Future<Future<NvdApiProcessor>> modified = downloadExecutorService.submit(runLast);
181 processDownload(modified, processFutures);
182 processFuture(processFutures);
183 }
184
185 } finally {
186 if (processingExecutorService != null) {
187 processingExecutorService.shutdownNow();
188 }
189 if (downloadExecutorService != null) {
190 downloadExecutorService.shutdownNow();
191 }
192 }
193 updatesMade = true;
194 }
195 storeLastModifiedDates(now, cacheProperties, updateable);
196 if (updatesMade) {
197 cveDb.persistEcosystemCache();
198 }
199 final int updateCount = cveDb.updateEcosystemCache();
200 LOGGER.debug("Corrected the ecosystem for {} ecoSystemCache entries", updateCount);
201 if (updatesMade || updateCount > 0) {
202 cveDb.cleanupDatabase();
203 }
204 }
205 } catch (UpdateException ex) {
206 if (ex.getCause() != null && ex.getCause() instanceof DownloadFailedException) {
207 final String jre = System.getProperty("java.version");
208 if (jre == null || jre.startsWith("1.4") || jre.startsWith("1.5") || jre.startsWith("1.6") || jre.startsWith("1.7")) {
209 LOGGER.error("An old JRE is being used ({} {}), and likely does not have the correct root certificates or algorithms "
210 + "to connect to the NVD - consider upgrading your JRE.", System.getProperty("java.vendor"), jre);
211 }
212 }
213 throw ex;
214 } catch (DatabaseException ex) {
215 throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex);
216 }
217 return updatesMade;
218 }
219
220 private void storeLastModifiedDates(final ZonedDateTime now, final Properties cacheProperties,
221 final Map<String, String> updateable) throws UpdateException {
222
223 final ZonedDateTime lastModifiedRequest = DatabaseProperties.getTimestamp(cacheProperties,
224 NVD_API_CACHE_MODIFIED_DATE + ".modified");
225 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_CHECKED, now);
226 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED, lastModifiedRequest);
227
228 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, now);
229 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
230
231 for (String entry : updateable.keySet()) {
232 final ZonedDateTime date = DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE + "." + entry);
233 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + entry, date);
234 }
235 }
236
237 private DownloadTask startDownloads(final Map<String, String> updateable, ExecutorService processingExecutorService, DownloadTask runLast,
238 final Set<Future<Future<NvdApiProcessor>>> downloadFutures, ExecutorService downloadExecutorService) throws UpdateException {
239 DownloadTask lastCall = runLast;
240 for (Map.Entry<String, String> cve : updateable.entrySet()) {
241 final DownloadTask call = new DownloadTask(cve.getValue(), processingExecutorService, cveDb, settings);
242 if (call.isModified()) {
243 lastCall = call;
244 } else {
245 final boolean added = downloadFutures.add(downloadExecutorService.submit(call));
246 if (!added) {
247 throw new UpdateException("Unable to add the download task for " + cve);
248 }
249 }
250 }
251 return lastCall;
252 }
253
254 private void processFuture(final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
255
256 for (Future<NvdApiProcessor> future : processFutures) {
257 try {
258 final NvdApiProcessor task = future.get();
259 } catch (InterruptedException ex) {
260 LOGGER.debug("Thread was interrupted during processing", ex);
261 Thread.currentThread().interrupt();
262 throw new UpdateException(ex);
263 } catch (ExecutionException ex) {
264 LOGGER.debug("Execution Exception during process", ex);
265 throw new UpdateException(ex);
266 }
267 }
268 }
269
270 private void processDownload(Future<Future<NvdApiProcessor>> future, final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
271 final Future<NvdApiProcessor> task;
272 try {
273 task = future.get();
274 if (task != null) {
275 processFutures.add(task);
276 }
277 } catch (InterruptedException ex) {
278 LOGGER.debug("Thread was interrupted during download", ex);
279 Thread.currentThread().interrupt();
280 throw new UpdateException("The download was interrupted", ex);
281 } catch (ExecutionException ex) {
282 LOGGER.debug("Thread was interrupted during download execution", ex);
283 throw new UpdateException("The execution of the download was interrupted", ex);
284 }
285 }
286
287 private boolean processApi() throws UpdateException {
288 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_CHECKED);
289 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
290 if (cveDb.dataExists() && lastChecked != null && validForHours > 0) {
291
292 final long validForSeconds = validForHours * 60L * 60L;
293 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
294 final Duration duration = Duration.between(lastChecked, now);
295 final long difference = duration.getSeconds();
296 if (difference < validForSeconds) {
297 LOGGER.info("Skipping the NVD API Update as it was completed within the last {} minutes", validForSeconds / 60);
298 return false;
299 }
300 }
301
302 ZonedDateTime lastModifiedRequest = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_MODIFIED);
303 final NvdCveClientBuilder builder = NvdCveClientBuilder.aNvdCveApi();
304 final String endpoint = settings.getString(Settings.KEYS.NVD_API_ENDPOINT);
305 if (endpoint != null) {
306 builder.withEndpoint(endpoint);
307 }
308 if (lastModifiedRequest != null) {
309
310 lastModifiedRequest = lastModifiedRequest.withZoneSameInstant(ZoneId.of("UTC"));
311 final ZonedDateTime end = lastModifiedRequest.plusDays(120);
312 builder.withLastModifiedFilter(lastModifiedRequest, end);
313 }
314 final String key = settings.getString(Settings.KEYS.NVD_API_KEY);
315 if (key != null) {
316
317 builder.withApiKey(key)
318 .withDelay(5000)
319 .withThreadCount(4);
320 } else {
321 LOGGER.warn("An NVD API Key was not provided - it is highly recommended to use "
322 + "an NVD API key as the update can take a VERY long time without an API Key");
323 builder.withDelay(10000);
324 }
325
326 final int resultsPerPage = Math.min(settings.getInt(Settings.KEYS.NVD_API_RESULTS_PER_PAGE, RESULTS_PER_PAGE), RESULTS_PER_PAGE);
327
328 builder.withResultsPerPage(resultsPerPage);
329
330
331
332
333
334
335 final int retryCount = settings.getInt(Settings.KEYS.NVD_API_MAX_RETRY_COUNT, 10);
336 builder.withMaxRetryCount(retryCount);
337 long delay = 0;
338 try {
339 delay = settings.getLong(Settings.KEYS.NVD_API_DELAY);
340 } catch (InvalidSettingException ex) {
341 LOGGER.warn("Invalid setting `NVD_API_DELAY`? ({}), using default delay", settings.getString(Settings.KEYS.NVD_API_DELAY));
342 }
343 if (delay > 0) {
344 builder.withDelay(delay);
345 }
346
347 ExecutorService processingExecutorService = null;
348 try {
349 processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE);
350 final List<Future<NvdApiProcessor>> submitted = new ArrayList<>();
351 int max = -1;
352 int ctr = 0;
353 try (NvdCveClient api = builder.build()) {
354 while (api.hasNext()) {
355 final Collection<DefCveItem> items = api.next();
356 max = api.getTotalAvailable();
357 if (ctr == 0) {
358 LOGGER.info(String.format("NVD API has %,d records in this update", max));
359 }
360 if (items != null && !items.isEmpty()) {
361 final ObjectMapper objectMapper = new ObjectMapper();
362 objectMapper.registerModule(new JavaTimeModule());
363 final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
364 try (FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream out = new GZIPOutputStream(fos);) {
365 objectMapper.writeValue(out, items);
366 final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
367 submitted.add(f);
368 }
369 ctr += 1;
370 if ((ctr % 5) == 0) {
371
372 final double percent = (double) (ctr * resultsPerPage) / max * 100;
373 if (percent < 100) {
374 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", ctr * resultsPerPage, max, percent));
375 }
376 }
377 }
378 final ZonedDateTime last = api.getLastUpdated();
379 if (last != null && (lastModifiedRequest == null || lastModifiedRequest.compareTo(last) < 0)) {
380 lastModifiedRequest = last;
381 }
382 }
383
384 } catch (Exception e) {
385 if (e instanceof NvdApiException && (e.getMessage().equals("NVD Returned Status Code: 404")
386 || e.getMessage().equals("NVD Returned Status Code: 403"))) {
387 final String msg;
388 if (key != null) {
389 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nPlease ensure your API Key is valid; "
390 + "see https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#api-key-is-used-and-a-403-or-404-error-occurs\n\n"
391 + "If your NVD API Key is valid try increasing the NVD API Delay.\n\n"
392 + "If this is occurring in a CI environment";
393 } else {
394 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nConsider using an NVD API Key; "
395 + "see https://github.com/jeremylong/DependencyCheck?tab=readme-ov-file#nvd-api-key-highly-recommended";
396 }
397 throw new UpdateException(msg);
398 } else {
399 throw new UpdateException("Error updating the NVD Data", e);
400 }
401 }
402 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", max, max, 100f));
403 max = submitted.size();
404 final boolean updated = max > 0;
405 ctr = 0;
406 for (Future<NvdApiProcessor> f : submitted) {
407 try {
408 final NvdApiProcessor proc = f.get();
409 ctr += 1;
410 final double percent = (double) ctr / max * 100;
411 LOGGER.info(String.format("Completed processing batch %d/%d (%.0f%%) in %,dms", ctr, max, percent, proc.getDurationMillis()));
412 } catch (InterruptedException ex) {
413 Thread.currentThread().interrupt();
414 throw new RuntimeException(ex);
415 } catch (ExecutionException ex) {
416 LOGGER.error("Exception processing NVD API Results", ex);
417 throw new RuntimeException(ex);
418 }
419 }
420 if (lastModifiedRequest != null) {
421 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, ZonedDateTime.now());
422 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
423 }
424 return updated;
425 } finally {
426 if (processingExecutorService != null) {
427 processingExecutorService.shutdownNow();
428 }
429 }
430 }
431
432
433
434
435
436
437
438 private boolean isUpdateConfiguredFalse() {
439 if (!settings.getBoolean(Settings.KEYS.UPDATE_NVDCVE_ENABLED, true)) {
440 return true;
441 }
442 boolean autoUpdate = true;
443 try {
444 autoUpdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
445 } catch (InvalidSettingException ex) {
446 LOGGER.debug("Invalid setting for auto-update; using true.");
447 }
448 return !autoUpdate;
449 }
450
451 @Override
452 public boolean purge(Engine engine) {
453 boolean result = true;
454 try {
455 final File dataDir = engine.getSettings().getDataDirectory();
456 final File db = new File(dataDir, engine.getSettings().getString(Settings.KEYS.DB_FILE_NAME, "odc.mv.db"));
457 if (db.exists()) {
458 if (db.delete()) {
459 LOGGER.info("Database file purged; local copy of the NVD has been removed");
460 } else {
461 LOGGER.error("Unable to delete '{}'; please delete the file manually", db.getAbsolutePath());
462 result = false;
463 }
464 } else {
465 LOGGER.info("Unable to purge database; the database file does not exist: {}", db.getAbsolutePath());
466 result = false;
467 }
468 final File traceFile = new File(dataDir, "odc.trace.db");
469 if (traceFile.exists() && !traceFile.delete()) {
470 LOGGER.error("Unable to delete '{}'; please delete the file manually", traceFile.getAbsolutePath());
471 result = false;
472 }
473 final File lockFile = new File(dataDir, "odc.update.lock");
474 if (lockFile.exists() && !lockFile.delete()) {
475 LOGGER.error("Unable to delete '{}'; please delete the file manually", lockFile.getAbsolutePath());
476 result = false;
477 }
478 } catch (IOException ex) {
479 final String msg = "Unable to delete the database";
480 LOGGER.error(msg, ex);
481 result = false;
482 }
483 return result;
484 }
485
486
487
488
489
490
491
492
493
494 private boolean checkUpdate() throws UpdateException {
495 boolean proceed = true;
496
497 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
498 if (dataExists() && 0 < validForHours) {
499
500 final long validForSeconds = validForHours * 60L * 60L;
501 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_CHECKED);
502 if (lastChecked != null) {
503 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
504 final Duration duration = Duration.between(lastChecked, now);
505 final long difference = duration.getSeconds();
506 proceed = difference > validForSeconds;
507 if (!proceed) {
508 LOGGER.info("Skipping NVD API Cache check since last check was within {} hours.", validForHours);
509 LOGGER.debug("Last NVD API was at {}, and now {} is within {} s.", lastChecked, now, validForSeconds);
510 }
511 } else {
512 LOGGER.warn("NVD cache last checked not present; updating the entire database. This could occur if you are "
513 + "switching back and forth from using the API vs a datafeed or if you are using a database created prior to ODC 9.x");
514 }
515 }
516 return proceed;
517 }
518
519
520
521
522
523
524 private boolean dataExists() {
525 return cveDb.dataExists();
526 }
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543 protected final Map<String, String> getUpdatesNeeded(String url, String filePattern,
544 Properties cacheProperties, ZonedDateTime now) throws UpdateException {
545 LOGGER.debug("starting getUpdatesNeeded() ...");
546 final Map<String, String> updates = new HashMap<>();
547 if (dbProperties != null && !dbProperties.isEmpty()) {
548 final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
549
550
551
552 final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
553 boolean needsFullUpdate = false;
554 for (int y = startYear; y <= endYear; y++) {
555 final ZonedDateTime val = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + y);
556 if (val == null) {
557 needsFullUpdate = true;
558 break;
559 }
560 }
561 final ZonedDateTime lastUpdated = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED);
562 final int days = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_VALID_FOR_DAYS, 7);
563
564 if (!needsFullUpdate && lastUpdated.equals(DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE))) {
565 return updates;
566 } else {
567 updates.put("modified", url + MessageFormat.format(filePattern, "modified"));
568 if (needsFullUpdate) {
569 for (int i = startYear; i <= endYear; i++) {
570 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
571 updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i)));
572 }
573 }
574 } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) {
575 for (int i = startYear; i <= endYear; i++) {
576 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
577 final ZonedDateTime lastModifiedCache = DatabaseProperties.getTimestamp(cacheProperties,
578 NVD_API_CACHE_MODIFIED_DATE + "." + i);
579 final ZonedDateTime lastModifiedDB = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + i);
580 if (lastModifiedDB == null || lastModifiedCache.compareTo(lastModifiedDB) > 0) {
581 updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i)));
582 }
583 }
584 }
585 }
586 }
587 }
588 if (updates.size() > 3) {
589 LOGGER.info("NVD API Cache requires several updates; this could take a couple of minutes.");
590 }
591 return updates;
592 }
593
594
595
596
597
598
599
600
601
602
603 protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException {
604 final Properties properties = new Properties();
605 try {
606 final URL u = new URI(url + "cache.properties").toURL();
607 final String content = Downloader.getInstance().fetchContent(u, StandardCharsets.UTF_8);
608 properties.load(new StringReader(content));
609
610 } catch (URISyntaxException ex) {
611 throw new UpdateException("Invalid NVD Cache URL", ex);
612 } catch (DownloadFailedException | ResourceNotFoundException ex) {
613 final String metaPattern;
614 if (pattern == null) {
615 metaPattern = "nvdcve-{0}.meta";
616 } else {
617 metaPattern = pattern.replace(".json.gz", ".meta");
618 }
619 try {
620 URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL();
621 String content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8);
622 final Properties props = new Properties();
623 props.load(new StringReader(content));
624 ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
625 DatabaseProperties.setTimestamp(properties, "lastModifiedDate.modified", lmd);
626 DatabaseProperties.setTimestamp(properties, "lastModifiedDate", lmd);
627 final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
628 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
629 final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
630 for (int y = startYear; y <= endYear; y++) {
631 metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL();
632 content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8);
633 props.clear();
634 props.load(new StringReader(content));
635 lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
636 DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + String.valueOf(y), lmd);
637 }
638 } catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex1) {
639 throw new UpdateException("Unable to download the data feed META files", ex);
640 }
641 } catch (TooManyRequestsException ex) {
642 throw new UpdateException("Unable to download the NVD API cache.properties", ex);
643 } catch (IOException ex) {
644 throw new UpdateException("Invalid NVD Cache Properties file contents", ex);
645 }
646 return properties;
647 }
648
649 protected static class UrlData {
650
651
652
653
654 private final String url;
655
656
657
658
659 private final String pattern;
660
661 public UrlData(String url, String pattern) {
662 this.url = url;
663 this.pattern = pattern;
664 }
665
666
667
668
669
670
671 public String getPattern() {
672 return pattern;
673 }
674
675
676
677
678
679
680 public String getUrl() {
681 return url;
682 }
683
684 }
685 }