1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.update;
19
20 import com.fasterxml.jackson.databind.ObjectMapper;
21 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
22 import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
23 import io.github.jeremylong.openvulnerability.client.nvd.NvdApiException;
24 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient;
25 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder;
26 import java.io.File;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.StringReader;
30 import java.net.URI;
31 import java.net.URISyntaxException;
32 import java.net.URL;
33 import java.nio.charset.StandardCharsets;
34 import java.text.MessageFormat;
35 import java.time.Duration;
36 import java.time.ZoneId;
37 import java.time.ZonedDateTime;
38 import java.util.ArrayList;
39 import java.util.Collection;
40 import java.util.HashMap;
41 import java.util.HashSet;
42 import java.util.List;
43 import java.util.Map;
44 import java.util.Properties;
45 import java.util.Set;
46 import java.util.concurrent.ExecutionException;
47 import java.util.concurrent.ExecutorService;
48 import java.util.concurrent.Executors;
49 import java.util.concurrent.Future;
50 import java.util.zip.GZIPOutputStream;
51 import org.owasp.dependencycheck.Engine;
52 import org.owasp.dependencycheck.data.nvdcve.CveDB;
53 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
54 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
55 import org.owasp.dependencycheck.data.update.exception.UpdateException;
56 import org.owasp.dependencycheck.data.update.nvd.api.DownloadTask;
57 import org.owasp.dependencycheck.data.update.nvd.api.NvdApiProcessor;
58 import org.owasp.dependencycheck.utils.DateUtil;
59 import org.owasp.dependencycheck.utils.DownloadFailedException;
60 import org.owasp.dependencycheck.utils.Downloader;
61 import org.owasp.dependencycheck.utils.InvalidSettingException;
62 import org.owasp.dependencycheck.utils.ResourceNotFoundException;
63 import org.owasp.dependencycheck.utils.Settings;
64 import org.owasp.dependencycheck.utils.TooManyRequestsException;
65 import org.slf4j.Logger;
66 import org.slf4j.LoggerFactory;
67
68
69
70
71
72 public class NvdApiDataSource implements CachedWebDataSource {
73
74
75
76
77 private static final Logger LOGGER = LoggerFactory.getLogger(NvdApiDataSource.class);
78
79
80
81 private static final int PROCESSING_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors();
82
83
84
85 private Settings settings;
86
87
88
89 private CveDB cveDb = null;
90
91
92
93 private DatabaseProperties dbProperties = null;
94
95
96
97 private static final String NVD_API_CACHE_MODIFIED_DATE = "lastModifiedDate";
98
99
100
101
102 private static final int RESULTS_PER_PAGE = 2000;
103
104 @Override
105 public boolean update(Engine engine) throws UpdateException {
106 this.settings = engine.getSettings();
107 this.cveDb = engine.getDatabase();
108 if (isUpdateConfiguredFalse()) {
109 return false;
110 }
111 dbProperties = cveDb.getDatabaseProperties();
112
113 final String nvdDataFeedUrl = settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL);
114 if (nvdDataFeedUrl != null) {
115 return processDatafeed(nvdDataFeedUrl);
116 }
117 return processApi();
118 }
119
120 protected UrlData extractUrlData(String nvdDataFeedUrl) {
121 String url;
122 String pattern = null;
123 if (nvdDataFeedUrl.endsWith(".json.gz")) {
124 final int lio = nvdDataFeedUrl.lastIndexOf("/");
125 pattern = nvdDataFeedUrl.substring(lio + 1);
126 url = nvdDataFeedUrl.substring(0, lio);
127 } else {
128 url = nvdDataFeedUrl;
129 }
130 if (!url.endsWith("/")) {
131 url += "/";
132 }
133 return new UrlData(url, pattern);
134 }
135
136 private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException {
137 boolean updatesMade = false;
138 try {
139 dbProperties = cveDb.getDatabaseProperties();
140 if (checkUpdate()) {
141 final UrlData data = extractUrlData(nvdDataFeedUrl);
142 final String url = data.getUrl();
143 String pattern = data.getPattern();
144 final Properties cacheProperties = getRemoteCacheProperties(url, pattern);
145 if (pattern == null) {
146 final String prefix = cacheProperties.getProperty("prefix", "nvdcve-");
147 pattern = prefix + "{0}.json.gz";
148 }
149
150 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
151 final Map<String, String> updateable = getUpdatesNeeded(url, pattern, cacheProperties, now);
152 if (!updateable.isEmpty()) {
153 final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1);
154 final int downloadPoolSize = Math.min(Runtime.getRuntime().availableProcessors(), max);
155
156 final int maxExec = PROCESSING_THREAD_POOL_SIZE;
157 final int execPoolSize = Math.min(maxExec, 2);
158
159 ExecutorService processingExecutorService = null;
160 ExecutorService downloadExecutorService = null;
161 try {
162 downloadExecutorService = Executors.newFixedThreadPool(downloadPoolSize);
163 processingExecutorService = Executors.newFixedThreadPool(execPoolSize);
164
165 DownloadTask runLast = null;
166 final Set<Future<Future<NvdApiProcessor>>> downloadFutures = new HashSet<>(updateable.size());
167 runLast = startDownloads(updateable, processingExecutorService, runLast, downloadFutures, downloadExecutorService);
168
169
170 final Set<Future<NvdApiProcessor>> processFutures = new HashSet<>(updateable.size());
171 for (Future<Future<NvdApiProcessor>> future : downloadFutures) {
172 processDownload(future, processFutures);
173 }
174
175 processFuture(processFutures);
176 processFutures.clear();
177
178
179 if (runLast != null) {
180 final Future<Future<NvdApiProcessor>> modified = downloadExecutorService.submit(runLast);
181 processDownload(modified, processFutures);
182 processFuture(processFutures);
183 }
184
185 } finally {
186 if (processingExecutorService != null) {
187 processingExecutorService.shutdownNow();
188 }
189 if (downloadExecutorService != null) {
190 downloadExecutorService.shutdownNow();
191 }
192 }
193 updatesMade = true;
194 }
195 storeLastModifiedDates(now, cacheProperties, updateable);
196 if (updatesMade) {
197 cveDb.persistEcosystemCache();
198 }
199 final int updateCount = cveDb.updateEcosystemCache();
200 LOGGER.debug("Corrected the ecosystem for {} ecoSystemCache entries", updateCount);
201 if (updatesMade || updateCount > 0) {
202 cveDb.cleanupDatabase();
203 }
204 }
205 } catch (UpdateException ex) {
206 if (ex.getCause() != null && ex.getCause() instanceof DownloadFailedException) {
207 final String jre = System.getProperty("java.version");
208 if (jre == null || jre.startsWith("1.4") || jre.startsWith("1.5") || jre.startsWith("1.6") || jre.startsWith("1.7")) {
209 LOGGER.error("An old JRE is being used ({} {}), and likely does not have the correct root certificates or algorithms "
210 + "to connect to the NVD - consider upgrading your JRE.", System.getProperty("java.vendor"), jre);
211 }
212 }
213 throw ex;
214 } catch (DatabaseException ex) {
215 throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex);
216 }
217 return updatesMade;
218 }
219
220 private void storeLastModifiedDates(final ZonedDateTime now, final Properties cacheProperties,
221 final Map<String, String> updateable) throws UpdateException {
222
223 final ZonedDateTime lastModifiedRequest = DatabaseProperties.getTimestamp(cacheProperties,
224 NVD_API_CACHE_MODIFIED_DATE + ".modified");
225 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_CHECKED, now);
226 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED, lastModifiedRequest);
227
228 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, now);
229 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
230
231 for (String entry : updateable.keySet()) {
232 final ZonedDateTime date = DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE + "." + entry);
233 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + entry, date);
234 }
235 }
236
237 private DownloadTask startDownloads(final Map<String, String> updateable, ExecutorService processingExecutorService, DownloadTask runLast,
238 final Set<Future<Future<NvdApiProcessor>>> downloadFutures, ExecutorService downloadExecutorService) throws UpdateException {
239 DownloadTask lastCall = runLast;
240 for (Map.Entry<String, String> cve : updateable.entrySet()) {
241 final DownloadTask call = new DownloadTask(cve.getValue(), processingExecutorService, cveDb, settings);
242 if (call.isModified()) {
243 lastCall = call;
244 } else {
245 final boolean added = downloadFutures.add(downloadExecutorService.submit(call));
246 if (!added) {
247 throw new UpdateException("Unable to add the download task for " + cve);
248 }
249 }
250 }
251 return lastCall;
252 }
253
254 private void processFuture(final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
255
256 for (Future<NvdApiProcessor> future : processFutures) {
257 try {
258 final NvdApiProcessor task = future.get();
259 } catch (InterruptedException ex) {
260 LOGGER.debug("Thread was interrupted during processing", ex);
261 Thread.currentThread().interrupt();
262 throw new UpdateException(ex);
263 } catch (ExecutionException ex) {
264 LOGGER.debug("Execution Exception during process", ex);
265 throw new UpdateException(ex);
266 }
267 }
268 }
269
270 private void processDownload(Future<Future<NvdApiProcessor>> future, final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
271 final Future<NvdApiProcessor> task;
272 try {
273 task = future.get();
274 if (task != null) {
275 processFutures.add(task);
276 }
277 } catch (InterruptedException ex) {
278 LOGGER.debug("Thread was interrupted during download", ex);
279 Thread.currentThread().interrupt();
280 throw new UpdateException("The download was interrupted", ex);
281 } catch (ExecutionException ex) {
282 LOGGER.debug("Thread was interrupted during download execution", ex);
283 throw new UpdateException("The execution of the download was interrupted", ex);
284 }
285 }
286
287 private boolean processApi() throws UpdateException {
288 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_CHECKED);
289 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
290 if (cveDb.dataExists() && lastChecked != null && validForHours > 0) {
291
292 final long validForSeconds = validForHours * 60L * 60L;
293 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
294 final Duration duration = Duration.between(lastChecked, now);
295 final long difference = duration.getSeconds();
296 if (difference < validForSeconds) {
297 LOGGER.info("Skipping the NVD API Update as it was completed within the last {} minutes", validForSeconds / 60);
298 return false;
299 }
300 }
301
302 ZonedDateTime lastModifiedRequest = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_MODIFIED);
303 final NvdCveClientBuilder builder = NvdCveClientBuilder.aNvdCveApi();
304 final String endpoint = settings.getString(Settings.KEYS.NVD_API_ENDPOINT);
305 if (endpoint != null) {
306 builder.withEndpoint(endpoint);
307 }
308 if (lastModifiedRequest != null) {
309 final ZonedDateTime end = lastModifiedRequest.minusDays(-120);
310 builder.withLastModifiedFilter(lastModifiedRequest, end);
311 }
312 final String key = settings.getString(Settings.KEYS.NVD_API_KEY);
313 if (key != null) {
314
315 builder.withApiKey(key)
316 .withDelay(5000)
317 .withThreadCount(4);
318 } else {
319 LOGGER.warn("An NVD API Key was not provided - it is highly recommended to use "
320 + "an NVD API key as the update can take a VERY long time without an API Key");
321 builder.withDelay(10000);
322 }
323
324 final int resultsPerPage = Math.min(settings.getInt(Settings.KEYS.NVD_API_RESULTS_PER_PAGE, RESULTS_PER_PAGE), RESULTS_PER_PAGE);
325
326 builder.withResultsPerPage(resultsPerPage);
327
328
329
330
331
332
333 final int retryCount = settings.getInt(Settings.KEYS.NVD_API_MAX_RETRY_COUNT, 10);
334 builder.withMaxRetryCount(retryCount);
335 long delay = 0;
336 try {
337 delay = settings.getLong(Settings.KEYS.NVD_API_DELAY);
338 } catch (InvalidSettingException ex) {
339 LOGGER.warn("Invalid setting `NVD_API_DELAY`? ({}), using default delay", settings.getString(Settings.KEYS.NVD_API_DELAY));
340 }
341 if (delay > 0) {
342 builder.withDelay(delay);
343 }
344
345 ExecutorService processingExecutorService = null;
346 try {
347 processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE);
348 final List<Future<NvdApiProcessor>> submitted = new ArrayList<>();
349 int max = -1;
350 int ctr = 0;
351 try (NvdCveClient api = builder.build()) {
352 while (api.hasNext()) {
353 final Collection<DefCveItem> items = api.next();
354 max = api.getTotalAvailable();
355 if (ctr == 0) {
356 LOGGER.info(String.format("NVD API has %,d records in this update", max));
357 }
358 if (items != null && !items.isEmpty()) {
359 final ObjectMapper objectMapper = new ObjectMapper();
360 objectMapper.registerModule(new JavaTimeModule());
361 final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
362 try (FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream out = new GZIPOutputStream(fos);) {
363 objectMapper.writeValue(out, items);
364 final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
365 submitted.add(f);
366 }
367 ctr += 1;
368 if ((ctr % 5) == 0) {
369
370 final double percent = (double) (ctr * resultsPerPage) / max * 100;
371 if (percent < 100) {
372 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", ctr * resultsPerPage, max, percent));
373 }
374 }
375 }
376 final ZonedDateTime last = api.getLastUpdated();
377 if (last != null && (lastModifiedRequest == null || lastModifiedRequest.compareTo(last) < 0)) {
378 lastModifiedRequest = last;
379 }
380 }
381
382 } catch (Exception e) {
383 if (e instanceof NvdApiException && (e.getMessage().equals("NVD Returned Status Code: 404")
384 || e.getMessage().equals("NVD Returned Status Code: 403"))) {
385 final String msg;
386 if (key != null) {
387 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nPlease ensure your API Key is valid; "
388 + "see https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#api-key-is-used-and-a-403-or-404-error-occurs\n\n"
389 + "If your NVD API Key is valid try increasing the NVD API Delay.\n\n"
390 + "If this is occurring in a CI environment";
391 } else {
392 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nConsider using an NVD API Key; "
393 + "see https://github.com/jeremylong/DependencyCheck?tab=readme-ov-file#nvd-api-key-highly-recommended";
394 }
395 throw new UpdateException(msg);
396 } else {
397 throw new UpdateException("Error updating the NVD Data", e);
398 }
399 }
400 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", max, max, 100f));
401 max = submitted.size();
402 final boolean updated = max > 0;
403 ctr = 0;
404 for (Future<NvdApiProcessor> f : submitted) {
405 try {
406 final NvdApiProcessor proc = f.get();
407 ctr += 1;
408 final double percent = (double) ctr / max * 100;
409 LOGGER.info(String.format("Completed processing batch %d/%d (%.0f%%) in %,dms", ctr, max, percent, proc.getDurationMillis()));
410 } catch (InterruptedException ex) {
411 Thread.currentThread().interrupt();
412 throw new RuntimeException(ex);
413 } catch (ExecutionException ex) {
414 LOGGER.error("Exception processing NVD API Results", ex);
415 throw new RuntimeException(ex);
416 }
417 }
418 if (lastModifiedRequest != null) {
419 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, ZonedDateTime.now());
420 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
421 }
422 return updated;
423 } finally {
424 if (processingExecutorService != null) {
425 processingExecutorService.shutdownNow();
426 }
427 }
428 }
429
430
431
432
433
434
435
436 private boolean isUpdateConfiguredFalse() {
437 if (!settings.getBoolean(Settings.KEYS.UPDATE_NVDCVE_ENABLED, true)) {
438 return true;
439 }
440 boolean autoUpdate = true;
441 try {
442 autoUpdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
443 } catch (InvalidSettingException ex) {
444 LOGGER.debug("Invalid setting for auto-update; using true.");
445 }
446 return !autoUpdate;
447 }
448
449 @Override
450 public boolean purge(Engine engine) {
451 boolean result = true;
452 try {
453 final File dataDir = engine.getSettings().getDataDirectory();
454 final File db = new File(dataDir, engine.getSettings().getString(Settings.KEYS.DB_FILE_NAME, "odc.mv.db"));
455 if (db.exists()) {
456 if (db.delete()) {
457 LOGGER.info("Database file purged; local copy of the NVD has been removed");
458 } else {
459 LOGGER.error("Unable to delete '{}'; please delete the file manually", db.getAbsolutePath());
460 result = false;
461 }
462 } else {
463 LOGGER.info("Unable to purge database; the database file does not exist: {}", db.getAbsolutePath());
464 result = false;
465 }
466 final File traceFile = new File(dataDir, "odc.trace.db");
467 if (traceFile.exists() && !traceFile.delete()) {
468 LOGGER.error("Unable to delete '{}'; please delete the file manually", traceFile.getAbsolutePath());
469 result = false;
470 }
471 final File lockFile = new File(dataDir, "odc.update.lock");
472 if (lockFile.exists() && !lockFile.delete()) {
473 LOGGER.error("Unable to delete '{}'; please delete the file manually", lockFile.getAbsolutePath());
474 result = false;
475 }
476 } catch (IOException ex) {
477 final String msg = "Unable to delete the database";
478 LOGGER.error(msg, ex);
479 result = false;
480 }
481 return result;
482 }
483
484
485
486
487
488
489
490
491
492 private boolean checkUpdate() throws UpdateException {
493 boolean proceed = true;
494
495 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
496 if (dataExists() && 0 < validForHours) {
497
498 final long validForSeconds = validForHours * 60L * 60L;
499 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_CHECKED);
500 if (lastChecked != null) {
501 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
502 final Duration duration = Duration.between(lastChecked, now);
503 final long difference = duration.getSeconds();
504 proceed = difference > validForSeconds;
505 if (!proceed) {
506 LOGGER.info("Skipping NVD API Cache check since last check was within {} hours.", validForHours);
507 LOGGER.debug("Last NVD API was at {}, and now {} is within {} s.", lastChecked, now, validForSeconds);
508 }
509 } else {
510 LOGGER.warn("NVD cache last checked not present; updating the entire database. This could occur if you are "
511 + "switching back and forth from using the API vs a datafeed or if you are using a database created prior to ODC 9.x");
512 }
513 }
514 return proceed;
515 }
516
517
518
519
520
521
522 private boolean dataExists() {
523 return cveDb.dataExists();
524 }
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541 protected final Map<String, String> getUpdatesNeeded(String url, String filePattern,
542 Properties cacheProperties, ZonedDateTime now) throws UpdateException {
543 LOGGER.debug("starting getUpdatesNeeded() ...");
544 final Map<String, String> updates = new HashMap<>();
545 if (dbProperties != null && !dbProperties.isEmpty()) {
546 final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
547
548
549
550 final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
551 boolean needsFullUpdate = false;
552 for (int y = startYear; y <= endYear; y++) {
553 final ZonedDateTime val = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + y);
554 if (val == null) {
555 needsFullUpdate = true;
556 break;
557 }
558 }
559 final ZonedDateTime lastUpdated = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED);
560 final int days = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_VALID_FOR_DAYS, 7);
561
562 if (!needsFullUpdate && lastUpdated.equals(DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE))) {
563 return updates;
564 } else {
565 updates.put("modified", url + MessageFormat.format(filePattern, "modified"));
566 if (needsFullUpdate) {
567 for (int i = startYear; i <= endYear; i++) {
568 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
569 updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i)));
570 }
571 }
572 } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) {
573 for (int i = startYear; i <= endYear; i++) {
574 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
575 final ZonedDateTime lastModifiedCache = DatabaseProperties.getTimestamp(cacheProperties,
576 NVD_API_CACHE_MODIFIED_DATE + "." + i);
577 final ZonedDateTime lastModifiedDB = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + i);
578 if (lastModifiedDB == null || lastModifiedCache.compareTo(lastModifiedDB) > 0) {
579 updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i)));
580 }
581 }
582 }
583 }
584 }
585 }
586 if (updates.size() > 3) {
587 LOGGER.info("NVD API Cache requires several updates; this could take a couple of minutes.");
588 }
589 return updates;
590 }
591
592
593
594
595
596
597
598
599
600
601 protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException {
602 final Properties properties = new Properties();
603 try {
604 final URL u = new URI(url + "cache.properties").toURL();
605 final String content = Downloader.getInstance().fetchContent(u, StandardCharsets.UTF_8);
606 properties.load(new StringReader(content));
607
608 } catch (URISyntaxException ex) {
609 throw new UpdateException("Invalid NVD Cache URL", ex);
610 } catch (DownloadFailedException | ResourceNotFoundException ex) {
611 final String metaPattern;
612 if (pattern == null) {
613 metaPattern = "nvdcve-{0}.meta";
614 } else {
615 metaPattern = pattern.replace(".json.gz", ".meta");
616 }
617 try {
618 URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL();
619 String content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8);
620 final Properties props = new Properties();
621 props.load(new StringReader(content));
622 ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
623 DatabaseProperties.setTimestamp(properties, "lastModifiedDate.modified", lmd);
624 DatabaseProperties.setTimestamp(properties, "lastModifiedDate", lmd);
625 final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
626 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
627 final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
628 for (int y = startYear; y <= endYear; y++) {
629 metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL();
630 content = Downloader.getInstance().fetchContent(metaUrl, StandardCharsets.UTF_8);
631 props.clear();
632 props.load(new StringReader(content));
633 lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
634 DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + String.valueOf(y), lmd);
635 }
636 } catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex1) {
637 throw new UpdateException("Unable to download the data feed META files", ex);
638 }
639 } catch (TooManyRequestsException ex) {
640 throw new UpdateException("Unable to download the NVD API cache.properties", ex);
641 } catch (IOException ex) {
642 throw new UpdateException("Invalid NVD Cache Properties file contents", ex);
643 }
644 return properties;
645 }
646
647 protected static class UrlData {
648
649
650
651
652 private final String url;
653
654
655
656
657 private final String pattern;
658
659 public UrlData(String url, String pattern) {
660 this.url = url;
661 this.pattern = pattern;
662 }
663
664
665
666
667
668
669 public String getPattern() {
670 return pattern;
671 }
672
673
674
675
676
677
678 public String getUrl() {
679 return url;
680 }
681
682 }
683 }