1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.update;
19
20 import com.fasterxml.jackson.databind.ObjectMapper;
21 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
22 import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
23 import io.github.jeremylong.openvulnerability.client.nvd.NvdApiException;
24 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient;
25 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder;
26 import java.io.File;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.StringReader;
30 import java.net.URI;
31 import java.net.URISyntaxException;
32 import java.net.URL;
33 import java.text.MessageFormat;
34 import java.time.Duration;
35 import java.time.ZoneId;
36 import java.time.ZonedDateTime;
37 import java.util.ArrayList;
38 import java.util.Collection;
39 import java.util.HashMap;
40 import java.util.HashSet;
41 import java.util.List;
42 import java.util.Map;
43 import java.util.Properties;
44 import java.util.Set;
45 import java.util.concurrent.ExecutionException;
46 import java.util.concurrent.ExecutorService;
47 import java.util.concurrent.Executors;
48 import java.util.concurrent.Future;
49 import java.util.zip.GZIPOutputStream;
50 import org.owasp.dependencycheck.Engine;
51 import org.owasp.dependencycheck.data.nvdcve.CveDB;
52 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
53 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
54 import org.owasp.dependencycheck.data.update.exception.UpdateException;
55 import org.owasp.dependencycheck.data.update.nvd.api.DownloadTask;
56 import org.owasp.dependencycheck.data.update.nvd.api.NvdApiProcessor;
57 import org.owasp.dependencycheck.utils.DateUtil;
58 import org.owasp.dependencycheck.utils.DownloadFailedException;
59 import org.owasp.dependencycheck.utils.Downloader;
60 import org.owasp.dependencycheck.utils.InvalidSettingException;
61 import org.owasp.dependencycheck.utils.ResourceNotFoundException;
62 import org.owasp.dependencycheck.utils.Settings;
63 import org.owasp.dependencycheck.utils.TooManyRequestsException;
64 import org.slf4j.Logger;
65 import org.slf4j.LoggerFactory;
66
67
68
69
70
71 public class NvdApiDataSource implements CachedWebDataSource {
72
73
74
75
76 private static final Logger LOGGER = LoggerFactory.getLogger(NvdApiDataSource.class);
77
78
79
80 private static final int PROCESSING_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors();
81
82
83
84 private Settings settings;
85
86
87
88 private CveDB cveDb = null;
89
90
91
92 private DatabaseProperties dbProperties = null;
93
94
95
96 private static final String NVD_API_CACHE_MODIFIED_DATE = "lastModifiedDate";
97
98
99
100
101 private static final int RESULTS_PER_PAGE = 2000;
102
103 @Override
104 public boolean update(Engine engine) throws UpdateException {
105 this.settings = engine.getSettings();
106 this.cveDb = engine.getDatabase();
107 if (isUpdateConfiguredFalse()) {
108 return false;
109 }
110 dbProperties = cveDb.getDatabaseProperties();
111
112 final String nvdDataFeedUrl = settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL);
113 if (nvdDataFeedUrl != null) {
114 return processDatafeed(nvdDataFeedUrl);
115 }
116 return processApi();
117 }
118
119 protected UrlData extractUrlData(String nvdDataFeedUrl) {
120 String url;
121 String pattern = null;
122 if (nvdDataFeedUrl.endsWith(".json.gz")) {
123 final int lio = nvdDataFeedUrl.lastIndexOf("/");
124 pattern = nvdDataFeedUrl.substring(lio + 1);
125 url = nvdDataFeedUrl.substring(0, lio);
126 } else {
127 url = nvdDataFeedUrl;
128 }
129 if (!url.endsWith("/")) {
130 url += "/";
131 }
132 return new UrlData(url, pattern);
133 }
134
135 private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException {
136 boolean updatesMade = false;
137 try {
138 dbProperties = cveDb.getDatabaseProperties();
139 if (checkUpdate()) {
140 final UrlData data = extractUrlData(nvdDataFeedUrl);
141 String url = data.getUrl();
142 String pattern = data.getPattern();
143 final Properties cacheProperties = getRemoteCacheProperties(url, pattern);
144 if (pattern == null) {
145 final String prefix = cacheProperties.getProperty("prefix", "nvdcve-");
146 pattern = prefix + "{0}.json.gz";
147 }
148
149 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
150 final Map<String, String> updateable = getUpdatesNeeded(url, pattern, cacheProperties, now);
151 if (!updateable.isEmpty()) {
152 final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1);
153 final int downloadPoolSize = Math.min(Runtime.getRuntime().availableProcessors(), max);
154
155 final int maxExec = PROCESSING_THREAD_POOL_SIZE;
156 final int execPoolSize = Math.min(maxExec, 2);
157
158 ExecutorService processingExecutorService = null;
159 ExecutorService downloadExecutorService = null;
160 try {
161 downloadExecutorService = Executors.newFixedThreadPool(downloadPoolSize);
162 processingExecutorService = Executors.newFixedThreadPool(execPoolSize);
163
164 DownloadTask runLast = null;
165 final Set<Future<Future<NvdApiProcessor>>> downloadFutures = new HashSet<>(updateable.size());
166 runLast = startDownloads(updateable, processingExecutorService, runLast, downloadFutures, downloadExecutorService);
167
168
169 final Set<Future<NvdApiProcessor>> processFutures = new HashSet<>(updateable.size());
170 for (Future<Future<NvdApiProcessor>> future : downloadFutures) {
171 processDownload(future, processFutures);
172 }
173
174 processFuture(processFutures);
175 processFutures.clear();
176
177
178 if (runLast != null) {
179 final Future<Future<NvdApiProcessor>> modified = downloadExecutorService.submit(runLast);
180 processDownload(modified, processFutures);
181 processFuture(processFutures);
182 }
183
184 } finally {
185 if (processingExecutorService != null) {
186 processingExecutorService.shutdownNow();
187 }
188 if (downloadExecutorService != null) {
189 downloadExecutorService.shutdownNow();
190 }
191 }
192 updatesMade = true;
193 }
194 storeLastModifiedDates(now, cacheProperties, updateable);
195 if (updatesMade) {
196 cveDb.persistEcosystemCache();
197 }
198 final int updateCount = cveDb.updateEcosystemCache();
199 LOGGER.debug("Corrected the ecosystem for {} ecoSystemCache entries", updateCount);
200 if (updatesMade || updateCount > 0) {
201 cveDb.cleanupDatabase();
202 }
203 }
204 } catch (UpdateException ex) {
205 if (ex.getCause() != null && ex.getCause() instanceof DownloadFailedException) {
206 final String jre = System.getProperty("java.version");
207 if (jre == null || jre.startsWith("1.4") || jre.startsWith("1.5") || jre.startsWith("1.6") || jre.startsWith("1.7")) {
208 LOGGER.error("An old JRE is being used ({} {}), and likely does not have the correct root certificates or algorithms "
209 + "to connect to the NVD - consider upgrading your JRE.", System.getProperty("java.vendor"), jre);
210 }
211 }
212 throw ex;
213 } catch (DatabaseException ex) {
214 throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex);
215 }
216 return updatesMade;
217 }
218
219 private void storeLastModifiedDates(final ZonedDateTime now, final Properties cacheProperties,
220 final Map<String, String> updateable) throws UpdateException {
221
222 ZonedDateTime lastModifiedRequest = DatabaseProperties.getTimestamp(cacheProperties,
223 NVD_API_CACHE_MODIFIED_DATE + ".modified");
224 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_CHECKED, now);
225 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED, lastModifiedRequest);
226
227 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, now);
228 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
229
230 for (String entry : updateable.keySet()) {
231 final ZonedDateTime date = DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE + "." + entry);
232 dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + entry, date);
233 }
234 }
235
236 private DownloadTask startDownloads(final Map<String, String> updateable, ExecutorService processingExecutorService, DownloadTask runLast,
237 final Set<Future<Future<NvdApiProcessor>>> downloadFutures, ExecutorService downloadExecutorService) throws UpdateException {
238 DownloadTask lastCall = runLast;
239 for (Map.Entry<String, String> cve : updateable.entrySet()) {
240 final DownloadTask call = new DownloadTask(cve.getValue(), processingExecutorService, cveDb, settings);
241 if (call.isModified()) {
242 lastCall = call;
243 } else {
244 final boolean added = downloadFutures.add(downloadExecutorService.submit(call));
245 if (!added) {
246 throw new UpdateException("Unable to add the download task for " + cve);
247 }
248 }
249 }
250 return lastCall;
251 }
252
253 private void processFuture(final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
254
255 for (Future<NvdApiProcessor> future : processFutures) {
256 try {
257 final NvdApiProcessor task = future.get();
258 } catch (InterruptedException ex) {
259 LOGGER.debug("Thread was interrupted during processing", ex);
260 Thread.currentThread().interrupt();
261 throw new UpdateException(ex);
262 } catch (ExecutionException ex) {
263 LOGGER.debug("Execution Exception during process", ex);
264 throw new UpdateException(ex);
265 }
266 }
267 }
268
269 private void processDownload(Future<Future<NvdApiProcessor>> future, final Set<Future<NvdApiProcessor>> processFutures) throws UpdateException {
270 final Future<NvdApiProcessor> task;
271 try {
272 task = future.get();
273 if (task != null) {
274 processFutures.add(task);
275 }
276 } catch (InterruptedException ex) {
277 LOGGER.debug("Thread was interrupted during download", ex);
278 Thread.currentThread().interrupt();
279 throw new UpdateException("The download was interrupted", ex);
280 } catch (ExecutionException ex) {
281 LOGGER.debug("Thread was interrupted during download execution", ex);
282 throw new UpdateException("The execution of the download was interrupted", ex);
283 }
284 }
285
286 private boolean processApi() throws UpdateException {
287 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_CHECKED);
288 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
289 if (cveDb.dataExists() && lastChecked != null && validForHours > 0) {
290
291 final long validForSeconds = validForHours * 60L * 60L;
292 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
293 final Duration duration = Duration.between(lastChecked, now);
294 final long difference = duration.getSeconds();
295 if (difference < validForSeconds) {
296 LOGGER.info("Skipping the NVD API Update as it was completed within the last {} minutes", validForSeconds / 60);
297 return false;
298 }
299 }
300
301 ZonedDateTime lastModifiedRequest = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_MODIFIED);
302 final NvdCveClientBuilder builder = NvdCveClientBuilder.aNvdCveApi();
303 final String endpoint = settings.getString(Settings.KEYS.NVD_API_ENDPOINT);
304 if (endpoint != null) {
305 builder.withEndpoint(endpoint);
306 }
307 if (lastModifiedRequest != null) {
308 final ZonedDateTime end = lastModifiedRequest.minusDays(-120);
309 builder.withLastModifiedFilter(lastModifiedRequest, end);
310 }
311 final String key = settings.getString(Settings.KEYS.NVD_API_KEY);
312 if (key != null) {
313
314 builder.withApiKey(key)
315 .withDelay(5000)
316 .withThreadCount(4);
317 } else {
318 LOGGER.warn("An NVD API Key was not provided - it is highly recommended to use "
319 + "an NVD API key as the update can take a VERY long time without an API Key");
320 builder.withDelay(10000);
321 }
322 builder.withResultsPerPage(RESULTS_PER_PAGE);
323
324
325
326
327
328
329 final int retryCount = settings.getInt(Settings.KEYS.NVD_API_MAX_RETRY_COUNT, 10);
330 builder.withMaxRetryCount(retryCount);
331 long delay = 0;
332 try {
333 delay = settings.getLong(Settings.KEYS.NVD_API_DELAY);
334 } catch (InvalidSettingException ex) {
335 LOGGER.warn("Invalid setting `NVD_API_DELAY`? ({}), using default delay", settings.getString(Settings.KEYS.NVD_API_DELAY));
336 }
337 if (delay > 0) {
338 builder.withDelay(delay);
339 }
340
341 ExecutorService processingExecutorService = null;
342 try {
343 processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE);
344 final List<Future<NvdApiProcessor>> submitted = new ArrayList<>();
345 int max = -1;
346 int ctr = 0;
347 try (NvdCveClient api = builder.build()) {
348 while (api.hasNext()) {
349 Collection<DefCveItem> items = api.next();
350 max = api.getTotalAvailable();
351 if (ctr == 0) {
352 LOGGER.info(String.format("NVD API has %,d records in this update", max));
353 }
354 if (items != null && !items.isEmpty()) {
355 final ObjectMapper objectMapper = new ObjectMapper();
356 objectMapper.registerModule(new JavaTimeModule());
357 final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
358 try (FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream out = new GZIPOutputStream(fos);) {
359 objectMapper.writeValue(out, items);
360 final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
361 submitted.add(f);
362 }
363 ctr += 1;
364 if ((ctr % 5) == 0) {
365 final double percent = (double) (ctr * RESULTS_PER_PAGE) / max * 100;
366 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", ctr * RESULTS_PER_PAGE, max, percent));
367 }
368 }
369 final ZonedDateTime last = api.getLastUpdated();
370 if (last != null && (lastModifiedRequest == null || lastModifiedRequest.compareTo(last) < 0)) {
371 lastModifiedRequest = last;
372 }
373 }
374
375 } catch (Exception e) {
376 if (e instanceof NvdApiException && (e.getMessage().equals("NVD Returned Status Code: 404") || e.getMessage().equals("NVD Returned Status Code: 403"))) {
377 final String msg;
378 if (key != null) {
379 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nPlease ensure your API Key is valid; "
380 + "see https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#api-key-is-used-and-a-403-or-404-error-occurs\n\n"
381 + "If your NVD API Key is valid try increasing the NVD API Delay.\n\n"
382 + "If this is ocurring in a CI environment";
383 } else {
384 msg = "Error updating the NVD Data; the NVD returned a 403 or 404 error\n\nConsider using an NVD API Key; "
385 + "see https://github.com/jeremylong/DependencyCheck?tab=readme-ov-file#nvd-api-key-highly-recommended";
386 }
387 throw new UpdateException(msg);
388 } else {
389 throw new UpdateException("Error updating the NVD Data", e);
390 }
391 }
392 LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", max, max, 100f));
393 max = submitted.size();
394 final boolean updated = max > 0;
395 ctr = 0;
396 for (Future<NvdApiProcessor> f : submitted) {
397 try {
398 final NvdApiProcessor proc = f.get();
399 ctr += 1;
400 final double percent = (double) ctr / max * 100;
401 LOGGER.info(String.format("Completed processing batch %d/%d (%.0f%%) in %,dms", ctr, max, percent, proc.getDurationMillis()));
402 } catch (InterruptedException ex) {
403 Thread.currentThread().interrupt();
404 throw new RuntimeException(ex);
405 } catch (ExecutionException ex) {
406 LOGGER.error("Exception processing NVD API Results", ex);
407 throw new RuntimeException(ex);
408 }
409 }
410 if (lastModifiedRequest != null) {
411 dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, ZonedDateTime.now());
412 dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest);
413 }
414 return updated;
415 } finally {
416 if (processingExecutorService != null) {
417 processingExecutorService.shutdownNow();
418 }
419 }
420 }
421
422
423
424
425
426
427
428 private boolean isUpdateConfiguredFalse() {
429 if (!settings.getBoolean(Settings.KEYS.UPDATE_NVDCVE_ENABLED, true)) {
430 return true;
431 }
432 boolean autoUpdate = true;
433 try {
434 autoUpdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
435 } catch (InvalidSettingException ex) {
436 LOGGER.debug("Invalid setting for auto-update; using true.");
437 }
438 return !autoUpdate;
439 }
440
441 @Override
442 public boolean purge(Engine engine) {
443 boolean result = true;
444 try {
445 final File dataDir = engine.getSettings().getDataDirectory();
446 final File db = new File(dataDir, engine.getSettings().getString(Settings.KEYS.DB_FILE_NAME, "odc.mv.db"));
447 if (db.exists()) {
448 if (db.delete()) {
449 LOGGER.info("Database file purged; local copy of the NVD has been removed");
450 } else {
451 LOGGER.error("Unable to delete '{}'; please delete the file manually", db.getAbsolutePath());
452 result = false;
453 }
454 } else {
455 LOGGER.info("Unable to purge database; the database file does not exist: {}", db.getAbsolutePath());
456 result = false;
457 }
458 final File traceFile = new File(dataDir, "odc.trace.db");
459 if (traceFile.exists() && !traceFile.delete()) {
460 LOGGER.error("Unable to delete '{}'; please delete the file manually", traceFile.getAbsolutePath());
461 result = false;
462 }
463 final File lockFile = new File(dataDir, "odc.update.lock");
464 if (lockFile.exists() && !lockFile.delete()) {
465 LOGGER.error("Unable to delete '{}'; please delete the file manually", lockFile.getAbsolutePath());
466 result = false;
467 }
468 } catch (IOException ex) {
469 final String msg = "Unable to delete the database";
470 LOGGER.error(msg, ex);
471 result = false;
472 }
473 return result;
474 }
475
476
477
478
479
480
481
482
483
484 private boolean checkUpdate() throws UpdateException {
485 boolean proceed = true;
486
487 final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0);
488 if (dataExists() && 0 < validForHours) {
489
490 final long validForSeconds = validForHours * 60L * 60L;
491 final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_CHECKED);
492 if (lastChecked != null) {
493 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
494 final Duration duration = Duration.between(lastChecked, now);
495 final long difference = duration.getSeconds();
496 proceed = difference > validForSeconds;
497 if (!proceed) {
498 LOGGER.info("Skipping NVD API Cache check since last check was within {} hours.", validForHours);
499 LOGGER.debug("Last NVD API was at {}, and now {} is within {} s.", lastChecked, now, validForSeconds);
500 }
501 } else {
502 LOGGER.warn("NVD cache last checked not present; updating the entire database. This could occur if you are "
503 + "switching back and forth from using the API vs a datafeed or if you are using a database created prior to ODC 9.x");
504 }
505 }
506 return proceed;
507 }
508
509
510
511
512
513
514 private boolean dataExists() {
515 return cveDb.dataExists();
516 }
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533 protected final Map<String, String> getUpdatesNeeded(String url, String filePattern,
534 Properties cacheProperties, ZonedDateTime now) throws UpdateException {
535 LOGGER.debug("starting getUpdatesNeeded() ...");
536 final Map<String, String> updates = new HashMap<>();
537 if (dbProperties != null && !dbProperties.isEmpty()) {
538 final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
539
540
541
542 final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
543 boolean needsFullUpdate = false;
544 for (int y = startYear; y <= endYear; y++) {
545 final ZonedDateTime val = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + y);
546 if (val == null) {
547 needsFullUpdate = true;
548 break;
549 }
550 }
551 final ZonedDateTime lastUpdated = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED);
552 final int days = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_VALID_FOR_DAYS, 7);
553
554 if (!needsFullUpdate && lastUpdated.equals(DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE))) {
555 return updates;
556 } else {
557 updates.put("modified", url + MessageFormat.format(filePattern, "modified"));
558 if (needsFullUpdate) {
559 for (int i = startYear; i <= endYear; i++) {
560 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
561 updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i)));
562 }
563 }
564 } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) {
565 for (int i = startYear; i <= endYear; i++) {
566 if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) {
567 final ZonedDateTime lastModifiedCache = DatabaseProperties.getTimestamp(cacheProperties,
568 NVD_API_CACHE_MODIFIED_DATE + "." + i);
569 final ZonedDateTime lastModifiedDB = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + i);
570 if (lastModifiedDB == null || lastModifiedCache.compareTo(lastModifiedDB) > 0) {
571 updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i)));
572 }
573 }
574 }
575 }
576 }
577 }
578 if (updates.size() > 3) {
579 LOGGER.info("NVD API Cache requires several updates; this could take a couple of minutes.");
580 }
581 return updates;
582 }
583
584
585
586
587
588
589
590
591
592 protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException {
593 final Downloader d = new Downloader(settings);
594 final Properties properties = new Properties();
595 try {
596 final URL u = new URI(url + "cache.properties").toURL();
597 final String content = d.fetchContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
598 properties.load(new StringReader(content));
599
600 } catch (URISyntaxException ex) {
601 throw new UpdateException("Invalid NVD Cache URL", ex);
602 } catch (DownloadFailedException | ResourceNotFoundException ex) {
603 String metaPattern;
604 if (pattern == null) {
605 metaPattern = "nvdcve-{0}.meta";
606 } else {
607 metaPattern = pattern.replace(".json.gz", ".meta");
608 }
609 try {
610 URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL();
611 String content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
612 Properties props = new Properties();
613 props.load(new StringReader(content));
614 ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
615 DatabaseProperties.setTimestamp(properties, "lastModifiedDate.modified", lmd);
616 DatabaseProperties.setTimestamp(properties, "lastModifiedDate", lmd);
617 final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
618 final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
619 final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
620 for (int y = startYear; y <= endYear; y++) {
621 metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL();
622 content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
623 props.clear();
624 props.load(new StringReader(content));
625 lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
626 DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + String.valueOf(y), lmd);
627 }
628 } catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex1) {
629 throw new UpdateException("Unable to download the data feed META files", ex);
630 }
631 } catch (TooManyRequestsException ex) {
632 throw new UpdateException("Unable to download the NVD API cache.properties", ex);
633 } catch (IOException ex) {
634 throw new UpdateException("Invalid NVD Cache Properties file contents", ex);
635 }
636 return properties;
637 }
638
639 protected static class UrlData {
640
641 private final String url;
642
643 private final String pattern;
644
645 public UrlData(String url, String pattern) {
646 this.url = url;
647 this.pattern = pattern;
648 }
649
650
651
652
653
654
655 public String getPattern() {
656 return pattern;
657 }
658
659
660
661
662
663
664 public String getUrl() {
665 return url;
666 }
667
668 }
669 }