1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import org.apache.commons.compress.archivers.ArchiveEntry;
21 import org.apache.commons.compress.archivers.ArchiveInputStream;
22 import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream;
23 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
24 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
25 import org.apache.commons.compress.archivers.zip.ZipFile;
26 import org.apache.commons.compress.compressors.CompressorInputStream;
27 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
28 import org.apache.commons.compress.compressors.bzip2.BZip2Utils;
29 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
30 import org.apache.commons.compress.compressors.gzip.GzipUtils;
31 import org.apache.commons.io.IOUtils;
32 import org.eclipse.packager.rpm.RpmTag;
33 import org.eclipse.packager.rpm.parse.RpmInputStream;
34 import org.owasp.dependencycheck.Engine;
35 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
36 import org.owasp.dependencycheck.analyzer.exception.ArchiveExtractionException;
37 import org.owasp.dependencycheck.analyzer.exception.UnexpectedAnalysisException;
38 import org.owasp.dependencycheck.dependency.Dependency;
39 import org.owasp.dependencycheck.exception.InitializationException;
40 import org.owasp.dependencycheck.utils.FileFilterBuilder;
41 import org.owasp.dependencycheck.utils.FileUtils;
42 import org.owasp.dependencycheck.utils.Settings;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45
46 import javax.annotation.concurrent.ThreadSafe;
47 import java.io.BufferedInputStream;
48 import java.io.File;
49 import java.io.FileFilter;
50 import java.io.FileInputStream;
51 import java.io.FileNotFoundException;
52 import java.io.FileOutputStream;
53 import java.io.IOException;
54 import java.nio.file.Files;
55 import java.nio.file.Path;
56 import java.util.Collections;
57 import java.util.Enumeration;
58 import java.util.HashSet;
59 import java.util.List;
60 import java.util.Set;
61 import java.util.concurrent.atomic.AtomicInteger;
62 import java.util.zip.ZipEntry;
63 import java.util.zip.ZipInputStream;
64
65 import static org.owasp.dependencycheck.analyzer.AbstractNpmAnalyzer.shouldProcess;
66
67
68
69
70
71
72
73
74 @ThreadSafe
75 public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
76
77
78
79
80 private static final Logger LOGGER = LoggerFactory.getLogger(ArchiveAnalyzer.class);
81
82
83
84
85 private static final AtomicInteger DIRECTORY_COUNT = new AtomicInteger(0);
86
87
88
89 private File tempFileLocation = null;
90
91
92
93
94 private int maxScanDepth;
95
96
97
98 private FileFilter fileFilter = null;
99
100
101
102 private static final Set<String> KNOWN_ZIP_EXT = Collections.unmodifiableSet(
103 newHashSet("zip", "ear", "war", "jar", "sar", "apk", "nupkg", "aar"));
104
105
106
107 private static final Set<String> ADDITIONAL_ZIP_EXT = new HashSet<>();
108
109
110
111
112
113 private static final Set<String> EXTENSIONS = Collections.unmodifiableSet(
114 newHashSet("tar", "gz", "tgz", "bz2", "tbz2", "rpm"));
115
116
117
118
119
120 private static final FileFilter REMOVE_FROM_ANALYSIS = FileFilterBuilder.newInstance()
121 .addExtensions("zip", "tar", "gz", "tgz", "bz2", "tbz2", "nupkg", "rpm").build();
122
123
124
125 private static final FileFilter ZIP_FILTER = FileFilterBuilder.newInstance().addExtensions("zip").build();
126
127
128
129
130
131 private static final String ANALYZER_NAME = "Archive Analyzer";
132
133
134
135 private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INITIAL;
136
137
138
139
140 public ArchiveAnalyzer() {
141 }
142
143
144
145
146
147
148 @Override
149 public void initialize(Settings settings) {
150 super.initialize(settings);
151 initializeSettings();
152 }
153
154 @Override
155 protected FileFilter getFileFilter() {
156 return fileFilter;
157 }
158
159
160
161
162
163
164 @Override
165 public String getName() {
166 return ANALYZER_NAME;
167 }
168
169
170
171
172
173
174 @Override
175 public AnalysisPhase getAnalysisPhase() {
176 return ANALYSIS_PHASE;
177 }
178
179
180
181
182
183
184
185
186 @Override
187 protected String getAnalyzerEnabledSettingKey() {
188 return Settings.KEYS.ANALYZER_ARCHIVE_ENABLED;
189 }
190
191
192
193
194
195
196
197
198 @Override
199 public void prepareFileTypeAnalyzer(Engine engine) throws InitializationException {
200 try {
201 final File baseDir = getSettings().getTempDirectory();
202 tempFileLocation = File.createTempFile("check", "tmp", baseDir);
203 if (!tempFileLocation.delete()) {
204 setEnabled(false);
205 final String msg = String.format("Unable to delete temporary file '%s'.", tempFileLocation.getAbsolutePath());
206 throw new InitializationException(msg);
207 }
208 if (!tempFileLocation.mkdirs()) {
209 setEnabled(false);
210 final String msg = String.format("Unable to create directory '%s'.", tempFileLocation.getAbsolutePath());
211 throw new InitializationException(msg);
212 }
213 } catch (IOException ex) {
214 setEnabled(false);
215 throw new InitializationException("Unable to create a temporary file", ex);
216 }
217 }
218
219
220
221
222
223
224
225
226 @Override
227 public void closeAnalyzer() throws Exception {
228 if (tempFileLocation != null && tempFileLocation.exists()) {
229 LOGGER.debug("Attempting to delete temporary files from `{}`", tempFileLocation.toString());
230 final boolean success = FileUtils.delete(tempFileLocation);
231 if (!success && tempFileLocation.exists()) {
232 final String[] l = tempFileLocation.list();
233 if (l != null && l.length > 0) {
234 LOGGER.warn("Failed to delete the Archive Analyzer's temporary files from `{}`, "
235 + "see the log for more details", tempFileLocation.toString());
236 }
237 }
238 }
239 }
240
241
242
243
244
245
246
247
248
249
250 @Override
251 public boolean accept(File pathname) {
252 boolean accept = super.accept(pathname);
253 final boolean npmEnabled = getSettings().getBoolean(Settings.KEYS.ANALYZER_NODE_AUDIT_ENABLED, false);
254 final boolean yarnEnabled = getSettings().getBoolean(Settings.KEYS.ANALYZER_YARN_AUDIT_ENABLED, false);
255 final boolean pnpmEnabled = getSettings().getBoolean(Settings.KEYS.ANALYZER_PNPM_AUDIT_ENABLED, false);
256 if (accept && (npmEnabled || yarnEnabled || pnpmEnabled)) {
257 try {
258 accept = shouldProcess(pathname);
259 } catch (AnalysisException ex) {
260 throw new UnexpectedAnalysisException(ex.getMessage(), ex.getCause());
261 }
262 }
263 return accept;
264 }
265
266
267
268
269
270
271
272
273
274
275 @Override
276 public void analyzeDependency(Dependency dependency, Engine engine) throws AnalysisException {
277 extractAndAnalyze(dependency, engine, 0);
278 engine.sortDependencies();
279 }
280
281
282
283
284
285
286
287
288
289
290
291
292 private void extractAndAnalyze(Dependency dependency, Engine engine, int scanDepth) throws AnalysisException {
293 final File f = new File(dependency.getActualFilePath());
294 final File tmpDir = getNextTempDirectory();
295 extractFiles(f, tmpDir, engine);
296
297
298 final List<Dependency> dependencySet = findMoreDependencies(engine, tmpDir);
299
300 if (dependencySet != null && !dependencySet.isEmpty()) {
301 for (Dependency d : dependencySet) {
302 if (d.getFilePath().startsWith(tmpDir.getAbsolutePath())) {
303
304 final String displayPath = String.format("%s%s",
305 dependency.getFilePath(),
306 d.getActualFilePath().substring(tmpDir.getAbsolutePath().length()));
307 final String displayName = String.format("%s: %s",
308 dependency.getFileName(),
309 d.getFileName());
310 d.setFilePath(displayPath);
311 d.setFileName(displayName);
312 d.addAllProjectReferences(dependency.getProjectReferences());
313
314
315
316 if (this.accept(d.getActualFile()) && scanDepth < maxScanDepth) {
317 extractAndAnalyze(d, engine, scanDepth + 1);
318 }
319 } else {
320 dependencySet.stream().filter((sub) -> sub.getFilePath().startsWith(tmpDir.getAbsolutePath())).forEach((sub) -> {
321 final String displayPath = String.format("%s%s",
322 dependency.getFilePath(),
323 sub.getActualFilePath().substring(tmpDir.getAbsolutePath().length()));
324 final String displayName = String.format("%s: %s",
325 dependency.getFileName(),
326 sub.getFileName());
327 sub.setFilePath(displayPath);
328 sub.setFileName(displayName);
329 });
330 }
331 }
332 }
333 if (REMOVE_FROM_ANALYSIS.accept(dependency.getActualFile())) {
334 addDisguisedJarsToDependencies(dependency, engine);
335 engine.removeDependency(dependency);
336 }
337 }
338
339
340
341
342
343
344
345
346
347 private void addDisguisedJarsToDependencies(Dependency dependency, Engine engine) throws AnalysisException {
348 if (ZIP_FILTER.accept(dependency.getActualFile()) && isZipFileActuallyJarFile(dependency)) {
349 final File tempDir = getNextTempDirectory();
350 final String fileName = dependency.getFileName();
351
352 LOGGER.info("The zip file '{}' appears to be a JAR file, making a copy and analyzing it as a JAR.", fileName);
353 final File tmpLoc = new File(tempDir, fileName.substring(0, fileName.length() - 3) + "jar");
354
355
356 final String archiveMd5 = dependency.getMd5sum();
357 final String archiveSha1 = dependency.getSha1sum();
358 final String archiveSha256 = dependency.getSha256sum();
359 try {
360 dependency.setMd5sum("");
361 dependency.setSha1sum("");
362 dependency.setSha256sum("");
363 Files.copy(dependency.getActualFile().toPath(), tmpLoc.toPath());
364 final List<Dependency> dependencySet = findMoreDependencies(engine, tmpLoc);
365 if (dependencySet != null && !dependencySet.isEmpty()) {
366 dependencySet.forEach((d) -> {
367
368 if (d.getActualFile().equals(tmpLoc)) {
369 d.setFilePath(dependency.getFilePath());
370 d.setDisplayFileName(dependency.getFileName());
371 } else {
372 d.getRelatedDependencies().stream().filter((rel) -> rel.getActualFile().equals(tmpLoc)).forEach((rel) -> {
373 rel.setFilePath(dependency.getFilePath());
374 rel.setDisplayFileName(dependency.getFileName());
375 });
376 }
377 });
378 }
379 } catch (IOException ex) {
380 LOGGER.debug("Unable to perform deep copy on '{}'", dependency.getActualFile().getPath(), ex);
381 } finally {
382 dependency.setMd5sum(archiveMd5);
383 dependency.setSha1sum(archiveSha1);
384 dependency.setSha256sum(archiveSha256);
385 }
386 }
387 }
388
389
390
391
392
393
394
395
396 private static List<Dependency> findMoreDependencies(Engine engine, File file) {
397 return engine.scan(file);
398 }
399
400
401
402
403
404
405
406 private File getNextTempDirectory() throws AnalysisException {
407 final File directory = new File(tempFileLocation, String.valueOf(DIRECTORY_COUNT.incrementAndGet()));
408
409 if (directory.exists()) {
410 return getNextTempDirectory();
411 }
412 if (!directory.mkdirs()) {
413 final String msg = String.format("Unable to create temp directory '%s'.", directory.getAbsolutePath());
414 throw new AnalysisException(msg);
415 }
416 return directory;
417 }
418
419
420
421
422
423
424
425
426
427 private void extractFiles(File archive, File destination, Engine engine) throws AnalysisException {
428 if (archive != null && destination != null) {
429 String archiveExt = FileUtils.getFileExtension(archive.getName());
430 if (archiveExt == null) {
431 return;
432 }
433 archiveExt = archiveExt.toLowerCase();
434
435 final FileInputStream fis;
436 try {
437 fis = new FileInputStream(archive);
438 } catch (FileNotFoundException ex) {
439 final String msg = String.format("Error extracting file `%s`: %s", archive.getAbsolutePath(), ex.getMessage());
440 LOGGER.debug(msg, ex);
441 throw new AnalysisException(msg);
442 }
443 BufferedInputStream in = null;
444
445 ZipInputStream zin = null;
446 TarArchiveInputStream tin = null;
447 GzipCompressorInputStream gin = null;
448 BZip2CompressorInputStream bzin = null;
449 RpmInputStream rin = null;
450 CpioArchiveInputStream cain = null;
451 try {
452 if (KNOWN_ZIP_EXT.contains(archiveExt) || ADDITIONAL_ZIP_EXT.contains(archiveExt)) {
453 in = new BufferedInputStream(fis);
454 ensureReadableJar(archiveExt, in);
455
456 zin = new ZipInputStream(in);
457 extractArchive(zin, destination, engine);
458 } else if ("tar".equals(archiveExt)) {
459 in = new BufferedInputStream(fis);
460 tin = new TarArchiveInputStream(in);
461 extractArchive(tin, destination, engine);
462 } else if ("gz".equals(archiveExt) || "tgz".equals(archiveExt)) {
463 final String uncompressedName = GzipUtils.getUncompressedFileName(archive.getName());
464 final File f = new File(destination, uncompressedName);
465 if (engine.accept(f)) {
466 final String destPath = destination.getCanonicalPath();
467 if (!f.getCanonicalPath().startsWith(destPath)) {
468 final String msg = String.format(
469 "Archive (%s) contains a file that would be written outside of the destination directory",
470 archive.getPath());
471 throw new AnalysisException(msg);
472 }
473 in = new BufferedInputStream(fis);
474 gin = new GzipCompressorInputStream(in);
475 decompressFile(gin, f);
476 }
477 } else if ("bz2".equals(archiveExt) || "tbz2".equals(archiveExt)) {
478 final String uncompressedName = BZip2Utils.getUncompressedFileName(archive.getName());
479 final File f = new File(destination, uncompressedName);
480 if (engine.accept(f)) {
481 final String destPath = destination.getCanonicalPath();
482 if (!f.getCanonicalPath().startsWith(destPath)) {
483 final String msg = String.format(
484 "Archive (%s) contains a file that would be written outside of the destination directory",
485 archive.getPath());
486 throw new AnalysisException(msg);
487 }
488 in = new BufferedInputStream(fis);
489 bzin = new BZip2CompressorInputStream(in);
490 decompressFile(bzin, f);
491 }
492 } else if ("rpm".equals(archiveExt)) {
493 rin = new RpmInputStream(fis);
494
495
496 rin.getPayloadHeader().getTag(RpmTag.NAME);
497 cain = new CpioArchiveInputStream(rin);
498 extractArchive(cain, destination, engine);
499 }
500 } catch (ArchiveExtractionException ex) {
501 LOGGER.error("Exception extracting archive '{}'.", archive.getName());
502 LOGGER.debug("", ex);
503 throw new AnalysisException(ex.getMessage(), ex);
504 } catch (IOException ex) {
505 LOGGER.error("Exception reading archive '{}'.", archive.getName());
506 LOGGER.debug("", ex);
507 throw new AnalysisException(ex.getMessage(), ex);
508 } finally {
509
510
511 FileUtils.close(fis);
512 FileUtils.close(in);
513 FileUtils.close(zin);
514 FileUtils.close(tin);
515 FileUtils.close(gin);
516 FileUtils.close(bzin);
517 }
518 }
519 }
520
521
522
523
524
525
526
527
528
529
530
531
532
533 private void ensureReadableJar(final String archiveExt, BufferedInputStream in) throws IOException {
534 if (("war".equals(archiveExt) || "jar".equals(archiveExt)) && in.markSupported()) {
535 in.mark(7);
536 final byte[] b = new byte[7];
537 final int read = in.read(b);
538 if (read == 7
539 && b[0] == '#'
540 && b[1] == '!'
541 && b[2] == '/'
542 && b[3] == 'b'
543 && b[4] == 'i'
544 && b[5] == 'n'
545 && b[6] == '/') {
546 boolean stillLooking = true;
547 int chr;
548 int nxtChr;
549
550
551 while (stillLooking && (chr = in.read()) != -1) {
552 if (chr == '\n' || chr == '\r') {
553 in.mark(4);
554 if ((chr = in.read()) != -1) {
555 if (chr == 'P' && (chr = in.read()) != -1) {
556 if (chr == 'K' && (chr = in.read()) != -1) {
557 if ((chr == 3 || chr == 5 || chr == 7) && (nxtChr = in.read()) != -1) {
558 if (nxtChr == chr + 1) {
559 stillLooking = false;
560 in.reset();
561 }
562 }
563 }
564 }
565 }
566 }
567 }
568
569
570 } else {
571 in.reset();
572 }
573 }
574 }
575
576 private void extractArchive(ZipInputStream input, File destination, Engine engine) throws ArchiveExtractionException {
577 ZipEntry entry;
578 try {
579
580 final Path d = destination.toPath();
581 while ((entry = input.getNextEntry()) != null) {
582
583 final Path f = d.resolve(entry.getName()).normalize();
584 if (!f.startsWith(d)) {
585 LOGGER.debug("ZipSlip detected\n-Destination: " + d + "\n-Path: " + f);
586 final String msg = String.format(
587 "Archive contains a file (%s) that would be extracted outside of the target directory.",
588 entry.getName());
589 throw new ArchiveExtractionException(msg);
590 }
591 final File file = f.toFile();
592 if (entry.isDirectory()) {
593 if (!file.exists() && !file.mkdirs()) {
594 final String msg = String.format("Unable to create directory '%s'.", file.getAbsolutePath());
595 throw new AnalysisException(msg);
596 }
597 } else if (engine.accept(file)) {
598 extractAcceptedFile(input, file);
599 }
600 }
601 } catch (IOException | AnalysisException ex) {
602 throw new ArchiveExtractionException(ex);
603 } finally {
604 FileUtils.close(input);
605 }
606 }
607
608
609
610
611
612
613
614
615
616
617 private void extractArchive(ArchiveInputStream input, File destination, Engine engine) throws ArchiveExtractionException {
618 ArchiveEntry entry;
619 try {
620
621 final Path d = destination.toPath();
622 while ((entry = input.getNextEntry()) != null) {
623
624 final Path f = d.resolve(entry.getName()).normalize();
625 if (!f.startsWith(d)) {
626 LOGGER.debug("ZipSlip detected\n-Destination: " + d + "\n-Path: " + f);
627 final String msg = String.format(
628 "Archive contains a file (%s) that would be extracted outside of the target directory.",
629 entry.getName());
630 throw new ArchiveExtractionException(msg);
631 }
632 final File file = f.toFile();
633 if (entry.isDirectory()) {
634 if (!file.exists() && !file.mkdirs()) {
635 final String msg = String.format("Unable to create directory '%s'.", file.getAbsolutePath());
636 throw new AnalysisException(msg);
637 }
638 } else if (engine.accept(file)) {
639 extractAcceptedFile(input, file);
640 }
641 }
642 } catch (IOException | AnalysisException ex) {
643 throw new ArchiveExtractionException(ex);
644 } finally {
645 FileUtils.close(input);
646 }
647 }
648
649
650
651
652
653
654
655
656 private static void extractAcceptedFile(ZipInputStream input, File file) throws AnalysisException {
657 LOGGER.debug("Extracting '{}'", file.getPath());
658 final File parent = file.getParentFile();
659 if (!parent.isDirectory() && !parent.mkdirs()) {
660 final String msg = String.format("Unable to build directory '%s'.", parent.getAbsolutePath());
661 throw new AnalysisException(msg);
662 }
663 try (FileOutputStream fos = new FileOutputStream(file)) {
664 IOUtils.copy(input, fos);
665 } catch (FileNotFoundException ex) {
666 LOGGER.debug("", ex);
667 final String msg = String.format("Unable to find file '%s'.", file.getName());
668 throw new AnalysisException(msg, ex);
669 } catch (IOException ex) {
670 LOGGER.debug("", ex);
671 final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
672 throw new AnalysisException(msg, ex);
673 }
674 }
675
676
677
678
679
680
681
682
683 private static void extractAcceptedFile(ArchiveInputStream input, File file) throws AnalysisException {
684 LOGGER.debug("Extracting '{}'", file.getPath());
685 final File parent = file.getParentFile();
686 if (!parent.isDirectory() && !parent.mkdirs()) {
687 final String msg = String.format("Unable to build directory '%s'.", parent.getAbsolutePath());
688 throw new AnalysisException(msg);
689 }
690 try (FileOutputStream fos = new FileOutputStream(file)) {
691 IOUtils.copy(input, fos);
692 } catch (FileNotFoundException ex) {
693 LOGGER.debug("", ex);
694 final String msg = String.format("Unable to find file '%s'.", file.getName());
695 throw new AnalysisException(msg, ex);
696 } catch (IOException ex) {
697 LOGGER.debug("", ex);
698 final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
699 throw new AnalysisException(msg, ex);
700 }
701 }
702
703
704
705
706
707
708
709
710
711 private void decompressFile(CompressorInputStream inputStream, File outputFile) throws ArchiveExtractionException {
712 LOGGER.debug("Decompressing '{}'", outputFile.getPath());
713 try (FileOutputStream out = new FileOutputStream(outputFile)) {
714 IOUtils.copy(inputStream, out);
715 } catch (IOException ex) {
716 LOGGER.debug("", ex);
717 throw new ArchiveExtractionException(ex);
718 }
719 }
720
721
722
723
724
725
726
727 private boolean isZipFileActuallyJarFile(Dependency dependency) {
728 boolean isJar = false;
729 ZipFile zip = null;
730 try {
731 zip = ZipFile.builder().setFile(dependency.getActualFilePath()).get();
732 if (zip.getEntry("META-INF/MANIFEST.MF") != null
733 || zip.getEntry("META-INF/maven") != null) {
734 final Enumeration<ZipArchiveEntry> entries = zip.getEntries();
735 while (entries.hasMoreElements()) {
736 final ZipArchiveEntry entry = entries.nextElement();
737 if (!entry.isDirectory()) {
738 final String name = entry.getName().toLowerCase();
739 if (name.endsWith(".class")) {
740 isJar = true;
741 break;
742 }
743 }
744 }
745 }
746 } catch (IOException ex) {
747 LOGGER.debug("Unable to unzip zip file '{}'", dependency.getFilePath(), ex);
748 } finally {
749 ZipFile.closeQuietly(zip);
750 }
751 return isJar;
752 }
753
754
755
756
757
758 private void initializeSettings() {
759 maxScanDepth = getSettings().getInt("archive.scan.depth", 3);
760 final Set<String> extensions = new HashSet<>(EXTENSIONS);
761 extensions.addAll(KNOWN_ZIP_EXT);
762 final String additionalZipExt = getSettings().getString(Settings.KEYS.ADDITIONAL_ZIP_EXTENSIONS);
763 if (additionalZipExt != null) {
764 final String[] ext = additionalZipExt.split("\\s*,\\s*");
765 Collections.addAll(extensions, ext);
766 Collections.addAll(ADDITIONAL_ZIP_EXT, ext);
767 }
768 fileFilter = FileFilterBuilder.newInstance().addExtensions(extensions).build();
769 }
770 }