1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import com.github.packageurl.MalformedPackageURLException;
21 import org.semver4j.Semver;
22 import org.semver4j.SemverException;
23 import java.io.File;
24 import java.util.Set;
25 import java.util.regex.Matcher;
26 import java.util.regex.Pattern;
27 import static java.util.stream.Collectors.toSet;
28 import javax.annotation.concurrent.ThreadSafe;
29 import org.owasp.dependencycheck.dependency.Dependency;
30 import org.owasp.dependencycheck.dependency.Vulnerability;
31 import org.owasp.dependencycheck.dependency.naming.Identifier;
32 import org.owasp.dependencycheck.dependency.naming.PurlIdentifier;
33 import org.owasp.dependencycheck.utils.DependencyVersion;
34 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
35 import org.owasp.dependencycheck.utils.Settings;
36 import org.slf4j.Logger;
37 import org.slf4j.LoggerFactory;
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 @ThreadSafe
53 public class DependencyBundlingAnalyzer extends AbstractDependencyComparingAnalyzer {
54
55
56
57
58 private static final Logger LOGGER = LoggerFactory.getLogger(DependencyBundlingAnalyzer.class);
59
60
61
62
63 private static final Pattern STARTING_TEXT_PATTERN = Pattern.compile("^[a-zA-Z0-9]*");
64
65
66
67
68 private static final String ANALYZER_NAME = "Dependency Bundling Analyzer";
69
70
71
72 private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.FINAL;
73
74
75
76
77
78
79 @Override
80 public String getName() {
81 return ANALYZER_NAME;
82 }
83
84
85
86
87
88
89 @Override
90 public AnalysisPhase getAnalysisPhase() {
91 return ANALYSIS_PHASE;
92 }
93
94
95
96
97
98
99
100 @Override
101 protected String getAnalyzerEnabledSettingKey() {
102 return Settings.KEYS.ANALYZER_DEPENDENCY_BUNDLING_ENABLED;
103 }
104
105
106
107
108
109
110
111
112
113 @Override
114 protected boolean evaluateDependencies(final Dependency dependency, final Dependency nextDependency, final Set<Dependency> dependenciesToRemove) {
115 if (hashesMatch(dependency, nextDependency)) {
116 if (!containedInWar(dependency.getFilePath())
117 && !containedInWar(nextDependency.getFilePath())) {
118 if (firstPathIsShortest(dependency.getFilePath(), nextDependency.getFilePath())) {
119 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
120 } else {
121 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
122 return true;
123 }
124 }
125 } else if (isShadedJar(dependency, nextDependency)) {
126 if (dependency.getFileName().toLowerCase().endsWith("pom.xml")) {
127 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
128 nextDependency.removeRelatedDependencies(dependency);
129 return true;
130 } else {
131 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
132 dependency.removeRelatedDependencies(nextDependency);
133 }
134 } else if (isWebJar(dependency, nextDependency)) {
135 if (dependency.getFileName().toLowerCase().endsWith(".js")) {
136 mergeDependencies(nextDependency, dependency, dependenciesToRemove, true);
137 nextDependency.removeRelatedDependencies(dependency);
138 return true;
139 } else {
140 mergeDependencies(dependency, nextDependency, dependenciesToRemove, true);
141 dependency.removeRelatedDependencies(nextDependency);
142 }
143 } else if (cpeIdentifiersMatch(dependency, nextDependency)
144 && hasSameBasePath(dependency, nextDependency)
145 && vulnerabilitiesMatch(dependency, nextDependency)
146 && fileNameMatch(dependency, nextDependency)) {
147 if (isCore(dependency, nextDependency)) {
148 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
149 } else {
150 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
151 return true;
152 }
153 } else if (ecosystemIs(AbstractNpmAnalyzer.NPM_DEPENDENCY_ECOSYSTEM, dependency, nextDependency)
154 && namesAreEqual(dependency, nextDependency)
155 && npmVersionsMatch(dependency.getVersion(), nextDependency.getVersion())) {
156
157 if (!dependency.isVirtual()) {
158 DependencyMergingAnalyzer.mergeDependencies(dependency, nextDependency, dependenciesToRemove);
159 } else {
160 DependencyMergingAnalyzer.mergeDependencies(nextDependency, dependency, dependenciesToRemove);
161 return true;
162 }
163 }
164 return false;
165 }
166
167
168
169
170
171
172
173
174
175
176
177 public static void mergeDependencies(final Dependency dependency,
178 final Dependency relatedDependency, final Set<Dependency> dependenciesToRemove) {
179 mergeDependencies(dependency, relatedDependency, dependenciesToRemove, false);
180 }
181
182
183
184
185
186
187
188
189
190
191
192
193
194 public static void mergeDependencies(final Dependency dependency,
195 final Dependency relatedDependency, final Set<Dependency> dependenciesToRemove,
196 final boolean copyVulnsAndIds) {
197 dependency.addRelatedDependency(relatedDependency);
198 relatedDependency.getRelatedDependencies()
199 .forEach(dependency::addRelatedDependency);
200 relatedDependency.clearRelatedDependencies();
201
202 if (copyVulnsAndIds) {
203 relatedDependency.getSoftwareIdentifiers()
204 .forEach(dependency::addSoftwareIdentifier);
205 relatedDependency.getVulnerableSoftwareIdentifiers()
206 .forEach(dependency::addVulnerableSoftwareIdentifier);
207 relatedDependency.getVulnerabilities()
208 .forEach(dependency::addVulnerability);
209 }
210
211
212 if (dependency.getSha1sum() != null && dependency.getSha1sum().equals(relatedDependency.getSha1sum())) {
213 dependency.addAllProjectReferences(relatedDependency.getProjectReferences());
214 dependency.addAllIncludedBy(relatedDependency.getIncludedBy());
215 }
216 if (dependenciesToRemove != null) {
217 dependenciesToRemove.add(relatedDependency);
218 }
219 }
220
221
222
223
224
225
226
227
228
229 private String getBaseRepoPath(final String path, final String repo) {
230 int pos = path.indexOf(repo + File.separator) + repo.length() + 1;
231 if (pos < repo.length() + 1) {
232 return path;
233 }
234 int tmp = path.indexOf(File.separator, pos);
235 if (tmp <= 0) {
236 return path;
237 }
238 pos = tmp + 1;
239 tmp = path.indexOf(File.separator, pos);
240 if (tmp > 0) {
241 pos = tmp + 1;
242 }
243 return path.substring(0, pos);
244 }
245
246
247
248
249
250
251
252
253
254
255 private boolean fileNameMatch(Dependency dependency1, Dependency dependency2) {
256 if (dependency1 == null || dependency1.getFileName() == null
257 || dependency2 == null || dependency2.getFileName() == null) {
258 return false;
259 }
260 final String fileName1 = dependency1.getActualFile().getName();
261 final String fileName2 = dependency2.getActualFile().getName();
262
263
264 final DependencyVersion version1 = DependencyVersionUtil.parseVersion(fileName1);
265 final DependencyVersion version2 = DependencyVersionUtil.parseVersion(fileName2);
266 if (version1 != null && version2 != null && !version1.equals(version2)) {
267 return false;
268 }
269
270
271 final Matcher match1 = STARTING_TEXT_PATTERN.matcher(fileName1);
272 final Matcher match2 = STARTING_TEXT_PATTERN.matcher(fileName2);
273 if (match1.find() && match2.find()) {
274 return match1.group().equals(match2.group());
275 }
276
277 return false;
278 }
279
280
281
282
283
284
285
286
287
288
289 private boolean cpeIdentifiersMatch(Dependency dependency1, Dependency dependency2) {
290 if (dependency1 == null || dependency1.getVulnerableSoftwareIdentifiers() == null
291 || dependency2 == null || dependency2.getVulnerableSoftwareIdentifiers() == null) {
292 return false;
293 }
294 boolean matches = false;
295 final int cpeCount1 = dependency1.getVulnerableSoftwareIdentifiers().size();
296 final int cpeCount2 = dependency2.getVulnerableSoftwareIdentifiers().size();
297 if (cpeCount1 > 0 && cpeCount1 == cpeCount2) {
298 for (Identifier i : dependency1.getVulnerableSoftwareIdentifiers()) {
299 matches |= dependency2.getVulnerableSoftwareIdentifiers().contains(i);
300 if (!matches) {
301 break;
302 }
303 }
304 }
305 LOGGER.trace("IdentifiersMatch={} ({}, {})", matches, dependency1.getFileName(), dependency2.getFileName());
306 return matches;
307 }
308
309
310
311
312
313
314
315
316 private boolean vulnerabilitiesMatch(Dependency dependency1, Dependency dependency2) {
317 final Set<Vulnerability> one = dependency1.getVulnerabilities();
318 final Set<Vulnerability> two = dependency2.getVulnerabilities();
319 return one != null && two != null
320 && one.size() == two.size()
321 && one.containsAll(two);
322 }
323
324
325
326
327
328
329
330
331 private boolean hasSameBasePath(Dependency dependency1, Dependency dependency2) {
332 if (dependency1 == null || dependency2 == null) {
333 return false;
334 }
335 final File lFile = new File(dependency1.getFilePath());
336 String left = lFile.getParent();
337 final File rFile = new File(dependency2.getFilePath());
338 String right = rFile.getParent();
339 if (left == null) {
340 return right == null;
341 } else if (right == null) {
342 return false;
343 }
344 if (left.equalsIgnoreCase(right)) {
345 return true;
346 }
347 final String localRepo = getSettings().getString(Settings.KEYS.MAVEN_LOCAL_REPO);
348 final Pattern p;
349 if (localRepo == null) {
350 p = Pattern.compile(".*[/\\\\](?<repo>repository|local-repo)[/\\\\].*");
351 } else {
352 final File f = new File(localRepo);
353 final String dir = f.getName();
354 p = Pattern.compile(".*[/\\\\](?<repo>repository|local-repo|" + Pattern.quote(dir) + ")[/\\\\].*");
355 }
356 final Matcher mleft = p.matcher(left);
357 final Matcher mright = p.matcher(right);
358 if (mleft.find() && mright.find()) {
359 left = getBaseRepoPath(left, mleft.group("repo"));
360 right = getBaseRepoPath(right, mright.group("repo"));
361 }
362
363 if (left.equalsIgnoreCase(right)) {
364 return true;
365 }
366
367 for (Dependency child : dependency2.getRelatedDependencies()) {
368 if (hasSameBasePath(child, dependency1)) {
369 return true;
370 }
371 }
372 return false;
373 }
374
375
376
377
378
379
380
381
382
383
384 protected boolean isCore(Dependency left, Dependency right) {
385 final String leftName = left.getFileName().toLowerCase();
386 final String rightName = right.getFileName().toLowerCase();
387
388 final boolean returnVal;
389
390
391 if (left.isVirtual() && !right.isVirtual()) {
392 returnVal = true;
393 } else if (!left.isVirtual() && right.isVirtual()) {
394 returnVal = false;
395 } else if ((!rightName.matches(".*\\.(tar|tgz|gz|zip|ear|war|rpm).+") && leftName.matches(".*\\.(tar|tgz|gz|zip|ear|war|rpm).+"))
396 || (rightName.contains("core") && !leftName.contains("core"))
397 || (rightName.contains("kernel") && !leftName.contains("kernel"))
398 || (rightName.contains("server") && !leftName.contains("server"))
399 || (rightName.contains("project") && !leftName.contains("project"))
400 || (rightName.contains("engine") && !leftName.contains("engine"))
401 || (rightName.contains("akka-stream") && !leftName.contains("akka-stream"))
402 || (rightName.contains("netty-transport") && !leftName.contains("netty-transport"))) {
403 returnVal = false;
404 } else if ((rightName.matches(".*\\.(tar|tgz|gz|zip|ear|war|rpm).+") && !leftName.matches(".*\\.(tar|tgz|gz|zip|ear|war|rpm).+"))
405 || (!rightName.contains("core") && leftName.contains("core"))
406 || (!rightName.contains("kernel") && leftName.contains("kernel"))
407 || (!rightName.contains("server") && leftName.contains("server"))
408 || (!rightName.contains("project") && leftName.contains("project"))
409 || (!rightName.contains("engine") && leftName.contains("engine"))
410 || (!rightName.contains("akka-stream") && leftName.contains("akka-stream"))
411 || (!rightName.contains("netty-transport") && leftName.contains("netty-transport"))) {
412 returnVal = true;
413 } else {
414
415
416
417
418
419
420
421
422
423 returnVal = leftName.length() <= rightName.length();
424 }
425 LOGGER.debug("IsCore={} ({}, {})", returnVal, left.getFileName(), right.getFileName());
426 return returnVal;
427 }
428
429
430
431
432
433
434
435
436
437
438 private boolean hashesMatch(Dependency dependency1, Dependency dependency2) {
439 if (dependency1 == null || dependency2 == null || dependency1.getSha1sum() == null || dependency2.getSha1sum() == null) {
440 return false;
441 }
442 return dependency1.getSha1sum().equals(dependency2.getSha1sum());
443 }
444
445
446
447
448
449
450
451
452
453 protected boolean isWebJar(Dependency dependency, Dependency nextDependency) {
454 if (dependency == null || dependency.getFileName() == null
455 || nextDependency == null || nextDependency.getFileName() == null
456 || dependency.getSoftwareIdentifiers().isEmpty()
457 || nextDependency.getSoftwareIdentifiers().isEmpty()) {
458 return false;
459 }
460 final String mainName = dependency.getFileName().toLowerCase();
461 final String nextName = nextDependency.getFileName().toLowerCase();
462 if (mainName.endsWith(".jar") && nextName.endsWith(".js") && nextName.startsWith(mainName)) {
463 return dependency.getSoftwareIdentifiers()
464 .stream().map(Identifier::getValue).collect(toSet())
465 .containsAll(nextDependency.getSoftwareIdentifiers().stream().map(this::identifierToWebJarForComparison).collect(toSet()));
466 } else if (nextName.endsWith(".jar") && mainName.endsWith("js") && mainName.startsWith(nextName)) {
467 return nextDependency.getSoftwareIdentifiers()
468 .stream().map(Identifier::getValue).collect(toSet())
469 .containsAll(dependency.getSoftwareIdentifiers().stream().map(this::identifierToWebJarForComparison).collect(toSet()));
470 }
471 return false;
472 }
473
474
475
476
477
478
479
480
481 private String identifierToWebJarForComparison(Identifier id) {
482 if (id instanceof PurlIdentifier) {
483 final PurlIdentifier pid = (PurlIdentifier) id;
484 try {
485 final Identifier nid = new PurlIdentifier("maven", "org.webjars", pid.getName(), pid.getVersion(), pid.getConfidence());
486 return nid.getValue();
487 } catch (MalformedPackageURLException ex) {
488 LOGGER.debug("Unable to build webjar purl id", ex);
489 return id.getValue();
490 }
491 } else {
492 return id == null ? "" : id.getValue();
493 }
494 }
495
496
497
498
499
500
501
502
503
504
505 protected boolean isShadedJar(Dependency dependency, Dependency nextDependency) {
506 if (dependency == null || dependency.getFileName() == null
507 || nextDependency == null || nextDependency.getFileName() == null
508 || dependency.getSoftwareIdentifiers().isEmpty()
509 || nextDependency.getSoftwareIdentifiers().isEmpty()) {
510 return false;
511 }
512 final String mainName = dependency.getFileName().toLowerCase();
513 final String nextName = nextDependency.getFileName().toLowerCase();
514 if (mainName.endsWith(".jar") && nextName.endsWith("pom.xml")) {
515 return dependency.getSoftwareIdentifiers().containsAll(nextDependency.getSoftwareIdentifiers());
516 } else if (nextName.endsWith(".jar") && mainName.endsWith("pom.xml")) {
517 return nextDependency.getSoftwareIdentifiers().containsAll(dependency.getSoftwareIdentifiers());
518 }
519 return false;
520 }
521
522
523
524
525
526
527
528
529
530
531 public static boolean firstPathIsShortest(String left, String right) {
532 if (left.contains("dctemp") && !right.contains("dctemp")) {
533 return false;
534 }
535 final String leftPath = left.replace('\\', '/');
536 final String rightPath = right.replace('\\', '/');
537
538 final int leftCount = countChar(leftPath, '/');
539 final int rightCount = countChar(rightPath, '/');
540 if (leftCount == rightCount) {
541 return leftPath.compareTo(rightPath) <= 0;
542 } else {
543 return leftCount < rightCount;
544 }
545 }
546
547
548
549
550
551
552
553
554 private static int countChar(String string, char c) {
555 int count = 0;
556 final int max = string.length();
557 for (int i = 0; i < max; i++) {
558 if (c == string.charAt(i)) {
559 count++;
560 }
561 }
562 return count;
563 }
564
565
566
567
568
569
570
571 private boolean containedInWar(String filePath) {
572 return filePath != null && filePath.matches(".*\\.(ear|war)[\\\\/].*");
573 }
574
575
576
577
578
579
580
581
582
583
584 private boolean ecosystemIs(String ecoSystem, Dependency dependency, Dependency nextDependency) {
585 return ecoSystem.equals(dependency.getEcosystem()) && ecoSystem.equals(nextDependency.getEcosystem());
586 }
587
588
589
590
591
592
593
594
595 private boolean namesAreEqual(Dependency dependency, Dependency nextDependency) {
596 return dependency.getName() != null && dependency.getName().equals(nextDependency.getName());
597 }
598
599
600
601
602
603
604
605
606
607
608 public static boolean npmVersionsMatch(String current, String next) {
609 String left = current;
610 String right = next;
611 if (left == null || right == null) {
612 return false;
613 }
614 if (left.equals(right) || "*".equals(left) || "*".equals(right)) {
615 return true;
616 }
617 if (left.contains(" ")) {
618 if (right.contains(" ")) {
619 return false;
620 }
621 if (!right.matches("^\\d.*$")) {
622 right = stripLeadingNonNumeric(right);
623 if (right == null) {
624 return false;
625 }
626 }
627 try {
628 final Semver v = new Semver(right);
629 return v.satisfies(left);
630 } catch (SemverException ex) {
631 LOGGER.trace("ignore", ex);
632 }
633 } else {
634 if (!left.matches("^\\d.*$")) {
635 left = stripLeadingNonNumeric(left);
636 if (left == null || left.isEmpty()) {
637 return false;
638 }
639 }
640 try {
641 Semver v = new Semver(left);
642 if (!right.isEmpty() && v.satisfies(right)) {
643 return true;
644 }
645 if (!right.contains(" ")) {
646 left = current;
647 right = stripLeadingNonNumeric(right);
648 if (right != null) {
649 v = new Semver(right);
650 return v.satisfies(left);
651 }
652 }
653 } catch (SemverException ex) {
654 LOGGER.trace("ignore", ex);
655 } catch (NullPointerException ex) {
656 LOGGER.error("SemVer comparison error: left:\"{}\", right:\"{}\"", left, right);
657 LOGGER.debug("SemVer comparison resulted in NPE", ex);
658 }
659 }
660 return false;
661 }
662
663
664
665
666
667
668
669
670 private static String stripLeadingNonNumeric(String str) {
671 for (int x = 0; x < str.length(); x++) {
672 if (Character.isDigit(str.codePointAt(x))) {
673 return str.substring(x);
674 }
675 }
676 return null;
677 }
678
679 }