Skip to content

Commit

Permalink
SONARTEXT-1 Fix source encoding WARN by converting BinaryFilePredicat…
Browse files Browse the repository at this point in the history
…e as a org.sonar.api.batch.fs.FilePredicate (#72)
  • Loading branch information
alban-auzeill authored Jan 16, 2023
1 parent b9ea722 commit dbd3ce4
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 92 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,11 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.sonar.api.batch.fs.FilePredicate;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.plugins.secrets.api.EntropyChecker;

public class BinaryFilePredicate {
public class NotBinaryFilePredicate implements FilePredicate {

private static final Set<String> DEFAULT_BINARY_EXTENSIONS = new HashSet<>(Arrays.asList(
"3dm",
Expand Down Expand Up @@ -406,7 +408,7 @@ public class BinaryFilePredicate {
private final Set<String> binaryFileExtensions;
private final List<String> binaryFileSuffixes;

public BinaryFilePredicate(String... additionalBinarySuffixes) {
public NotBinaryFilePredicate(String... additionalBinarySuffixes) {
binaryFileExtensions = new HashSet<>(DEFAULT_BINARY_EXTENSIONS);
binaryFileSuffixes = new ArrayList<>(DEFAULT_BINARY_SUFFIXES);
List<String> cleanedSuffixes = Arrays.stream(additionalBinarySuffixes)
Expand All @@ -424,11 +426,14 @@ public BinaryFilePredicate(String... additionalBinarySuffixes) {
}
}

public boolean hasBinaryFileName(String filename) {
@Override
public boolean apply(InputFile inputFile) {
String filename = inputFile.filename();
String extension = extension(filename);
return (extension != null && binaryFileExtensions.contains(extension)) ||
binaryFileSuffixes.stream().anyMatch(filename::endsWith) ||
isMd5OrSha1(filename);
boolean hasBinaryExtension = extension != null && binaryFileExtensions.contains(extension);
return !hasBinaryExtension &&
binaryFileSuffixes.stream().noneMatch(filename::endsWith) &&
!isMd5OrSha1(filename);
}

public boolean isMd5OrSha1(String filename) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ public void execute(SensorContext sensorContext) {
return;
}

BinaryFilePredicate binaryFilePredicate = binaryFilePredicate(sensorContext);
List<InputFile> allInputFiles = getAllInputFiles(sensorContext, binaryFilePredicate);
NotBinaryFilePredicate notBinaryFilePredicate = binaryFilePredicate(sensorContext);
List<InputFile> allInputFiles = getAllInputFiles(sensorContext, notBinaryFilePredicate);
if (allInputFiles.isEmpty()) {
return;
}
Expand All @@ -83,7 +83,7 @@ public void execute(SensorContext sensorContext) {
cancelled = true;
break;
}
analyze(sensorContext, activeChecks, inputFile, binaryFilePredicate);
analyze(sensorContext, activeChecks, inputFile, notBinaryFilePredicate);
progressReport.nextFile();
}
} finally {
Expand All @@ -96,27 +96,25 @@ public void execute(SensorContext sensorContext) {

}

private static BinaryFilePredicate binaryFilePredicate(SensorContext sensorContext) {
return new BinaryFilePredicate(sensorContext.config().getStringArray(TextAndSecretsSensor.EXCLUDED_FILE_SUFFIXES_KEY));
private static NotBinaryFilePredicate binaryFilePredicate(SensorContext sensorContext) {
return new NotBinaryFilePredicate(sensorContext.config().getStringArray(TextAndSecretsSensor.EXCLUDED_FILE_SUFFIXES_KEY));
}

private static List<InputFile> getAllInputFiles(SensorContext sensorContext, BinaryFilePredicate binaryFilePredicate) {
private static List<InputFile> getAllInputFiles(SensorContext sensorContext, NotBinaryFilePredicate notBinaryFilePredicate) {
List<InputFile> allInputFiles = new ArrayList<>();
FileSystem fileSystem = sensorContext.fileSystem();
for (InputFile inputFile : fileSystem.inputFiles(fileSystem.predicates().all())) {
if (!binaryFilePredicate.hasBinaryFileName(inputFile.filename())) {
allInputFiles.add(inputFile);
}
for (InputFile inputFile : fileSystem.inputFiles(notBinaryFilePredicate)) {
allInputFiles.add(inputFile);
}
return allInputFiles;
}

private void analyze(SensorContext sensorContext, List<Check> activeChecks, InputFile inputFile, BinaryFilePredicate binaryFilePredicate) {
if (!binaryFilePredicate.hasBinaryFileName(inputFile.filename())) {
private void analyze(SensorContext sensorContext, List<Check> activeChecks, InputFile inputFile, NotBinaryFilePredicate notBinaryFilePredicate) {
if (notBinaryFilePredicate.apply(inputFile)) {
try {
InputFileContext inputFileContext = new InputFileContext(sensorContext, inputFile);
if (inputFileContext.hasNonTextCharacters()) {
excludeBinaryFileExtension(binaryFilePredicate, inputFile);
excludeBinaryFileExtension(notBinaryFilePredicate, inputFile);
} else {
for (Check check : activeChecks) {
check.analyze(inputFileContext);
Expand All @@ -128,10 +126,10 @@ private void analyze(SensorContext sensorContext, List<Check> activeChecks, Inpu
}
}

private void excludeBinaryFileExtension(BinaryFilePredicate binaryFilePredicate, InputFile inputFile) {
String extension = BinaryFilePredicate.extension(inputFile.filename());
private void excludeBinaryFileExtension(NotBinaryFilePredicate notBinaryFilePredicate, InputFile inputFile) {
String extension = NotBinaryFilePredicate.extension(inputFile.filename());
if (extension != null) {
binaryFilePredicate.addBinaryFileExtension(extension);
notBinaryFilePredicate.addBinaryFileExtension(extension);
LOG.warn("'{}' was added to the binary file filter because the file '{}' is a binary file.", extension, inputFile);
if (displayHelpAboutExcludingBinaryFile) {
displayHelpAboutExcludingBinaryFile = false;
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* SonarQube Text Plugin
* Copyright (C) 2021-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.plugins.common;

import java.nio.file.Path;
import org.junit.jupiter.api.Test;
import org.sonar.api.batch.fs.InputFile;

import static org.assertj.core.api.Assertions.assertThat;

class NotBinaryFilePredicateTest {

@Test
void all_values() {
NotBinaryFilePredicate predicate = new NotBinaryFilePredicate(".foo", ".foo.bar", "_my_suffix", "", "oo.");
assertThat(predicate.apply(inputFile("foo.foo"))).isFalse();
assertThat(predicate.apply(inputFile("foo.exe"))).isFalse();
assertThat(predicate.apply(inputFile("foo.EXE"))).isFalse();
assertThat(predicate.apply(inputFile("foo.UNKNOWN"))).isTrue();
assertThat(predicate.apply(inputFile("foo.txt"))).isTrue();
assertThat(predicate.apply(inputFile("boom.foo.bar"))).isFalse();
assertThat(predicate.apply(inputFile("boom.foo.other"))).isTrue();
assertThat(predicate.apply(inputFile("boom_suffix"))).isTrue();
assertThat(predicate.apply(inputFile("boom_my_suffix"))).isFalse();
assertThat(predicate.apply(inputFile("_"))).isTrue();
assertThat(predicate.apply(inputFile("foo."))).isFalse();
assertThat(predicate.apply(inputFile("bar."))).isTrue();
assertThat(predicate.apply(inputFile(""))).isTrue();
assertThat(predicate.apply(inputFile("cacerts"))).isFalse();
assertThat(predicate.apply(inputFile("ec54a0d82c5938c"))).isTrue();
assertThat(predicate.apply(inputFile("EC54A0D82C5938C"))).isTrue();
assertThat(predicate.apply(inputFile("9f17e505386ec54a0d82c5938ca15805"))).isFalse();
assertThat(predicate.apply(inputFile("9F17E505386EC54A0D82C5938CA15805"))).isFalse();
assertThat(predicate.apply(inputFile("abac0000111122222233333444455555"))).isTrue();
assertThat(predicate.apply(inputFile("Boum123Boum456Boum789Boum123Boum"))).isTrue();
assertThat(predicate.apply(inputFile("9f17e505386ec54a0d82c5938ca15805.txt"))).isTrue();
assertThat(predicate.apply(inputFile("71c686d48275f9b552946aedf50d67e218064cbd"))).isFalse();
assertThat(predicate.apply(inputFile("9dc59c1853a762fb0fe420b0e18b7861b0045f291f8dd75ab9e9d578a31e729a"))).isFalse();
assertThat(predicate.apply(inputFile("faccb33a9627c7269e7ce4e4a37beb2809410a90612977b61944b1680411f39eb2b773adf539d320c3b48c1a0ddfc9b01fc0673a1d893cdb30878b7432f0ebc6")))
.isFalse();
predicate.addBinaryFileExtension("txt");
assertThat(predicate.apply(inputFile("foo.txt"))).isFalse();
}

private static InputFile inputFile(String filename) {
return TestUtils.inputFile(Path.of(filename), "// empty content");
}

@Test
void extensions() {
assertThat(NotBinaryFilePredicate.extension("")).isNull();
assertThat(NotBinaryFilePredicate.extension("foo.txt")).isEqualTo("txt");
assertThat(NotBinaryFilePredicate.extension("foo")).isNull();
assertThat(NotBinaryFilePredicate.extension(".txt")).isEqualTo("txt");
assertThat(NotBinaryFilePredicate.extension("foo.bar.c")).isEqualTo("c");
assertThat(NotBinaryFilePredicate.extension("...c")).isEqualTo("c");
assertThat(NotBinaryFilePredicate.extension("foo.")).isNull();
assertThat(NotBinaryFilePredicate.extension("foo.bar.")).isNull();
;
}

}

0 comments on commit dbd3ce4

Please sign in to comment.