Skip to content

Commit

Permalink
Add SARIF as a reporter option (#166)
Browse files Browse the repository at this point in the history
* cmd/validator/validator_test.go: Fix typo.

* Add SARIF reporter

cmd/validator/validator.go: Add option to select reporter type as SARIF.
pkg/reporter/sarif_reporter.go: Create SARIF report.

https://sarifweb.azurewebsites.net/

* Add tests for SARIF reporter.

cmd/validator/validator_test.go: Test for --reporter=sarif flag.
pkg/reporter/reporter_test.go: Test for SARIF report.

* Add test for SARIF reporter writer.

* cmd/validator/validator.go: Satisfy goreportcard.

* Fix SARIF validation errors

* Combine groupOutput check for junit and sarif.

* Add groupby test for sarif reporter.

* make version, schema, driver name and infoUri constants and move them outside.

* Make gocyclo happy
  • Loading branch information
shiina4119 authored Oct 23, 2024
1 parent 405d9dd commit 5193c9c
Show file tree
Hide file tree
Showing 5 changed files with 323 additions and 9 deletions.
22 changes: 14 additions & 8 deletions cmd/validator/validator.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ optional flags:
-output
Destination of a file to outputting results
-reporter string
Format of the printed report. Options are standard and json (default "standard")
Format of the printed report. Options are standard, json, junit and sarif (default "standard")
-version
Version prints the release version of validator
*/
Expand Down Expand Up @@ -76,7 +76,7 @@ func getFlags() (validatorConfig, error) {
excludeDirsPtr := flag.String("exclude-dirs", "", "Subdirectories to exclude when searching for configuration files")
excludeFileTypesPtr := flag.String("exclude-file-types", "", "A comma separated list of file types to ignore")
outputPtr := flag.String("output", "", "Destination to a file to output results")
reportTypePtr := flag.String("reporter", "standard", "Format of the printed report. Options are standard and json")
reportTypePtr := flag.String("reporter", "standard", "Format of the printed report. Options are standard, json, junit and sarif")
versionPtr := flag.Bool("version", false, "Version prints the release version of validator")
groupOutputPtr := flag.String("groupby", "", "Group output by filetype, directory, pass-fail. Supported for Standard and JSON reports")
quietPtr := flag.Bool("quiet", false, "If quiet flag is set. It doesn't print any output to stdout.")
Expand Down Expand Up @@ -110,16 +110,20 @@ func getFlags() (validatorConfig, error) {
searchPaths = append(searchPaths, flag.Args()...)
}

if *reportTypePtr != "standard" && *reportTypePtr != "json" && *reportTypePtr != "junit" {
fmt.Println("Wrong parameter value for reporter, only supports standard, json or junit")
acceptedReportTypes := map[string]bool{"standard": true, "json": true, "junit": true, "sarif": true}

if !acceptedReportTypes[*reportTypePtr] {
fmt.Println("Wrong parameter value for reporter, only supports standard, json, junit or sarif")
flag.Usage()
return validatorConfig{}, errors.New("Wrong parameter value for reporter, only supports standard, json or junit")
return validatorConfig{}, errors.New("Wrong parameter value for reporter, only supports standard, json, junit or sarif")
}

if *reportTypePtr == "junit" && *groupOutputPtr != "" {
fmt.Println("Wrong parameter value for reporter, groupby is not supported for JUnit reports")
groupOutputReportTypes := map[string]bool{"standard": true, "json": true}

if !groupOutputReportTypes[*reportTypePtr] && *groupOutputPtr != "" {
fmt.Println("Wrong parameter value for reporter, groupby is only supported for standard and JSON reports")
flag.Usage()
return validatorConfig{}, errors.New("Wrong parameter value for reporter, groupby is not supported for JUnit reports")
return validatorConfig{}, errors.New("Wrong parameter value for reporter, groupby is only supported for standard and JSON reports")
}

if depthPtr != nil && isFlagSet("depth") && *depthPtr < 0 {
Expand Down Expand Up @@ -200,6 +204,8 @@ func getReporter(reportType, outputDest *string) reporter.Reporter {
return reporter.NewJunitReporter(*outputDest)
case "json":
return reporter.NewJSONReporter(*outputDest)
case "sarif":
return reporter.NewSARIFReporter(*outputDest)
default:
return reporter.StdoutReporter{}
}
Expand Down
4 changes: 3 additions & 1 deletion cmd/validator/validator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ func Test_flags(t *testing.T) {
{"depth set", []string{"-depth=1", "."}, 0},
{"flags set, wrong reporter", []string{"--exclude-dirs=subdir", "--reporter=wrong", "."}, 1},
{"flags set, json reporter", []string{"--exclude-dirs=subdir", "--reporter=json", "."}, 0},
{"flags set, junit reported", []string{"--exclude-dirs=subdir", "--reporter=junit", "."}, 0},
{"flags set, junit reporter", []string{"--exclude-dirs=subdir", "--reporter=junit", "."}, 0},
{"flags set, sarif reporter", []string{"--exclude-dirs=subdir", "--reporter=sarif", "."}, 0},
{"bad path", []string{"/path/does/not/exit"}, 1},
{"exclude file types set", []string{"--exclude-file-types=json", "."}, 0},
{"multiple paths", []string{"../../test/fixtures/subdir/good.json", "../../test/fixtures/good.json"}, 0},
Expand All @@ -33,6 +34,7 @@ func Test_flags(t *testing.T) {
{"incorrect group", []string{"-groupby=badgroup", "."}, 1},
{"correct group", []string{"-groupby=directory", "."}, 0},
{"grouped junit", []string{"-groupby=directory", "--reporter=junit", "."}, 1},
{"grouped sarif", []string{"-groupby=directory", "--reporter=sarif", "."}, 1},
{"groupby duplicate", []string{"--groupby=directory,directory", "."}, 1},
{"quiet flag", []string{"--quiet=true", "."}, 0},
}
Expand Down
140 changes: 140 additions & 0 deletions pkg/reporter/reporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,40 @@ func Test_junitReport(t *testing.T) {
}
}

func Test_sarifReport(t *testing.T) {
reportNoValidationError := Report{
"good.xml",
"/fake/path/good.xml",
true,
nil,
false,
}

reportWithBackslashPath := Report{
"good.xml",
"\\fake\\path\\good.xml",
true,
nil,
false,
}

reportWithValidationError := Report{
"bad.xml",
"/fake/path/bad.xml",
false,
errors.New("Unable to parse bad.xml file"),
false,
}

reports := []Report{reportNoValidationError, reportWithValidationError, reportWithBackslashPath}

sarifReporter := SARIFReporter{}
err := sarifReporter.Print(reports)
if err != nil {
t.Errorf("Reporting failed")
}
}

func Test_jsonReporterWriter(t *testing.T) {
report := Report{
"good.json",
Expand Down Expand Up @@ -251,6 +285,112 @@ func Test_jsonReporterWriter(t *testing.T) {
}
}

func Test_sarifReporterWriter(t *testing.T) {
report := Report{
"good.json",
"test/output/example/good.json",
true,
nil,
false,
}
deleteFiles(t)

bytes, err := os.ReadFile("../../test/output/example/result.sarif")
require.NoError(t, err)

type args struct {
reports []Report
outputDest string
}
type want struct {
fileName string
data []byte
err assert.ErrorAssertionFunc
}

tests := map[string]struct {
args args
want want
}{
"normal/existing dir/default name": {
args: args{
reports: []Report{
report,
},
outputDest: "../../test/output",
},
want: want{
fileName: "result.sarif",
data: bytes,
err: assert.NoError,
},
},
"normal/file name is given": {
args: args{
reports: []Report{
report,
},
outputDest: "../../test/output/validator_result.sarif",
},
want: want{
fileName: "validator_result.sarif",
data: bytes,
err: assert.NoError,
},
},
"quash normal/empty string": {
args: args{
reports: []Report{
report,
},
outputDest: "",
},
want: want{
fileName: "",
data: nil,
err: assert.NoError,
},
},
"abnormal/non-existing dir": {
args: args{
reports: []Report{
report,
},
outputDest: "../../test/wrong/output",
},
want: want{
fileName: "",
data: nil,
err: assertRegexpError("failed to create a file: "),
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
sut := NewSARIFReporter(tt.args.outputDest)
err := sut.Print(tt.args.reports)
tt.want.err(t, err)
if tt.want.data != nil {
info, err := os.Stat(tt.args.outputDest)
require.NoError(t, err)
var filePath string
if info.IsDir() {
filePath = tt.args.outputDest + "/result.sarif"
} else { // if file was named with outputDest value
assert.Equal(t, tt.want.fileName, info.Name())
filePath = tt.args.outputDest
}
bytes, err := os.ReadFile(filePath)
require.NoError(t, err)
assert.Equal(t, tt.want.data, bytes)
err = os.Remove(filePath)
require.NoError(t, err)
}
},
)
}
}

func Test_JunitReporter_OutputBytesToFile(t *testing.T) {
report := Report{
"good.json",
Expand Down
133 changes: 133 additions & 0 deletions pkg/reporter/sarif_reporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package reporter

import (
"encoding/json"
"fmt"
"strings"
)

const SARIFVersion = "2.1.0"
const SARIFSchema = "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json"
const DriverName = "config-file-validator"
const DriverInfoURI = "https://github.com/Boeing/config-file-validator"
const DriverVersion = "1.7.1"

type SARIFReporter struct {
outputDest string
}

type SARIFLog struct {
Version string `json:"version"`
Schema string `json:"$schema"`
Runs []runs `json:"runs"`
}

type runs struct {
Tool tool `json:"tool"`
Results []result `json:"results"`
}

type tool struct {
Driver driver `json:"driver"`
}

type driver struct {
Name string `json:"name"`
InfoURI string `json:"informationUri"`
Version string `json:"version"`
}

type result struct {
Kind string `json:"kind"`
Level string `json:"level"`
Message message `json:"message"`
Locations []location `json:"locations"`
}

type message struct {
Text string `json:"text"`
}

type location struct {
PhysicalLocation physicalLocation `json:"physicalLocation"`
}

type physicalLocation struct {
ArtifactLocation artifactLocation `json:"artifactLocation"`
}

type artifactLocation struct {
URI string `json:"uri"`
}

func NewSARIFReporter(outputDest string) *SARIFReporter {
return &SARIFReporter{
outputDest: outputDest,
}
}

func createSARIFReport(reports []Report) (*SARIFLog, error) {
var log SARIFLog

n := len(reports)

log.Version = SARIFVersion
log.Schema = SARIFSchema

log.Runs = make([]runs, 1)
runs := &log.Runs[0]

runs.Tool.Driver.Name = DriverName
runs.Tool.Driver.InfoURI = DriverInfoURI
runs.Tool.Driver.Version = DriverVersion

runs.Results = make([]result, n)

for i, report := range reports {
if strings.Contains(report.FilePath, "\\") {
report.FilePath = strings.ReplaceAll(report.FilePath, "\\", "/")
}

result := &runs.Results[i]
if !report.IsValid {
result.Kind = "fail"
result.Level = "error"
result.Message.Text = report.ValidationError.Error()
} else {
result.Kind = "pass"
result.Level = "none"
result.Message.Text = "No errors detected"
}

result.Locations = make([]location, 1)
location := &result.Locations[0]

location.PhysicalLocation.ArtifactLocation.URI = "file:///" + report.FilePath
}

return &log, nil
}

func (sr SARIFReporter) Print(reports []Report) error {
report, err := createSARIFReport(reports)
if err != nil {
return err
}

sarifBytes, err := json.MarshalIndent(report, "", " ")
if err != nil {
return err
}

sarifBytes = append(sarifBytes, '\n')

if len(reports) > 0 && !reports[0].IsQuiet {
fmt.Print(string(sarifBytes))
}

if sr.outputDest != "" {
return outputBytesToFile(sr.outputDest, "result", "sarif", sarifBytes)
}

return nil
}
33 changes: 33 additions & 0 deletions test/output/example/result.sarif
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"version": "2.1.0",
"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
"runs": [
{
"tool": {
"driver": {
"name": "config-file-validator",
"informationUri": "https://github.com/Boeing/config-file-validator",
"version": "1.7.1"
}
},
"results": [
{
"kind": "pass",
"level": "none",
"message": {
"text": "No errors detected"
},
"locations": [
{
"physicalLocation": {
"artifactLocation": {
"uri": "file:///test/output/example/good.json"
}
}
}
]
}
]
}
]
}

0 comments on commit 5193c9c

Please sign in to comment.