Merge branch 'add_environment_variable_baits' of https://github.com/e…

…lainechien/package-analysis into add_environment_variable_baits

.github/workflows/codeql-analysis.yml

@@ -36,6 +36,11 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+    - name: setup-go
+      uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
+      with:
+        go-version: '1.21.0'
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4

.github/workflows/depsreview.yml

@@ -15,4 +15,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 # v3.1.0
+        uses: actions/dependency-review-action@7bbfa034e752445ea40215fff1c3bf9597993d3f # v3.1.3

.github/workflows/osv-scanner-pr.yml

@@ -0,0 +1,19 @@
+name: OSV-Scanner PR Scan
+
+# Change "main" to your default branch if you use a different name, i.e. "master"
+on:
+  pull_request:
+    branches: [ main ]
+  merge_group:
+    branches: [ main ]
+
+# Declare default permissions as read only.
+permissions:
+  # Only need to read contents
+  contents: read
+  # Require writing security events to upload SARIF file to security tab
+  security-events: write
+
+jobs:
+  scan-pr:
+    uses: "google/osv-scanner/.github/workflows/osv-scanner-reusable-pr.yml@main"

.github/workflows/osv-scanner-scheduled.yml

@@ -0,0 +1,18 @@
+name: OSV-Scanner Scheduled Scan
+
+on:
+  schedule:
+    - cron: '50 1 * * 6' # run at 01:50 UTC every Saturday
+# Change "main" to your default branch if you use a different name, i.e. "master"
+  push:
+    branches: [ main ]
+
+permissions:
+  # Only need to read contents
+  contents: read
+  # Require writing security events to upload SARIF file to security tab
+  security-events: write
+
+jobs:
+  scan-scheduled:
+    uses: "google/osv-scanner/.github/workflows/osv-scanner-reusable.yml@main"

Makefile

@@ -129,11 +129,16 @@ run:
 
 E2E_TEST_COMPOSE_ARGS := -p pa-e2e-testing -f ./configs/e2e/docker-compose.yml -f ./test/e2e/docker-compose.test.yml
 
+.PHONY: e2e_test_build
+e2e_test_build: build_e2e_test_images
+
 .PHONY: e2e_test_start
 e2e_test_start:
 	docker-compose $(E2E_TEST_COMPOSE_ARGS) up -d
 	@echo
 	@echo "To see analysis results, go to http://localhost:9000/minio/package-analysis"
+	@echo "Username: minio"
+	@echo "Password: minio123"
 	@echo
 	@echo "Remember to run 'make e2e_test_stop' when done!"
 	@sleep 5
@@ -160,6 +165,13 @@ e2e_test_logs_scheduler:
 e2e_test_logs_analysis:
 	docker-compose $(E2E_TEST_COMPOSE_ARGS) logs -f analysis
 
+
+.PHONY: build_e2e_test_images
+build_e2e_test_images: TAG=test
+build_e2e_test_images: sync_prod_sandboxes build_analysis_image build_scheduler_image
+
+
+
 .PHONY: test_go
 test_go:
 	go test -v ./...

cmd/analyze/main.go

@@ -24,42 +24,45 @@ import (
 )
 
 var (
-	pkgName             = flag.String("package", "", "package name")
-	localPkg            = flag.String("local", "", "local package path")
-	ecosystem           pkgecosystem.Ecosystem
-	version             = flag.String("version", "", "version")
-	noPull              = flag.Bool("nopull", false, "disables pulling down sandbox images")
-	imageTag            = flag.String("image-tag", "", "set image tag for analysis sandboxes")
-	dynamicUpload       = flag.String("upload", "", "bucket path for uploading dynamic analysis results")
-	staticUpload        = flag.String("upload-static", "", "bucket path for uploading static analysis results")
-	uploadFileWriteInfo = flag.String("upload-file-write-info", "", "bucket path for uploading information from file writes")
-	uploadAnalyzedPkg   = flag.String("upload-analyzed-pkg", "", "bucket path for uploading analyzed packages")
-	offline             = flag.Bool("offline", false, "disables sandbox network access")
-	customSandbox       = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image")
-	customAnalysisCmd   = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)")
-	listModes           = flag.Bool("list-modes", false, "prints out a list of available analysis modes")
-	features            = flag.String("features", "", "override features that are enabled/disabled by default")
-	listFeatures        = flag.Bool("list-features", false, "list available features that can be toggled")
-	help                = flag.Bool("help", false, "print help on available options")
-	analysisMode        = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"},
+	pkgName            = flag.String("package", "", "package name")
+	localPkg           = flag.String("local", "", "local package path")
+	ecosystem          pkgecosystem.Ecosystem
+	version            = flag.String("version", "", "version")
+	noPull             = flag.Bool("nopull", false, "disables pulling down sandbox images")
+	imageTag           = flag.String("image-tag", "", "set image tag for analysis sandboxes")
+	dynamicBucket      = flag.String("dynamic-bucket", "", "bucket path for uploading dynamic analysis results")
+	staticBucket       = flag.String("static-bucket", "", "bucket path for uploading static analysis results")
+	executionLogBucket = flag.String("execution-log-bucket", "", "bucket path for uploading execution log (dynamic analysis)")
+	fileWritesBucket   = flag.String("file-writes-bucket", "", "bucket path for uploading file writes data (dynamic analysis)")
+	analyzedPkgBucket  = flag.String("analyzed-pkg-bucket", "", "bucket path for uploading analyzed packages")
+	offline            = flag.Bool("offline", false, "disables sandbox network access")
+	customSandbox      = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image")
+	customAnalysisCmd  = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)")
+	listModes          = flag.Bool("list-modes", false, "prints out a list of available analysis modes")
+	features           = flag.String("features", "", "override features that are enabled/disabled by default")
+	listFeatures       = flag.Bool("list-features", false, "list available features that can be toggled")
+	help               = flag.Bool("help", false, "print help on available options")
+	analysisMode       = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"},
 		"list of analysis modes to run, separated by commas. Use -list-modes to see available options")
 )
 
 func makeResultStores() worker.ResultStores {
 	rs := worker.ResultStores{}
 
-	if *dynamicUpload != "" {
-		rs.DynamicAnalysis = resultstore.New(*dynamicUpload)
+	if *analyzedPkgBucket != "" {
+		rs.AnalyzedPackage = resultstore.New(*analyzedPkgBucket)
 	}
-	if *staticUpload != "" {
-		rs.StaticAnalysis = resultstore.New(*staticUpload)
+	if *dynamicBucket != "" {
+		rs.DynamicAnalysis = resultstore.New(*dynamicBucket)
 	}
-	if *uploadFileWriteInfo != "" {
-		rs.FileWrites = resultstore.New(*uploadFileWriteInfo)
+	if *executionLogBucket != "" {
+		rs.ExecutionLog = resultstore.New(*executionLogBucket)
 	}
-
-	if *uploadAnalyzedPkg != "" {
-		rs.AnalyzedPackage = resultstore.New(*uploadAnalyzedPkg)
+	if *fileWritesBucket != "" {
+		rs.FileWrites = resultstore.New(*fileWritesBucket)
+	}
+	if *staticBucket != "" {
+		rs.StaticAnalysis = resultstore.New(*staticBucket)
 	}
 
 	return rs

cmd/worker/main.go

@@ -37,10 +37,11 @@ const (
 
 // resultBucketPaths holds bucket paths for the different types of results.
 type resultBucketPaths struct {
+	analyzedPkg     string
 	dynamicAnalysis string
-	staticAnalysis  string
+	executionLog    string
 	fileWrites      string
-	analyzedPkg     string
+	staticAnalysis  string
 }
 
 type sandboxImageSpec struct {
@@ -79,17 +80,20 @@ func copyPackageToLocalFile(ctx context.Context, packagesBucket *blob.Bucket, bu
 func makeResultStores(dest resultBucketPaths) worker.ResultStores {
 	resultStores := worker.ResultStores{}
 
+	if dest.analyzedPkg != "" {
+		resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath())
+	}
 	if dest.dynamicAnalysis != "" {
 		resultStores.DynamicAnalysis = resultstore.New(dest.dynamicAnalysis, resultstore.ConstructPath())
 	}
-	if dest.staticAnalysis != "" {
-		resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath())
+	if dest.executionLog != "" {
+		resultStores.ExecutionLog = resultstore.New(dest.executionLog, resultstore.ConstructPath())
 	}
 	if dest.fileWrites != "" {
 		resultStores.FileWrites = resultstore.New(dest.fileWrites, resultstore.ConstructPath())
 	}
-	if dest.analyzedPkg != "" {
-		resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath())
+	if dest.staticAnalysis != "" {
+		resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath())
 	}
 
 	return resultStores
@@ -274,10 +278,11 @@ func main() {
 	}
 
 	resultsBuckets := resultBucketPaths{
+		analyzedPkg:     os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"),
 		dynamicAnalysis: os.Getenv("OSSF_MALWARE_ANALYSIS_RESULTS"),
-		staticAnalysis:  os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"),
+		executionLog:    os.Getenv("OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS"),
 		fileWrites:      os.Getenv("OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS"),
-		analyzedPkg:     os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"),
+		staticAnalysis:  os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"),
 	}
 	resultStores := makeResultStores(resultsBuckets)
 
@@ -305,6 +310,7 @@ func main() {
 		"static_results_bucket", resultsBuckets.staticAnalysis,
 		"file_write_results_bucket", resultsBuckets.fileWrites,
 		"analyzed_packages_bucket", resultsBuckets.analyzedPkg,
+		"execution_log_bucket", resultsBuckets.executionLog,
 		"image_tag", imageSpec.tag,
 		"image_nopull", imageSpec.noPull,
 		"topic_notification", notificationTopicURL,

configs/e2e/docker-compose.yml

@@ -47,7 +47,7 @@ services:
       MINIO_ROOT_PASSWORD: minio123
       MINIO_REGION_NAME: dummy_region
     entrypoint: sh
-    command: -c 'mkdir -p /data/package-analysis{,-static,-file-writes,-analyzed-packages} && /usr/bin/minio server /data'
+    command: -c 'mkdir -p /data/package-analysis/{analyzed-packages,dynamic,execution-logs,file-writes,static} && /usr/bin/minio server /data'
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -64,10 +64,11 @@ services:
     environment:
       OSSMALWARE_WORKER_SUBSCRIPTION: kafka://worker?topic=workers
       OSSF_MALWARE_NOTIFICATION_TOPIC: kafka://notifications
-      OSSF_MALWARE_ANALYSIS_RESULTS: s3://package-analysis?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
-      OSSF_MALWARE_STATIC_ANALYSIS_RESULTS: s3://package-analysis-static?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
-      OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS: s3://package-analysis-file-writes?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
-      OSSF_MALWARE_ANALYZED_PACKAGES: s3://package-analysis-analyzed-packages?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYZED_PACKAGES: s3://package-analysis/analyzed-packages?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYSIS_RESULTS: s3://package-analysis/dynamic?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS: s3://package-analysis/execution-logs?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS: s3://package-analysis/file-writes?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_STATIC_ANALYSIS_RESULTS: s3://package-analysis/static?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
       OSSF_MALWARE_ANALYSIS_ENABLE_PROFILER: "true"
       OSSF_MALWARE_FEATURE_FLAGS: ""
       KAFKA_BROKERS: kafka:9092