Tyche logging support

fuzz/pom.xml

@@ -55,6 +55,12 @@
             <artifactId>eclipse-collections</artifactId>
             <version>10.4.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.15.2</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
 

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/afl/AFLGuidance.java

@@ -167,6 +167,11 @@ public void finalize() {
         }
     }
 
+    @Override
+    public String observeGuidance() {
+        return "AFL";
+    }
+
 
     /**
      * Returns an input stream containing the bytes that AFL

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/ei/ZestGuidance.java

@@ -437,6 +437,14 @@ protected void appendLineToFile(File file, String line) throws GuidanceException
 
     }
 
+    @Override
+    public String observeGuidance() {
+        if (blind) {
+            return "Random";
+        }
+        return "Zest";
+    }
+
     /* Writes a line of text to the log file. */
     protected void infoLog(String str, Object... args) {
         if (verbose) {

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/guidance/Guidance.java

@@ -132,6 +132,14 @@ default void observeGeneratedArgs(Object[] args) {
         // Do nothing
     }
 
+    /**
+     * <p>This method is only invoked during optional observability
+     * logging to provide the type of guidance used for input generation.</p>
+     */
+    default String observeGuidance() {
+        return "";
+    }
+
     /**
      * Handles the end of a fuzzing trial.
      *

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java

@@ -41,13 +41,17 @@
 import com.pholser.junit.quickcheck.internal.ParameterTypeContext;
 import com.pholser.junit.quickcheck.internal.generator.GeneratorRepository;
 import com.pholser.junit.quickcheck.random.SourceOfRandomness;
+import edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance;
 import edu.berkeley.cs.jqf.fuzz.guidance.Guidance;
 import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException;
 import edu.berkeley.cs.jqf.fuzz.guidance.TimeoutException;
 import edu.berkeley.cs.jqf.fuzz.guidance.Result;
 import edu.berkeley.cs.jqf.fuzz.guidance.StreamBackedRandom;
 import edu.berkeley.cs.jqf.fuzz.junit.TrialRunner;
+import edu.berkeley.cs.jqf.fuzz.random.NoGuidance;
+import edu.berkeley.cs.jqf.fuzz.repro.ReproGuidance;
 import edu.berkeley.cs.jqf.instrument.InstrumentationException;
+import edu.berkeley.cs.jqf.fuzz.util.Observability;
 import org.junit.AssumptionViolatedException;
 import org.junit.runners.model.FrameworkMethod;
 import org.junit.runners.model.MultipleFailureException;
@@ -73,6 +77,7 @@ public class FuzzStatement extends Statement {
     private final List<Class<?>> expectedExceptions;
     private final List<Throwable> failures = new ArrayList<>();
     private final Guidance guidance;
+    private final Observability observability;
     private boolean skipExceptionSwallow;
 
     public FuzzStatement(FrameworkMethod method, TestClass testClass,
@@ -85,6 +90,7 @@ public FuzzStatement(FrameworkMethod method, TestClass testClass,
         this.expectedExceptions = Arrays.asList(method.getMethod().getExceptionTypes());
         this.guidance = fuzzGuidance;
         this.skipExceptionSwallow = Boolean.getBoolean("jqf.failOnDeclaredExceptions");
+        this.observability = new Observability(testClass.getName(), method.getName(), System.currentTimeMillis());
     }
 
     /**
@@ -101,16 +107,19 @@ public void evaluate() throws Throwable {
                 .collect(Collectors.toList());
 
         // Keep fuzzing until no more input or I/O error with guidance
+        // Get current time in unix timestamp
+        long endGenerationTime = 0;
         try {
 
             // Keep fuzzing as long as guidance wants to
             while (guidance.hasInput()) {
                 Result result = INVALID;
                 Throwable error = null;
+                long startTrialTime = System.currentTimeMillis();
 
                 // Initialize guided fuzzing using a file-backed random number source
+                Object [] args = {};
                 try {
-                    Object[] args;
                     try {
 
                         // Generate input values
@@ -142,6 +151,7 @@ public void evaluate() throws Throwable {
                         throw new GuidanceException(e);
                     }
 
+                    endGenerationTime = System.currentTimeMillis();
                     // Attempt to run the trial
                     guidance.run(testClass, method, args);
 
@@ -170,6 +180,17 @@ public void evaluate() throws Throwable {
                         failures.add(e);
                     }
                 }
+                long endTrialTime = System.currentTimeMillis();
+                if (System.getProperty("jqfObservability") != null) {
+                    observability.addStatus(result);
+                    if (result == SUCCESS) {
+                        observability.addTiming(startTrialTime, endGenerationTime, endTrialTime);
+                    }
+                    observability.addArgs(args);
+                    observability.add("how_generated", guidance.observeGuidance());
+
+                    observability.writeToFile();
+                }
 
                 // Inform guidance about the outcome of this trial
                 try {
@@ -180,8 +201,6 @@ public void evaluate() throws Throwable {
                     // Anything else thrown from handleResult is an internal error, so wrap
                     throw new GuidanceException(e);
                 }
-
-
             }
         } catch (GuidanceException e) {
             System.err.println("Fuzzing stopped due to guidance exception: " + e.getMessage());
@@ -198,6 +217,7 @@ public void evaluate() throws Throwable {
             }
         }
 
+
     }
 
     /**

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/repro/ReproGuidance.java

@@ -191,6 +191,11 @@ public void observeGeneratedArgs(Object[] args) {
         }
     }
 
+    @Override
+    public String observeGuidance() {
+        return "Repro";
+    }
+
     /**
      * Writes an object to a file
      *

fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java

@@ -0,0 +1,110 @@
+package edu.berkeley.cs.jqf.fuzz.util;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import edu.berkeley.cs.jqf.fuzz.guidance.Result;
+
+import java.io.FileWriter;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+import static edu.berkeley.cs.jqf.fuzz.guidance.Result.FAILURE;
+import static edu.berkeley.cs.jqf.fuzz.guidance.Result.INVALID;
+
+public class Observability {
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private final String testClass;
+    private final String testMethod;
+    private final Path obsPath;
+    private static ObjectNode testCaseJsonObject;
+    private final long startTime;
+
+    public Observability(String testClass, String testMethod, long startTime) {
+        this.testClass = testClass;
+        this.testMethod = testMethod;
+        this.obsPath = Paths.get("target", "fuzz-results", testClass, testMethod, "observations.jsonl");
+        if (obsPath.toFile().exists()) {
+            obsPath.toFile().delete();
+        }
+        this.startTime = startTime;
+        this.initializeTestCase();
+    }
+
+    public void initializeTestCase() {
+        testCaseJsonObject = objectMapper.createObjectNode();
+        testCaseJsonObject.putObject("features");
+        testCaseJsonObject.putObject("timing");
+        testCaseJsonObject.putObject("coverage");
+        testCaseJsonObject.putObject("args");
+        testCaseJsonObject.putObject("metadata");
+        testCaseJsonObject.put("type", "test_case");
+        testCaseJsonObject.put("run_start", startTime);
+        testCaseJsonObject.put("property", testMethod);
+    }
+
+    public static void event(String value, Object payload) throws RuntimeException {
+        // Add the payload to the features object
+        JsonNode jsonFeaturesNode = testCaseJsonObject.get("features");
+        ObjectNode featuresNode = (ObjectNode) jsonFeaturesNode;
+
+        if (payload instanceof Integer) {
+            featuresNode.put(value, (Integer) payload);
+        } else if (payload instanceof String) {
+            featuresNode.put(value, (String) payload);
+        } else if (payload instanceof Float) {
+            featuresNode.put(value, (Float) payload);
+        } else {
+            throw new RuntimeException("Unsupported payload type for event");
+        }
+    }
+
+    public void addStatus(Result result) {
+        if (result == INVALID) {
+            testCaseJsonObject.put("status", "gave_up");
+            testCaseJsonObject.put("status_reason", "assumption violated");
+        } else if (result == FAILURE) {
+            testCaseJsonObject.put("status", "failed");
+            testCaseJsonObject.put("status_reason", "Encountered exception");
+        } else {
+            testCaseJsonObject.put("status", "passed");
+            testCaseJsonObject.put("status_reason", "");
+        }
+
+    }
+
+    public void addTiming(long startTime, long endGenerationTime, long endExecutionTime) {
+        JsonNode timingNode = testCaseJsonObject.get("timing");
+        ObjectNode timingObject = (ObjectNode) timingNode;
+        timingObject.put("generation", endGenerationTime - startTime);
+        timingObject.put("execution", endExecutionTime - endGenerationTime);
+    }
+
+    public void addArgs(Object[] args) {
+        JsonNode argsNode = testCaseJsonObject.get("args");
+        ObjectNode argsObject = (ObjectNode) argsNode;
+        for (int i = 0; i < args.length; i++) {
+            argsObject.put("arg" + i, args[i].toString());
+        }
+        add("representation", Arrays.toString(args));
+    }
+
+    public void add(String key, String value) {
+        testCaseJsonObject.put(key, value);
+    }
+
+    public void writeToFile() {
+        // Append the JSON object to a file followed by a newline
+        try {
+            String jsonString = objectMapper.writeValueAsString(testCaseJsonObject);
+            try (FileWriter writer = new FileWriter(obsPath.toFile(), true)) {
+                writer.write(jsonString);
+                writer.write(System.lineSeparator()); // Add a new line after each object
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to write observations to file", e);
+        }
+    }
+}