decouplt unit from integration testing (keep-starknet-strange#296)

* decouplt unit from integration testing

* small refactoring of output stacktracke error

* readme integration test instruction

* fix tests to new version of zig build module

---------

Co-authored-by: lanaivina <[email protected]>

.github/workflows/test.yml

@@ -57,3 +57,19 @@ jobs:
         run: |
           zig version
           make test
+
+  integration_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Zig
+        uses: korandoru/setup-zig@v1
+        with:
+          zig-version: master
+
+      - name: Integration test
+        run: |
+          zig version
+          make build-integration-test
+          ./zig-out/bin/integration_test

Makefile

@@ -10,6 +10,9 @@ test: libstarknet_crypto.a
 test-filter: libstarknet_crypto.a
 	@zig build test --summary all -Dtest-filter="$(FILTER)"
 
+build-integration-test:
+	@zig build integration_test
+
 libstarknet_crypto.a:
 	@rm -f src/math/crypto/starknet_crypto/libstarknet_crypto.a
 	@cd src/math/crypto/starknet_crypto/starknet_crypto && cargo build --release

README.md

@@ -70,7 +70,13 @@ You can display the help message by running:
 
 ### 🧪 Testing
 
-Run all tests with test summary:
+Run all integration tests with summary:
+```bash
+make build-integration-test
+./zig-out/bin/integration_test
+```
+
+Run all unit tests with test summary:
 
 ```bash
 make test

build.zig

@@ -113,6 +113,8 @@ pub fn build(b: *std.Build) void {
         run_cmd.addArgs(args);
     }
 
+    integration_test(b, optimize, target);
+
     // This creates a build step. It will be visible in the `zig build --help` menu,
     // and can be selected like this: `zig build run`
     // This will evaluate the `run` step rather than the default, which is "install".
@@ -159,3 +161,22 @@ pub fn build(b: *std.Build) void {
     test_step.dependOn(&lib.step);
     test_step.dependOn(&run_unit_tests.step);
 }
+
+fn integration_test(
+    b: *std.Build,
+    mode: std.builtin.Mode,
+    target: std.Build.ResolvedTarget,
+) void {
+    const binary = b.addExecutable(.{
+        .name = "integration_test",
+        .root_source_file = .{ .path = "src/integration_tests.zig" },
+        .target = target,
+        .optimize = mode,
+    });
+
+    const integration_test_build = b.step("integration_test", "Build cli integration tests");
+    integration_test_build.dependOn(&binary.step);
+
+    const install_step = b.addInstallArtifact(binary, .{});
+    integration_test_build.dependOn(&install_step.step);
+}

src/integration_tests.zig

@@ -0,0 +1,90 @@
+const std = @import("std");
+const ProgramJson = @import("vm/types/programjson.zig").ProgramJson;
+const CairoVM = @import("vm/core.zig").CairoVM;
+const CairoRunner = @import("vm/runners/cairo_runner.zig").CairoRunner;
+
+pub fn main() void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    // Given
+    const allocator = gpa.allocator();
+
+    const cairo_programs: [3]struct {
+        pathname: []const u8,
+        layout: []const u8,
+    } = .{
+        .{ .pathname = "cairo_programs/factorial.json", .layout = "plain" },
+        .{ .pathname = "cairo_programs/fibonacci.json", .layout = "plain" },
+        .{ .pathname = "cairo_programs/bitwise_builtin_test.json", .layout = "all_cairo" },
+    };
+
+    var ok_count: usize = 0;
+    var fail_count: usize = 0;
+    var progress = std.Progress{
+        .dont_print_on_dumb = true,
+    };
+    const root_node = progress.start("Test", cairo_programs.len);
+    const have_tty = progress.terminal != null and
+        (progress.supports_ansi_escape_codes or progress.is_windows_terminal);
+
+    for (cairo_programs, 0..) |test_cairo_program, i| {
+        var test_node = root_node.start(test_cairo_program.pathname, 0);
+        test_node.activate();
+        progress.refresh();
+        if (!have_tty) {
+            std.debug.print("{d}/{d} {s}... \n", .{ i + 1, cairo_programs.len, test_cairo_program.pathname });
+        }
+        const result = cairo_run(allocator, test_cairo_program.pathname, test_cairo_program.layout);
+        if (result) |_| {
+            ok_count += 1;
+            test_node.end();
+            if (!have_tty) std.debug.print("OK\n", .{});
+        } else |err| {
+            fail_count += 1;
+            progress.log("FAIL ({s})\n", .{@errorName(err)});
+            if (@errorReturnTrace()) |trace| {
+                std.debug.dumpStackTrace(trace.*);
+            }
+            test_node.end();
+        }
+    }
+
+    root_node.end();
+
+    if (ok_count == cairo_programs.len) {
+        std.debug.print("All {d} tests passed.\n", .{ok_count});
+    } else {
+        std.debug.print("{d} passed; {d} failed.\n", .{ ok_count, fail_count });
+    }
+}
+
+pub fn cairo_run(allocator: std.mem.Allocator, pathname: []const u8, layout: []const u8) !void {
+    var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
+    const path = try std.os.realpath(pathname, &buffer);
+
+    var parsed_program = try ProgramJson.parseFromFile(allocator, path);
+    defer parsed_program.deinit();
+
+    const instructions = try parsed_program.value.readData(allocator);
+
+    const vm = try CairoVM.init(
+        allocator,
+        .{},
+    );
+
+    // when
+    var runner = try CairoRunner.init(
+        allocator,
+        parsed_program.value,
+        layout,
+        instructions,
+        vm,
+        false,
+    );
+    defer runner.deinit();
+    const end = try runner.setupExecutionState();
+    errdefer std.debug.print("failed on step: {}\n", .{runner.vm.current_step});
+
+    // then
+    try runner.runUntilPC(end);
+    try runner.endRun();
+}

src/vm/cairo_run.zig

@@ -162,109 +162,3 @@ test "EncodedMemory: can round trip from valid memory binary" {
     try std.testing.expectEqualSlices(u8, expected_file_bytes.items, actual_bytes.items);
     try tmp.dir.deleteFile(tmp_file_name);
 }
-
-test "Fibonacci: can evaluate without runtime error" {
-
-    // Given
-    const allocator = std.testing.allocator;
-    var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
-    const path = try std.os.realpath("cairo_programs/fibonacci.json", &buffer);
-
-    var parsed_program = try ProgramJson.parseFromFile(allocator, path);
-    defer parsed_program.deinit();
-
-    const instructions = try parsed_program.value.readData(allocator);
-
-    const vm = try CairoVM.init(
-        allocator,
-        .{},
-    );
-
-    // when
-    var runner = try CairoRunner.init(
-        allocator,
-        parsed_program.value,
-        "plain",
-        instructions,
-        vm,
-        false,
-    );
-    defer runner.deinit();
-    const end = try runner.setupExecutionState();
-    errdefer std.debug.print("failed on step: {}\n", .{runner.vm.current_step});
-
-    // then
-    try runner.runUntilPC(end);
-    try runner.endRun();
-}
-
-test "Factorial: can evaluate without runtime error" {
-
-    // Given
-    const allocator = std.testing.allocator;
-    var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
-    const path = try std.os.realpath("cairo_programs/factorial.json", &buffer);
-
-    var parsed_program = try ProgramJson.parseFromFile(allocator, path);
-    defer parsed_program.deinit();
-
-    const instructions = try parsed_program.value.readData(allocator);
-
-    const vm = try CairoVM.init(
-        allocator,
-        .{},
-    );
-
-    // when
-    var runner = try CairoRunner.init(
-        allocator,
-        parsed_program.value,
-        "plain",
-        instructions,
-        vm,
-        false,
-    );
-    defer runner.deinit();
-    const end = try runner.setupExecutionState();
-    errdefer std.debug.print("failed on step: {}\n", .{runner.vm.current_step});
-
-    // then
-    try runner.runUntilPC(end);
-    try runner.endRun();
-}
-
-test "Bitwise builtin test: can evaluate without runtime error" {
-
-    // Given
-    const allocator = std.testing.allocator;
-    var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
-    const path = try std.os.realpath("cairo_programs/bitwise_builtin_test.json", &buffer);
-
-    var parsed_program = try ProgramJson.parseFromFile(allocator, path);
-    defer parsed_program.deinit();
-
-    const instructions = try parsed_program.value.readData(allocator);
-
-    const vm = try CairoVM.init(
-        allocator,
-        .{},
-    );
-
-    // when
-    var runner = try CairoRunner.init(
-        allocator,
-        parsed_program.value,
-        "all_cairo",
-        instructions,
-        vm,
-        false,
-    );
-    defer runner.deinit();
-    const end = try runner.setupExecutionState();
-
-    errdefer std.debug.print("failed on step: {}\n", .{runner.vm.current_step});
-
-    // then
-    try runner.runUntilPC(end);
-    try runner.endRun();
-}