[WIP] Add a managed assembly loader and static method interop mechanism

.github/workflows/master.yml

@@ -35,6 +35,9 @@ jobs:
       - uses: actions/setup-dotnet@v1
         with:
           dotnet-version: ${{ matrix.dot-net-version }}
+      - name: Build .NET lib
+        run: dotnet build
+        working-directory: pyjion-lib
 
       - name: Create Build Environment
         run: cmake -E make_directory ${{runner.workspace}}/build
@@ -57,7 +60,8 @@ jobs:
         working-directory: ${{runner.workspace}}/build
         shell: bash
         run: ./unit_tests --out test.log -s --use-colour yes
-
+        env:
+          DOTNET_VERSION: ${{ matrix.dot-net-version }}
       - name: Setup Python tests
         shell: bash
         run: pip install -r Tests/requirements_test.txt
@@ -159,6 +163,10 @@ jobs:
       - uses: actions/setup-dotnet@v1
         with:
           dotnet-version: ${{ matrix.dot-net-version }}
+      - name: Build .NET lib
+        run: dotnet build
+        working-directory: pyjion-lib
+
       - name: Create Build Environment
         run: cmake -E make_directory ${{runner.workspace}}/build
 
@@ -180,7 +188,9 @@ jobs:
         working-directory: ${{runner.workspace}}/build
         shell: bash
         run: ./unit_tests --out test.log -s --use-colour yes
-
+        env:
+          DOTNET_VERSION: ${{ matrix.dot-net-version }}
+
       - name: Run gcov
         shell: bash
         run: gcov Tests/
@@ -266,7 +276,9 @@ jobs:
       - uses: actions/setup-dotnet@v1
         with:
           dotnet-version: ${{ matrix.dot-net-version }}
-
+      - name: Build .NET lib
+        run: dotnet build
+        working-directory: pyjion-lib
       - name: Add msbuild to PATH
         uses: microsoft/[email protected]
 
@@ -310,9 +322,9 @@ jobs:
 
       - name: Run C++ unit tests
         working-directory: ${{runner.workspace}}/build/Release
-        run: |
-          cp c:\Users\runneradmin\AppData\Local\Microsoft\dotnet\shared\Microsoft.NETCore.App\6.0.0-rc.2.21480.5\clrjit.dll .
-          .\unit_tests.exe
+        run: .\unit_tests.exe
+        env:
+          DOTNET_VERSION: ${{ matrix.dot-net-version }}
 
       - name: Setup Python tests
         shell: bash

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "Tests/Catch"]
 	path = Tests/Catch
 	url = https://github.com/philsquared/Catch.git
+[submodule "pe-parse"]
+	path = pe-parse
+	url = [email protected]:trailofbits/pe-parse.git

CMakeLists.txt

@@ -82,6 +82,7 @@ if(REPORT_CLR_FAULTS)
     add_definitions(-DREPORT_CLR_FAULTS)
 endif()
 if(ASAN)
+    message(STATUS "Enabling Address Sanitizer")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -g")
 endif()
 if(NOT WIN32)
@@ -152,7 +153,7 @@ else()
     message(STATUS "Enabling x86")
 endif()
 
-include_directories(CoreCLR/src/coreclr/inc CoreCLR/src/coreclr/jit)
+include_directories(CoreCLR/src/coreclr/inc CoreCLR/src/coreclr/jit CoreCLR/src/coreclr/vm CoreCLR/src/native/corehost CoreCLR/src/native/corehost/nethost pe-parse/pe-parser-library/include)
 
 if (UNIX)
     if(DEFINED ENV{DOTNET_ROOT})
@@ -195,6 +196,8 @@ if (UNIX AND NOT APPLE)
     set(CLR_JIT_LIB "libclrjit.so")
     set(CLR_CMAKE_HOST_UNIX 1)
     add_compile_options(-fPIC)
+    add_definitions(-DHOST_UNIX)
+    add_definitions(-DHOST_LINUX)
     add_definitions(-DINDIRECT_HELPERS)
 endif()
 
@@ -212,6 +215,8 @@ if (APPLE)
     add_definitions(-D_XOPEN_SOURCE)
     add_definitions(-DTARGET_OSX)
     add_definitions(-DHOST_OSX)
+    add_definitions(-DHOST_DARWIN)
+    add_definitions(-DHOST_UNIX)
     add_compile_options(-fdeclspec)
 endif()
 
@@ -221,7 +226,7 @@ if(NET_SRC_LIB)  # use the libraries compiled from source
     message(STATUS "Using .NET builds " ${DOTNETPATH})
 endif()
 
-set(SOURCES src/pyjion/absint.cpp src/pyjion/absvalue.cpp src/pyjion/intrins.cpp src/pyjion/jitinit.cpp src/pyjion/pycomp.cpp src/pyjion/pyjit.cpp src/pyjion/exceptionhandling.cpp src/pyjion/stack.cpp src/pyjion/codemodel.cpp src/pyjion/binarycomp.cpp src/pyjion/instructions.cpp src/pyjion/unboxing.cpp src/pyjion/frame.h src/pyjion/pgc.cpp src/pyjion/base.cpp src/pyjion/objects/unboxedrangeobject.cpp)
+set(SOURCES src/pyjion/absint.cpp src/pyjion/absvalue.cpp src/pyjion/intrins.cpp src/pyjion/jitinit.cpp src/pyjion/pycomp.cpp src/pyjion/pyjit.cpp src/pyjion/exceptionhandling.cpp src/pyjion/stack.cpp src/pyjion/codemodel.cpp src/pyjion/binarycomp.cpp src/pyjion/instructions.cpp src/pyjion/unboxing.cpp src/pyjion/frame.h src/pyjion/pgc.cpp src/pyjion/base.cpp src/pyjion/objects/unboxedrangeobject.cpp src/pyjion/peloader.cpp src/pyjion/objects/assemblyobject.cpp src/pyjion/corehost.cpp)
 
 if (WIN32)
     enable_language(ASM_MASM)
@@ -248,6 +253,23 @@ else()
     endif()
 endif()
 
+list(APPEND PEPARSERLIB_SOURCEFILES
+        pe-parse/pe-parser-library/include/pe-parse/parse.h
+        pe-parse/pe-parser-library/include/pe-parse/nt-headers.h
+        pe-parse/pe-parser-library/include/pe-parse/to_string.h
+
+        pe-parse/pe-parser-library/src/buffer.cpp
+        pe-parse/pe-parser-library/src/parse.cpp
+        )
+
+if(MSVC)
+    list(APPEND PEPARSERLIB_SOURCEFILES pe-parse/pe-parser-library/src/unicode_winapi.cpp)
+else()
+    list(APPEND PEPARSERLIB_SOURCEFILES pe-parse/pe-parser-library/src/unicode_codecvt.cpp)
+endif()
+
+set(SOURCES ${SOURCES} ${PEPARSERLIB_SOURCEFILES})
+message(STATUS "Building " ${SOURCES})
 add_library(pyjionlib OBJECT ${SOURCES})
 add_library(_pyjion MODULE $<TARGET_OBJECTS:pyjionlib>)
 
@@ -260,6 +282,13 @@ if (WIN32)
     )
 endif(WIN32)
 
+add_custom_command(
+        TARGET _pyjion POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+        ${CMAKE_SOURCE_DIR}/pyjion.runtimeconfig.json
+        ${CMAKE_CURRENT_BINARY_DIR}/
+)
+
 set_target_properties(
         _pyjion
         PROPERTIES
@@ -275,7 +304,7 @@ endif()
 if (BUILD_TESTS)
     # Testing
     add_subdirectory(Tests/Catch)
-    set(TEST_SOURCES Tests/testing_util.cpp Tests/test_basics.cpp Tests/test_compiler.cpp Tests/Tests.cpp Tests/test_wrappers.cpp Tests/test_exceptions.cpp Tests/test_scopes.cpp Tests/test_tracing.cpp Tests/test_inference.cpp Tests/test_math.cpp Tests/test_pgc.cpp Tests/test_unpack.cpp Tests/test_class.cpp Tests/test_coro.cpp Tests/test_graph.cpp Tests/test_big_build.cpp Tests/test_ilgen.cpp Tests/test_with.cpp Tests/test_containers.cpp Tests/test_bigint.cpp Tests/test_globals.cpp)
+    set(TEST_SOURCES Tests/testing_util.cpp Tests/test_basics.cpp Tests/test_compiler.cpp Tests/Tests.cpp Tests/test_wrappers.cpp Tests/test_exceptions.cpp Tests/test_scopes.cpp Tests/test_tracing.cpp Tests/test_inference.cpp Tests/test_math.cpp Tests/test_pgc.cpp Tests/test_unpack.cpp Tests/test_class.cpp Tests/test_coro.cpp Tests/test_graph.cpp Tests/test_big_build.cpp Tests/test_ilgen.cpp Tests/test_with.cpp Tests/test_containers.cpp Tests/test_bigint.cpp Tests/test_globals.cpp Tests/test_loader.cpp)
 
     add_executable(unit_tests ${TEST_SOURCES} $<TARGET_OBJECTS:pyjionlib>)
     if (NOT WIN32)
@@ -285,7 +314,6 @@ if (BUILD_TESTS)
     target_include_directories(unit_tests PRIVATE src/pyjion)
     target_link_libraries(unit_tests Catch2::Catch2)
     target_link_libraries(unit_tests ${Python3_LIBRARIES})
-
     if (NOT WIN32)
         target_link_libraries(unit_tests ${DOTNETPATH}/${CLR_JIT_LIB})
     endif()

Tests/Tests.cpp

@@ -32,11 +32,13 @@
 int main(int argc, char* const argv[]) {
     Py_Initialize();
     PyjionUnboxingError = PyErr_NewException("pyjion.PyjionUnboxingError", PyExc_ValueError, nullptr);
-#ifdef WINDOWS
-    JitInit(L"clrjit.dll");
-#else
-    JitInit(L"libclrjit.so");
-#endif
+    auto dotnetroot = getenv("DOTNET_ROOT");
+    auto version = getenv("DOTNET_VERSION");
+    if (dotnetroot == nullptr || version == nullptr){
+        printf("Test suite requires both DOTNET_ROOT and DOTNET_VERSION environment variables to be set");
+        exit(1);
+    }
+    JitInit(dotnetroot, version);
     g_pyjionSettings.graph = true;
     g_pyjionSettings.debug = true;
     g_pyjionSettings.codeObjectSizeLimit = 1000000;

Tests/test_loader.cpp

@@ -0,0 +1,202 @@
+/*
+* The MIT License (MIT)
+*
+* Copyright (c) Microsoft Corporation
+*
+* Permission is hereby granted, free of charge, to any person obtaining a
+* copy of this software and associated documentation files (the "Software"),
+* to deal in the Software without restriction, including without limitation
+* the rights to use, copy, modify, merge, publish, distribute, sublicense,
+* and/or sell copies of the Software, and to permit persons to whom the
+* Software is furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included
+* in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+* OTHER DEALINGS IN THE SOFTWARE.
+*
+*/
+
+#include <catch2/catch.hpp>
+#include "peloader.h"
+#include "corehost.h"
+#include "error_codes.h"
+#include <filesystem>
+
+TEST_CASE("Test basic loader") {
+    SECTION("Test builtin R2R image") {
+        auto dotnetroot = getenv("DOTNET_ROOT");
+        auto version = getenv("DOTNET_VERSION");
+        std::filesystem::path console_path = ( std::filesystem::path(dotnetroot) / std::filesystem::path("shared/Microsoft.NETCore.App/") / std::filesystem::path(version) / std::filesystem::path("System.Console.dll"));
+        PEDecoder decoder = PEDecoder(console_path.c_str());
+        CHECK(decoder.GetCorHeader()->Flags & COMIMAGE_FLAGS_IL_LIBRARY);
+        CHECK(decoder.GetReadyToRunHeader()->MajorVersion == 5);
+        CHECK(decoder.GetModuleName() == "System.Console.dll");
+        auto typeRefs = decoder.GetTypeRefs();
+        CHECK(typeRefs.size() == 125);
+        CHECK(typeRefs[124].Name == 6055);
+        auto typeDefs = decoder.GetTypeDefs();
+        CHECK(typeDefs.size() == 53);
+        CHECK(typeDefs[0].Flags == 0);
+        CHECK(typeDefs[0].Name == 0x0577);
+        CHECK(typeDefs[0].Namespace == 0);
+        CHECK(typeDefs[0].Extends == 0);
+        CHECK(typeDefs[0].FieldList == 1);
+        CHECK(typeDefs[0].MethodList == 1);
+
+        CHECK(typeDefs[1].Flags == 0x00100100);
+        CHECK(typeDefs[1].Name == 0x1523);
+        CHECK(typeDefs[1].Namespace == 0x2e74);
+        CHECK(typeDefs[1].Extends == 5);
+        CHECK(typeDefs[1].FieldList == 1);
+        CHECK(typeDefs[1].MethodList == 1);
+        // ...
+        CHECK(typeDefs[40].Name == 2380);
+
+        auto publicTypeDefs = decoder.GetPublicClasses();
+        CHECK(publicTypeDefs.size() == 8);
+        CHECK(decoder.GetString(publicTypeDefs[0].Name) == "Console");
+        auto methods = decoder.GetClassMethods(publicTypeDefs[0]);
+        CHECK(methods.size() == 59);
+        CHECK(decoder.GetString(methods[0].Name) == "ReadKey");
+        CHECK(decoder.GetString(methods[58].Name) == "Write");
+
+        auto signature = decoder.GetBlob(methods[0].Signature);
+        CHECK(signature.size() == 4);
+        CHECK(!decoder.GetMethodParams(methods[58]).empty());
+    }
+
+    SECTION("Test method execution"){
+        std::filesystem::path libPath = filesystem::path("../pyjion-lib/bin/Debug/Pyjionlib.dll");
+        REQUIRE(std::filesystem::exists(libPath));
+        CHECK(load_hostfxr());
+        auto t = get_dotnet_load_assembly();
+        REQUIRE(t != nullptr);
+
+        component_entry_point_fn hello = nullptr;
+        int ret = t(
+                libPath.c_str(),
+                "Pyjion.Test, Pyjionlib",
+                "Hello",
+                nullptr /*delegate_type_name*/,
+                nullptr,
+                (void**) &hello);
+        REQUIRE(ret == Success);
+        REQUIRE(hello != nullptr);
+
+        struct lib_args
+        {
+            const char_t *message;
+            int number;
+        };
+        for (int i = 0; i < 3; ++i)
+        {
+            lib_args args
+                    {
+                        "from host!",
+                        i
+                    };
+
+            CHECK(hello(&args, sizeof(args)) == 0);
+        }
+    }
+
+    SECTION("Test delegate method execution"){
+        std::filesystem::path libPath = filesystem::path("../pyjion-lib/bin/Debug/Pyjionlib.dll");
+        REQUIRE(std::filesystem::exists(libPath));
+        CHECK(load_hostfxr());
+        auto t = get_dotnet_load_assembly();
+        REQUIRE(t != nullptr);
+        typedef int (CORECLR_DELEGATE_CALLTYPE *custom_entry_point_fn)(int, int);
+        custom_entry_point_fn hello2 = nullptr;
+        int ret = t(
+                libPath.c_str(),
+                "Pyjion.Test, Pyjionlib",
+                "Hello2",
+                "Pyjion.Test+HelloDelegate, Pyjionlib" /*delegate_type_name*/,
+                nullptr,
+                (void**) &hello2);
+        REQUIRE(ret == Success);
+        REQUIRE(hello2 != nullptr);
+        CHECK(hello2(3, 4) == 12);
+    }
+
+    SECTION("Test delegate plane execution"){
+        std::filesystem::path libPath = filesystem::path("../pyjion-lib/bin/Debug/Pyjionlib.dll");
+        REQUIRE(std::filesystem::exists(libPath));
+        CHECK(load_hostfxr());
+        auto load = get_dotnet_load_assembly();
+        REQUIRE(load != nullptr);
+        component_entry_point_fn dot = nullptr;
+        struct lib_args
+        {
+            double x1;
+            double y1;
+            double z1;
+            double d1;
+            double x2;
+            double y2;
+            double z2;
+            double d2;
+        };
+
+        int ret = load(
+                libPath.c_str(),
+                "Pyjion.PlaneOperations, Pyjionlib",
+                "VectorDotProduct",
+                nullptr,//"Pyjion.PlaneOperations+VectorDotProductDelegate, Pyjionlib" /*delegate_type_name*/,
+                nullptr,
+                (void**) &dot);
+
+        switch (ret) {
+            case 0: {
+                REQUIRE(ret == Success);
+                REQUIRE(dot != nullptr);
+                lib_args args = {1., 2., 3., 4., 5., 6., 7., 8.};
+                CHECK(dot(&args, sizeof(lib_args)) == 70.);
+            }
+                break;
+            case COR_E_MISSINGMETHOD:
+                FAIL("Missing method exception");
+                break;
+            case COR_E_ARGUMENTOUTOFRANGE:
+                FAIL("Argument out of range exception");
+                break;
+            case COR_E_INVALIDOPERATION:
+                FAIL("Invalid operation exception");
+                break;
+            case E_POINTER:
+                FAIL("Invalid argument exception");
+                break;
+            case COR_E_TYPELOAD:
+                FAIL("Type load failure");
+                break;
+            default:
+                FAIL("Unexpected exception");
+                break;
+        }
+
+    }
+
+    SECTION("Test custom image") {
+        std::filesystem::path libPath = filesystem::path("../pyjion-lib/bin/Debug/Pyjionlib.dll");
+        REQUIRE(std::filesystem::exists(libPath));
+        PEDecoder decoder = PEDecoder(libPath.c_str());
+        CHECK(decoder.GetCorHeader()->Flags & COMIMAGE_FLAGS_ILONLY);
+        CHECK(decoder.GetModuleName() == "Pyjionlib.dll");
+        auto typeRefs = decoder.GetTypeRefs();
+        CHECK(!typeRefs.empty());
+        auto typeDefs = decoder.GetTypeDefs();
+        CHECK(!typeDefs.empty());
+        auto publicTypeDefs = decoder.GetPublicClasses();
+        CHECK(!publicTypeDefs.empty());
+        auto methods = decoder.GetClassMethods(publicTypeDefs[0]);
+        CHECK(!methods.empty());
+    }
+}

pyjion-lib/.gitignore

@@ -0,0 +1,2 @@
+**/bin/
+**/obj/