51 remove size limitation from capnpreaderoptions to allow parsing of…

… big inputs2 (#53)

* adding test and fix

* dump verilog in test

* cleaning and adding long tests

* more testing

* more coverage

* add faulty script

.github/workflows/build.yml

@@ -29,7 +29,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_SANITIZERS=OFF
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_SANITIZERS=OFF -DLONG_TESTS=ON
 
     - name: Build
       # Build your program with the given configuration

.github/workflows/coverage.yml

@@ -31,7 +31,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCODE_COVERAGE=ON
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCODE_COVERAGE=ON -DLONG_TESTS=ON
 
     - name: Build
       # Build your program with the given configuration

.github/workflows/macos-build.yml

@@ -32,7 +32,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_SANITIZERS=OFF
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLONG_TESTS=ON -DENABLE_SANITIZERS=OFF
 
     - name: Build
       # Build your program with the given configuration

.github/workflows/valgrind.yml

@@ -32,7 +32,7 @@ jobs:
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLONG_TESTS=ON
 
     - name: Build
       # Build your program with the given configuration

CMakeLists.txt

@@ -62,7 +62,6 @@ endif(ENABLE_SANITIZERS)
 find_package(TBB REQUIRED)
 find_package(Boost REQUIRED)
 find_package(Python3 3.9...<3.99 COMPONENTS Development Interpreter REQUIRED)
-	#PATHS /usr/lib/python3.9.5)
 find_package(CapnProto CONFIG REQUIRED)
 
 add_subdirectory(thirdparty)
@@ -71,6 +70,7 @@ add_subdirectory(cmake)
 add_subdirectory(primitives)
 
 if(PROJECT_IS_TOP_LEVEL)
+option(LONG_TESTS "Enable long tests" OFF)
 include(CTest)
 enable_testing()
 add_subdirectory(test)

README.md

@@ -94,8 +94,8 @@ export NAJA_INSTALL=<path_to_installation_dir>
 # Create a build dir and go inside it
 mkdir build
 cd build
-cmake <path_to_naja_sources_dir> -DCMAKE_INSTALL_PREFIX=$NAJA_INSTALL
-#For instance: cmake ~/srcs/naja -DCMAKE_INSTALL_PREFIX=$NAJA_INSTALL
+cmake <path_to_naja_sources_dir> -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$NAJA_INSTALL
+#For instance: cmake ~/srcs/naja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$NAJA_INSTALL
 make
 make test
 make install

src/snl/python/pyloader/SNLPyLoader.cpp

@@ -8,13 +8,14 @@
 #include <sstream>
 #include <cstdio>
 #include <Python.h>
+#include <frameobject.h> // Include the header for PyFrameObject
 
 #include "SNLUniverse.h"
 #include "SNLException.h"
 
 #include "PySNLDB.h"
 #include "PySNLLibrary.h"
-#include <frameobject.h> // Include the header for PyFrameObject
+#include "PySNLDesign.h"
 
 namespace {
 
@@ -200,6 +201,44 @@ void SNLPyLoader::loadLibrary(
   Py_Finalize();
 }
 
+void SNLPyLoader::loadDesign(
+    SNLDesign* design,
+    const std::filesystem::path& path) {
+  if (design->isPrimitive()) {
+    std::ostringstream reason;
+    reason << "Cannot construct design if it is a primitive";
+    throw SNLException(reason.str());
+  }
+  auto module = loadModule(path);
+
+  PyObject* pyDesign = PYSNL::PySNLDesign_Link(design);
+  PyObject* constructString = PyUnicode_FromString("construct");
+
+  PyObject* res =
+    PyObject_CallMethodObjArgs(module, constructString, pyDesign, NULL);
+  if (not res) {
+    std::ostringstream reason;
+    reason << "Error while calling construct";
+    std::string pythonError = getPythonError();
+    if (not pythonError.empty()) {
+      reason << ": " << pythonError;
+    } else {
+      reason << ": empty error message";
+    }
+    //Cleaning
+    Py_DECREF(module);
+    Py_DECREF(pyDesign);
+    Py_DECREF(constructString);
+    Py_Finalize();
+    throw SNLException(reason.str());
+  }
+  //Cleaning
+  Py_DECREF(module);
+  Py_DECREF(pyDesign);
+  Py_DECREF(constructString);
+  Py_Finalize();
+}
+
 void SNLPyEdit::edit(const std::filesystem::path& path) {
   auto module = loadModule(path);
 

src/snl/python/pyloader/SNLPyLoader.h

@@ -11,6 +11,7 @@ namespace naja { namespace SNL {
 
 class SNLDB;
 class SNLLibrary;
+class SNLDesign;
 
 class SNLPyLoader {
   public:
@@ -21,6 +22,9 @@ class SNLPyLoader {
       SNLLibrary* library,
       const std::filesystem::path& scriptPath,
       bool loadPrimitives=false);
+    static void loadDesign(
+      SNLDesign* design,
+      const std::filesystem::path& scriptPath);
     static void loadPrimitives(
       SNLLibrary* library,
       const std::filesystem::path& scriptPath);

src/snl/python/snl_wrapping/PySNLUniverse.cpp

@@ -6,6 +6,8 @@
 
 #include "PyInterface.h"
 #include "PySNLDesign.h"
+#include "PySNLDB.h"
+
 #include "SNLUniverse.h"
 
 namespace PYSNL {
@@ -28,6 +30,7 @@ static PyObject* PySNLUniverse_get() {
 }
 
 GetObjectMethod(Universe, Design, getTopDesign)
+GetObjectByIndex(Universe, DB, DB)
 
 DBoDestroyAttribute(PySNLUniverse_destroy, PySNLUniverse)
 
@@ -40,6 +43,8 @@ PyMethodDef PySNLUniverse_Methods[] = {
     "get the SNL Universe (static object)"},
   { "getTopDesign", (PyCFunction)PySNLUniverse_getTopDesign, METH_NOARGS,
     "get the top SNLDesign"},
+  { "getDB", (PyCFunction)PySNLUniverse_getDB, METH_VARARGS,
+    "get the SNLDB with the given index"},
   {NULL, NULL, 0, NULL}           /* sentinel */
 };
 

src/snl/snl/serialization/capnp/SNLCapnPImplementation.cpp

@@ -507,7 +507,9 @@ void SNLCapnP::sendImplementation(
 //LCOV_EXCL_STOP
 
 SNLDB* SNLCapnP::loadImplementation(int fileDescriptor) {
-  ::capnp::PackedFdMessageReader message(fileDescriptor);
+  ::capnp::ReaderOptions options;
+  options.traversalLimitInWords = std::numeric_limits<uint64_t>::max();
+  ::capnp::PackedFdMessageReader message(fileDescriptor, options);
 
   DBImplementation::Reader dbImplementation = message.getRoot<DBImplementation>();
   auto dbID = dbImplementation.getId();

test/snl/python/pyloader/CMakeLists.txt

@@ -6,6 +6,7 @@ INCLUDE(GoogleTest)
 
 SET(snl_pyloader_tests
   SNLPyDBLoaderTest0.cpp
+  SNLPyDesignLoaderTest0.cpp
   SNLPrimitivesTest0.cpp
 )
 
@@ -24,9 +25,28 @@ SET(snl_pyedit_tests
   SNLPyEditTest0.cpp
 )
 
+SET(snl_pyedit_long_tests
+  SNLPyHugeMatrixTest.cpp
+)
+ADD_EXECUTABLE(snlPyEditLongTests ${snl_pyedit_long_tests})
+TARGET_LINK_LIBRARIES(snlPyEditLongTests naja_snl_pyloader naja_snl_dump naja_snl_verilog gmock gtest_main)
+target_compile_definitions(snlPyEditLongTests PRIVATE
+  SNL_PYEDIT_TEST_PATH="${CMAKE_CURRENT_SOURCE_DIR}"
+  SNL_DUMP_PATH="${CMAKE_CURRENT_BINARY_DIR}"
+  )
+IF(LONG_TESTS)
+gtest_discover_tests(
+  snlPyEditLongTests
+  PROPERTIES ENVIRONMENT
+  PYTHONPATH=${PROJECT_BINARY_DIR}/src/snl/python/snl_wrapping
+)
+ENDIF(LONG_TESTS)
+
 ADD_EXECUTABLE(snlPyEditTests ${snl_pyedit_tests})
 target_compile_definitions(snlPyEditTests PRIVATE
-  SNL_PYEDIT_TEST_PATH="${CMAKE_CURRENT_SOURCE_DIR}")
+  SNL_PYEDIT_TEST_PATH="${CMAKE_CURRENT_SOURCE_DIR}"
+  SNL_DUMP_PATH="${CMAKE_CURRENT_BINARY_DIR}"
+  )
 TARGET_LINK_LIBRARIES(snlPyEditTests naja_snl_pyloader gmock gtest_main)
 gtest_discover_tests(
   snlPyEditTests

test/snl/python/pyloader/SNLPyDesignLoaderTest0.cpp

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: 2023 The Naja authors <https://github.com/najaeda/naja/blob/main/AUTHORS>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#include "gtest/gtest.h"
+
+#include "SNLUniverse.h"
+#include "SNLScalarTerm.h"
+#include "SNLPyLoader.h"
+#include "SNLException.h"
+using namespace naja::SNL;
+
+#ifndef SNL_PRIMITIVES_TEST_PATH
+#define SNL_PRIMITIVES_TEST_PATH "Undefined"
+#endif
+
+class SNLPyDesignLoaderTest0: public ::testing::Test {
+  protected:
+    void SetUp() override {
+      auto universe = SNLUniverse::create();
+      auto db = SNLDB::create(universe);
+      primitives_ = SNLLibrary::create(db, SNLLibrary::Type::Primitives, SNLName("primitives"));
+      auto primitive = SNLDesign::create(primitives_, SNLDesign::Type::Primitive, SNLName("primitive"));
+      auto pi = SNLScalarTerm::create(primitive, SNLTerm::Direction::Input, SNLName("i"));
+      auto po = SNLScalarTerm::create(primitive, SNLTerm::Direction::Output, SNLName("o"));
+      auto designsLibrary = SNLLibrary::create(db, SNLName("designs"));
+      design_ = SNLDesign::create(designsLibrary, SNLName("design"));
+    }
+    void TearDown() override {
+      if (SNLUniverse::get()) {
+        SNLUniverse::get()->destroy();
+      }
+    }
+    SNLLibrary* primitives_;
+    SNLDesign*  design_;
+};
+
+TEST_F(SNLPyDesignLoaderTest0, test) {
+  auto scriptPath = std::filesystem::path(SNL_PRIMITIVES_TEST_PATH);
+  scriptPath /= "scripts";
+  scriptPath /= "design_loader.py";
+  SNLPyLoader::loadDesign(design_, scriptPath);
+
+  //ASSERT_EQ(2, db->getLibraries().size());
+}
+
+TEST_F(SNLPyDesignLoaderTest0, testDesignLoadingError) {
+  auto scriptPath = std::filesystem::path(SNL_PRIMITIVES_TEST_PATH);
+  scriptPath /= "scripts";
+  scriptPath /= "design_faulty.py";
+  EXPECT_THROW(SNLPyLoader::loadDesign(design_, scriptPath), SNLException);
+}
+
+TEST_F(SNLPyDesignLoaderTest0, testPrimitiveLoadingError) {
+  auto design = SNLDesign::create(primitives_, SNLDesign::Type::Primitive, SNLName("top"));
+  auto scriptPath = std::filesystem::path(SNL_PRIMITIVES_TEST_PATH);
+  scriptPath /= "scripts";
+  scriptPath /= "design_loader.py";
+  EXPECT_THROW(SNLPyLoader::loadDesign(design, scriptPath), SNLException);
+}

test/snl/python/pyloader/SNLPyEditTest0.cpp

@@ -49,7 +49,7 @@ TEST_F(SNLPyDBEditTest0, test) {
   EXPECT_EQ(instance0, top->getInstance(SNLName("instance1")));
 }
 
-TEST_F(SNLPyDBEditTest0, testEditError) {
+TEST_F(SNLPyDBEditTest0, testEditDBError) {
   auto db = SNLDB::create(SNLUniverse::get());
   auto scriptPath = std::filesystem::path(SNL_PYEDIT_TEST_PATH);
   scriptPath /= "edit";

test/snl/python/pyloader/SNLPyHugeMatrixTest.cpp

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: 2023 The Naja authors <https://github.com/najaeda/naja/blob/main/AUTHORS>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#include "gtest/gtest.h"
+
+#include "SNLUniverse.h"
+#include "SNLScalarTerm.h"
+#include "SNLException.h"
+#include "SNLPyLoader.h"
+#include "SNLCapnP.h"
+#include "SNLVRLDumper.h"
+using namespace naja::SNL;
+
+#ifndef SNL_PYEDIT_TEST_PATH
+#define SNL_PYEDIT_TEST_PATH "Undefined"
+#endif
+#ifndef SNL_DUMP_PATH
+#define SNL_DUMP_PATH "Undefined"
+#endif
+
+class SNLPyHugeMatrixTest: public ::testing::Test {
+  protected:
+    void SetUp() override {
+      SNLUniverse::create();
+      auto db = SNLDB::create(SNLUniverse::get());
+      auto primitivesLibrary = SNLLibrary::create(db, SNLLibrary::Type::Primitives, SNLName("primitives"));
+      auto square = SNLDesign::create(primitivesLibrary, SNLDesign::Type::Primitive, SNLName("square"));
+      auto n = SNLScalarTerm::create(square, SNLTerm::Direction::Output, SNLName("n"));
+      auto e = SNLScalarTerm::create(square, SNLTerm::Direction::Output, SNLName("e"));
+      auto s = SNLScalarTerm::create(square, SNLTerm::Direction::Input, SNLName("s"));
+      auto w = SNLScalarTerm::create(square, SNLTerm::Direction::Input, SNLName("w"));
+      designsLibrary_ = SNLLibrary::create(db, SNLName("designs"));
+    }
+    void TearDown() override {
+      if (SNLUniverse::get()) {
+        SNLUniverse::get()->destroy();
+      }
+    }
+    SNLLibrary* designsLibrary_;
+};
+
+TEST_F(SNLPyHugeMatrixTest, test) {
+  auto db = SNLDB::create(SNLUniverse::get());
+  auto scriptPath = std::filesystem::path(SNL_PYEDIT_TEST_PATH);
+  scriptPath /= "scripts";
+  scriptPath /= "huge_matrix.py";
+  auto top = SNLDesign::create(designsLibrary_, SNLName("top"));
+  SNLPyLoader::loadDesign(top, scriptPath);
+
+  //dump the design
+  auto dumpPath = std::filesystem::path(SNL_DUMP_PATH);
+  dumpPath /= "huge_matrix";
+  if (std::filesystem::exists(dumpPath)) {
+    std::filesystem::remove_all(dumpPath);
+  }
+  SNLCapnP::dump(top->getDB(), dumpPath);
+
+  //dump verilog
+  auto outPath = std::filesystem::path(SNL_DUMP_PATH);
+  SNLVRLDumper dumper;
+  dumper.setTopFileName(top->getName().getString() + ".v");
+  dumper.setSingleFile(true);
+  dumper.dumpDesign(top, outPath);
+
+  SNLUniverse::get()->destroy();  
+  top = nullptr;
+  designsLibrary_ = nullptr;
+
+  SNLCapnP::load(dumpPath);
+}

test/snl/python/pyloader/scripts/design_faulty.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2023 The Naja authors <https://github.com/najaeda/naja/blob/main/AUTHORS>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import snl
+
+def construct(design):
+  ERROR
+  pass
+
+def constructDB(design):
+  construct(design)

test/snl/python/pyloader/scripts/design_loader.py

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2023 The Naja authors <https://github.com/najaeda/naja/blob/main/AUTHORS>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import snl
+
+def construct(design):
+  universe = snl.SNLUniverse.get()
+  db = universe.getDB(1)
+  if db is None:
+    raise Exception("Cannot find the db containing the primitives and top")
+  primitives = db.getLibrary("primitives")
+  if primitives is None:
+    raise Exception("Cannot find the primitives library")
+  prim = primitives.getDesign("primitive")
+  if prim is None:
+    raise Exception("Cannot find the primitive design")
+  snl.SNLInstance.create(design, prim, "myins")
+
+def constructDB(db):
+  construct(db)