Merge pull request #24 from IITH-Compilers/pip-package

Pip package

.github/workflows/build.yml

@@ -45,6 +45,7 @@ jobs:
       run: |
         conda init
         conda activate mlbridge
+        pip install compilerinterface
         cd $GITHUB_WORKSPACE/test
         bash mlbridge-test.sh
     - uses: actions/upload-artifact@v2

.github/workflows/upload_pypi.yml

@@ -0,0 +1,45 @@
+name: Upload to PyPI
+
+on:
+  release:
+    types:
+      - published
+  workflow_dispatch:
+    inputs:
+      pypi_repo:
+        description: 'Repo to upload to pypi'
+        default: 'pypi'
+        required: true
+        type: choice
+        options:
+          - testpypi
+          - pypi
+
+jobs:
+  build_wheels:
+    uses: ./.github/workflows/wheel.yml
+
+  upload_pypi:
+    permissions:
+      id-token: write
+    needs: [build_wheels]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: artifact
+          path: ./CompilerInterface/dist
+
+      # - name: Publish package to TestPyPI
+      #   uses: pypa/[email protected]
+      #   with:
+      #     repository-url: https://test.pypi.org/legacy/
+      #     packages-dir: ./CompilerInterface/dist
+      #   if: ${{ github.event.inputs.pypi_repo != 'pypi' }}
+
+      - name: Publish package to PyPI
+        uses: pypa/[email protected]
+        with:
+          repository-url: https://upload.pypi.org/legacy/
+          packages-dir: ./CompilerInterface/dist
+        # if: ${{ github.event.inputs.pypi_repo == 'pypi' }}

.github/workflows/wheel.yml

@@ -0,0 +1,27 @@
+name: Build wheels
+
+on: [push, workflow_dispatch, workflow_call]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build wheels
+        run: |
+            cd $GITHUB_WORKSPACE/CompilerInterface
+            python fetch_version.py
+            cp ../README.md ./
+            pip wheel . -w ./dist
+            pip install dist/compilerinterface*.whl
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: ./CompilerInterface/dist

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.10)
 
-project(MLCompilerBridge VERSION 0.0.1)
+project(MLCompilerBridge VERSION 0.0.2)
 add_compile_options("$<$<CONFIG:${CMAKE_BUILD_TYPE}>:-UNDEBUG>")
 set(protobuf_MODULE_COMPATIBLE TRUE)
 find_package(Protobuf CONFIG REQUIRED)

CompilerInterface/BaseCompilerInterface.py → CompilerInterface/compilerinterface/BaseCompilerInterface.py

@@ -15,9 +15,7 @@
 
 
 from abc import ABC, abstractmethod
-from SerDes import SerDes
-import os
-import io
+from .SerDes import SerDes
 
 ## This base class specifies methods for communication with compiler.
 class BaseCompilerInterface(ABC):

CompilerInterface/GrpcCompilerInterface.py → CompilerInterface/compilerinterface/GrpcCompilerInterface.py

@@ -13,13 +13,9 @@
 # ------------------------------------------------------------------------------
 
 
-from abc import ABC, abstractmethod
-from BaseCompilerInterface import BaseCompilerInterface
-import os
-import io
+from .BaseCompilerInterface import BaseCompilerInterface
 import time
 
-import sys
 import grpc
 from concurrent import futures
 
@@ -91,7 +87,7 @@ def start_server(self):
                     "{}:{}".format(self.host, self.server_port)
                 )
 
-                if added_port == self.server_port:
+                if str(added_port) == str(self.server_port):
                     server.start()
                     print("Server Running")
                     server.wait_for_termination()

CompilerInterface/PipeCompilerInterface.py → CompilerInterface/compilerinterface/PipeCompilerInterface.py

@@ -12,8 +12,7 @@
 ##
 # ------------------------------------------------------------------------------
 
-from abc import ABC, abstractmethod
-from BaseCompilerInterface import BaseCompilerInterface
+from .BaseCompilerInterface import BaseCompilerInterface
 import os
 import io
 

CompilerInterface/SerDes.py → CompilerInterface/compilerinterface/SerDes.py

@@ -11,9 +11,8 @@
 ##
 # ------------------------------------------------------------------------------
 
-import os, io
 import json
-import log_reader
+from . import log_reader
 import ctypes
 import struct
 

CompilerInterface/compilerinterface/__init__.py

@@ -0,0 +1,12 @@
+# ------------------------------------------------------------------------------
+#
+# Part of the MLCompilerBridge Project, under the Apache License v2.0 with LLVM
+# Exceptions. See the LICENSE file for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ------------------------------------------------------------------------------
+
+from .BaseCompilerInterface import BaseCompilerInterface
+from .PipeCompilerInterface import PipeCompilerInterface
+from .GrpcCompilerInterface import GrpcCompilerInterface
+from .SerDes import SerDes

CompilerInterface/log_reader.py → CompilerInterface/compilerinterface/log_reader.py

@@ -15,12 +15,10 @@
 import dataclasses
 import io
 import json
-import math
 import sys
 from typing import List, Optional
 from functools import reduce
 import operator
-import numpy
 
 _element_types = {
     "float": ctypes.c_float,

CompilerInterface/fetch_version.py

@@ -0,0 +1,33 @@
+# ------------------------------------------------------------------------------
+#
+# Part of the MLCompilerBridge Project, under the Apache License v2.0 with LLVM
+# Exceptions. See the LICENSE file for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ------------------------------------------------------------------------------
+
+import re
+
+version_regex = re.compile(r"^project\(MLCompilerBridge VERSION (?P<version>[^)]+)\)$")
+toml_field_regex = r'version[ ]*=[ ]*"(.*)"'
+
+VERSION = ""
+with open("../CMakeLists.txt", "r") as f:
+    for line in f:
+        vmatch = version_regex.match(line)  # Not using walrus because Python3.6
+        if vmatch:
+            VERSION = vmatch.group("version")
+            break
+
+print("Version detected =", VERSION)
+lines = []
+with open("./pyproject.toml", "r") as f:
+    lines = f.readlines()
+
+with open("./pyproject.toml", "w") as f:
+    for line in lines:
+        if re.search(toml_field_regex, line):
+            new_text = f'version = "{VERSION}"\n'
+            f.write(new_text)
+        else:
+            f.write(line)

CompilerInterface/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "compilerinterface"
+version = "0.0.1"
+authors = [ {name = "The authors of \"The Next 700 ML-Enabled Compiler Optimizations\"" }]
+description = "Communication framework for ML-Enabled Compiler Optimizations."
+license = { text= "Apache License v2.0 with LLVM Exceptions"}
+readme = "README.md"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Compilers",
+    "Development Status :: 4 - Beta",
+    "Operating System :: POSIX :: Linux",
+]
+
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

CompilerInterface/setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+url = https://github.com/IITH-Compilers/ML-Compiler-Bridge/

README.md

@@ -11,6 +11,7 @@ Please see [here](https://iith-compilers.github.io/ML-Compiler-Bridge/) for docu
 [![Build and Tests](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/build.yml/badge.svg)](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/build.yml)
 [![Doxygen Action](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/main.yml/badge.svg)](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/main.yml)
 [![pre-commit checks](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/formatting.yml/badge.svg)](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/formatting.yml)
+[![Upload to Pypi](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/upload_pypi.yml/badge.svg)](https://github.com/IITH-Compilers/MLCompilerBridge/actions/workflows/upload_pypi.yml)
 
 
 ![Image](https://github.com/IITH-Compilers/ML-Compiler-Bridge/raw/main/images/component-ml-compiler-bridge.png)
@@ -43,10 +44,11 @@ Please see [here](https://iith-compilers.github.io/ML-Compiler-Bridge/) for docu
 1. `mkdir build && cd build`
 2. `cmake [-DCMAKE_BUILD_TYPE=Release|Debug] [-DCMAKE_INSTALL_PREFIX=<Install_path>] [-DMLBRIDGE_ENABLE_TEST=ON|OFF] -DONNXRUNTIME_ROOTDIR=<Path to ONNX install dir> -DPROTOS_DIRECTORY=<Path to protobuf files> -DTENSORFLOW_AOT_PATH=<Path to TensorFlow pip install dir> ../`
 3. `make -j [&& make install]`
+4. `pip install compilerinterface`
 
 This process would generate `libMLCompilerBridge.a` and `libMLCompilerBridgeC.a` libraries under `build/lib` directory, required headers under `build/include` directory. `libMLCompilerBridgeC.a` exposes C APIs for using with C-based compilers like Pluto, where as `libMLCompilerBridge.a` exposes C++ APIs that can be used with any compiler written in C++.
 
-Python end points are available under [`CompilerInterface`](./CompilerInterface/).
+Python end points are available under [`CompilerInterface`](./CompilerInterface/). They can be downloaded as a [`package`](https://pypi.org/project/compilerinterface/) from pypi.
 
 To ensure the correctness, run `make verify-all`. This would need enabling tests in cmake (`-DMLBRIDGE_ENABLE_TEST=ON`) and `PROTOS_DIRECTORY` should point to `test/protos`.
 

test/mlbridge-test.py

@@ -19,12 +19,10 @@
 BUILD_DIR = "../build_release"
 sys.path.extend(
     [
-        f"{BUILD_DIR}/MLModelRunner/CompilerInterface",
         f"{BUILD_DIR}/MLModelRunner/gRPCModelRunner/Python-Utilities",
     ]
 )
-from PipeCompilerInterface import PipeCompilerInterface
-from GrpcCompilerInterface import GrpcCompilerInterface
+from compilerinterface import PipeCompilerInterface, GrpcCompilerInterface
 
 FAIL = 1
 SUCCESS = 0