Merge pull request #63 from ewanwm/feature_pypi_distribution

Add a workflow to upload python module to pypi

.github/workflows/pypi-distribution-reusable.yaml

@@ -0,0 +1,179 @@
+on:
+  workflow_call:
+    inputs:
+      build_type:
+        required: true
+        type: string
+      platform:
+        required: true
+        type: string
+
+jobs:
+  build:
+
+    name: Build ${{ inputs.platform }} ${{ inputs.build_type }} distribution 📦
+    runs-on: ${{ inputs.platform }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    ## runner will run out of space if we don't clear some up by removing some unused tools
+    - name: Clear space
+      run: >-
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        
+    #- name: cuda-toolkit
+    #  if: ${{ startsWith( inputs.build_type, 'cuda' ) }}
+    #  uses: Jimver/[email protected]
+
+    #- name: Set up Python
+    #  uses: actions/setup-python@v5
+    #  with:
+    #    python-version: "3.11"
+
+    #- name: Install Protoc 
+    #  run: sudo apt install protobuf-compiler
+
+    #- name: Install Python dependencies
+    #  uses: py-actions/py-dependency-install@v4
+    #  with:
+    #    path: "PyTorch_requirements.txt"
+
+    #- name: Install cibuildwheel
+    #  run: python -m pip install cibuildwheel==2.21.1
+
+    - uses: yezz123/setup-uv@v4
+
+    - uses: pypa/[email protected]
+      if: runner.os == 'Linux'
+      env:
+        CIBW_BUILD_VERBOSITY: 1
+        CIBW_ARCHS_LINUX: "x86_64"
+        CIBW_SKIP: "*musl*"
+        CIBW_BEFORE_ALL: >
+          yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo &&
+          yum clean all &&
+          yum -y install cuda-toolkit &&
+          ls -al /usr/local &&
+          nvcc --version &&
+          echo nvcc location: &&
+          which nvcc &&
+          export CUDACXX="$(which nvcc)" &&
+          echo $CUDACXX &&
+          echo gcc version: &&
+          gcc --version &&
+          export BUILD_WITH_CUDA=1 &&
+          export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0" 
+        CIBW_ENVIRONMENT: >
+          PATH=$PATH:/usr/local/cuda/bin LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(python -c 'import sys; print(sys.prefix + "/lib/python" + sys.version[:3] + "/site-packages")')/torch/lib/ 
+        CIBW_BEFORE_BUILD: >
+          python -m site &&
+          python -c 'import sys; print(sys.prefix + "/lib/python" + sys.version[:3] + "/site-packages")' &&
+          echo $LD_LIBRARY_PATH &&
+          ls $(python -c 'import sys; print(sys.prefix + "/lib/python" + sys.version[:3] + "/site-packages")')
+        
+      
+    #- name: Build wheels
+    #  run: python -m cibuildwheel --output-dir dist
+    #  env:
+    #    CIBW_BEFORE_BUILD: pip install scikit-build-core
+
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    needs:
+    - build
+    runs-on: ${{ inputs.platform }}
+
+    environment:
+      name: testPyPi-distribution
+      url: https://test.pypi.org/p/nuTens
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ${{ inputs.platform }}
+    environment:
+      name: PyPi-distribution
+      url: https://pypi.org/p/nuTens  # Replace <package-name> with your PyPI project name
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    name: >-
+      Sign the Python 🐍 distribution 📦 with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ${{ inputs.platform }}
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/[email protected]
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        '${{ github.ref_name }}'
+        --repo '${{ github.repository }}'
+        --notes ""
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        '${{ github.ref_name }}' dist/**
+        --repo '${{ github.repository }}'

.github/workflows/pypi-distribution.yaml

@@ -0,0 +1,100 @@
+name: Publish to PyPi
+
+on: 
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+
+  build:
+    name: Make SDist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Optional, use if you use setuptools_scm
+        submodules: true  # Optional, use if you have submodules
+
+    - name: Build SDist
+      run: pipx run build --sdist
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/*.tar.gz
+
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testPyPi-distribution
+      url: https://test.pypi.org/p/nuTens
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: PyPi-distribution
+      url: https://pypi.org/p/nuTens  # Replace <package-name> with your PyPI project name
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+
+    ## one day we will buld wheels and upload them to pypi... but it is not this day
+    ## when that day comes, the below might be useful
+
+    ##pypi_distribution:
+
+    ##permissions:
+    ##  contents: write
+    ##  id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    ## secrets: inherit
+
+    ##strategy: 
+    ##  fail-fast: false
+    ##  matrix:
+    ##    include:
+    ##      - build_type: cpu
+    ##        platform: ubuntu-latest
+
+    ##      - build_type: cuda
+    ##        platform: ubuntu-latest
+
+    ## ./ At start of reusable workflow indicates to use the one in current repo i.e. the from the current branch
+    ## uses: ./.github/workflows/pypi-distribution-reusable.yaml
+    ## with:
+    ##  platform: ${{ matrix.platform }}
+    ##  build_type: ${{ matrix.build_type }}
+

CMakeLists.txt

@@ -1,5 +1,11 @@
 cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
 set(CMAKE_CXX_STANDARD 17)
+set(CUDA_STANDARD 14)
+set( CMAKE_CUDA_COMPILER "nvcc" )
+
+if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+  set(CMAKE_CUDA_ARCHITECTURES 75)
+endif()
 
 project(nuTens)
 

pyproject.toml

@@ -1,15 +1,41 @@
-[build-system]
-requires = ["scikit-build-core", "pybind11", "torch"]
-build-backend = "scikit_build_core.build"
 
 [project]
-name = "pyNuTens"
-version = "0.0.1"
+name = "nuTens"
+version = "0.0.3"
 description="Library to calculate neutrino oscillation probabilities using tensors"
 readme = "README.md"
 authors = [
   { name = "Ewan Miller", email = "[email protected]" },
 ]
+requires-python = ">=3.9"
+license = {file="LICENSE"}
+keywords = ["neutrino", "oscillations", "physics", "particle", "tensor", "experiment", "autograd", "differentiable programming"]
+classifiers = [
+  "Development Status :: 2 - Pre-Alpha",
+  "Environment :: GPU",
+  "Programming Language :: C++",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Topic :: Scientific/Engineering :: Physics",
+  "Intended Audience :: Science/Research"
+]
+dependencies = [
+  "torch"
+]
+
+[project.urls]
+Repository = "https://github.com/ewanwm/nuTens"
+Issues = "https://github.com/ewanwm/nuTens/issues"
+Documentation = "https://ewanwm.github.io/nuTens/"
+
+[build-system]
+requires = ["scikit-build-core>=0.3.3", "pybind11", "torch"]
+build-backend = "scikit_build_core.build"
+
+[tool.cibuildwheel]
+build-frontend = "build[uv]"
 
-[tool.scikit-build]
-cmake.args = ["-DNT_ENABLE_PYTHON=ON"]
+[tool.scikit-build.cmake]
+args = ["-DNT_ENABLE_PYTHON=ON"]

python/CMakeLists.txt

@@ -1,15 +1,15 @@
 
 pybind11_add_module( 
-    pyNuTens MODULE 
+    _pyNuTens MODULE 
     binding.cpp 
 )
 
 if(NT_USE_PCH)
-    target_precompile_headers( pyNuTens REUSE_FROM nuTens-pch )
+    target_precompile_headers( _pyNuTens REUSE_FROM nuTens-pch )
 endif()
-target_link_libraries( pyNuTens PUBLIC nuTens )
+target_link_libraries( _pyNuTens PUBLIC nuTens )
 
 # This is passing in the version as a define just as an example
-target_compile_definitions( pyNuTens PRIVATE VERSION_INFO=${PROJECT_VERSION} )
+target_compile_definitions( _pyNuTens PRIVATE VERSION_INFO=${PROJECT_VERSION} )
 
-install( TARGETS pyNuTens DESTINATION pyNuTens/ )
+install( TARGETS _pyNuTens DESTINATION nuTens/ )

python/README.md

@@ -0,0 +1,5 @@
+## Python Interface
+
+nuTens provides a python interface that can be installed manually or via [pyPi](https://pypi.org/). [binding.cpp](binding.cpp) defines the python bindings of many of the nuTens c++ objects, which are then compiled into a python module using [pybind11](https://github.com/pybind/pybind11) (see [CMakeLists.txt](CMakeLists.txt)).
+
+The [nuTens](nuTens) folder defines the python module and any pure python extension code for nuTens should go in there.

python/binding.cpp

@@ -18,9 +18,9 @@ void initTensor(py::module & /*m*/);
 void initPropagator(py::module & /*m*/);
 void initDtypes(py::module & /*m*/);
 
-// initialise the top level module "pyNuTens"
+// initialise the top level module "_pyNuTens"
 // NOLINTNEXTLINE
-PYBIND11_MODULE(pyNuTens, m)
+PYBIND11_MODULE(_pyNuTens, m)
 {
     m.doc() = "Library to calculate neutrino oscillations";
     initTensor(m);

python/nuTens/__init__.py

@@ -0,0 +1,3 @@
+from ._pyNuTens import __doc__, __version__, tensor, propagator, dtype
+
+__all__ = ["__doc__", "__version__", "tensor", "propagator", "dtype"]

python/nuTens/dtype.py

@@ -0,0 +1,2 @@
+from ._pyNuTens import dtype
+from ._pyNuTens.dtype import *

python/nuTens/propagator.py

@@ -0,0 +1,2 @@
+from ._pyNuTens import propagator
+from ._pyNuTens.propagator import *