Merge pull request #34 from neutrons/update_packaging

Update python and RPM packaging
neutrons · Dec 13, 2023 · 0268826 · 0268826
2 parents 159fe71 + ca9c375
commit 0268826
Show file tree

Hide file tree

Showing 20 changed files with 143 additions and 237 deletions.
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Set up MicroMamba
       uses: mamba-org/setup-micromamba@v1
@@ -39,7 +39,7 @@ jobs:
     needs: tests
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Set up MicroMamba
       uses: mamba-org/setup-micromamba@v1
@@ -86,42 +86,45 @@ jobs:
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-# disable until spec file is updated to python 3
-#  rpm:
-#    runs-on: ubuntu-latest
-#
-#    steps:
-#    - name: Checkout Repository
-#      uses: actions/checkout@v2
-#
-#    - name: Build RPM inside Docker
-#      run: |
-#        docker build --tag postprocess --target=package -f Dockerfile .
-#        fname=`docker run -v $(pwd):/store postprocess ls /root/rpmbuild/RPMS/noarch`
-#        docker run -v `pwd`:/store postprocess mv /root/rpmbuild/RPMS/noarch/$fname /store
-#        one=${fname#*postprocessing-}
-#        two=${one%.noarch*}
-#        echo "::set-output name=version::$two"
-#        echo "::set-output name=fname::$fname"
-#      continue-on-error: false
-
-# TODO: uncomment this once we have switched to python3 and can conda install the module build
-#  wheel:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - name: Checkout Repository
-#      uses: actions/checkout@v2
-#
-#    - name: Set up MicroMamba
-#      uses: mamba-org/setup-micromamba@v1
-#      with:
-#        environment-file: environment.yml
-#        condarc: |
-#          channels:
-#            - conda-forge
-#            - default
-#        cache-environment: true
-#
-#    - name: Build wheel
-#      shell: bash -l {0}
-#      run: python -m build --wheel
+  rpm:
+    runs-on: ubuntu-latest
+    needs: integration-tests
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+
+    - name: Build RPM inside Docker
+      run: |
+        docker build --tag postprocess -f Dockerfile .
+        fname=`docker run --name postprocess  postprocess ls /root/rpmbuild/RPMS/noarch`
+        docker cp postprocess:/root/rpmbuild/RPMS/noarch/$fname .
+        one=${fname#*postprocessing-}
+        two=${one%.noarch*}
+        echo "{version}={$two}" >> $GITHUB_OUTPUT
+        echo "{fname}={$fname}" >> $GITHUB_OUTPUT
+      continue-on-error: false
+
+    # now run a couple integration test with RPM installed package
+    - name: Set up MicroMamba
+      uses: mamba-org/setup-micromamba@v1
+      with:
+        environment-file: environment.yml
+        condarc: |
+          channels:
+            - conda-forge
+            - default
+        cache-environment: true
+
+    - name: Start docker containers
+      run: docker-compose -f tests/integration/docker-compose-rpm.yml up --build -d
+
+    - name: Sleep, wait for containers to start up
+      run: sleep 2
+
+    - name: Run integration tests
+      shell: bash -l {0}
+      run: python -m pytest -k "test_heartbeat or test_missing_data"
+
+    - name: Bring down docker containers
+      run: docker-compose -f tests/integration/docker-compose-rpm.yml down
diff --git a/Dockerfile b/Dockerfile
@@ -1,31 +1,22 @@
-FROM --platform=linux/amd64 centos:7 as package
+FROM registry.access.redhat.com/ubi9/ubi
 
-RUN yum install -y make rpm-build
-# lots of packages don't exist in rhel7
-# the Makefile will print lots of warnings as a result
-
-WORKDIR /app
+RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN dnf install -y make rpm-build python3-build python3-pip python-unversioned-command
 
 COPY scripts /app/scripts
 COPY configuration /app/configuration
 COPY postprocessing /app/postprocessing
+COPY pyproject.toml /app/
+COPY rpmbuild.sh /app/
+COPY README.md /app/
+COPY LICENSE.rst /app/
 COPY SPECS /app/SPECS
-COPY Makefile /app/
-
 
 RUN mkdir -p /root/rpmbuild/SOURCES
 
-RUN make rpm || exit 1
-
-FROM --platform=linux/amd64 centos:7 as app
-COPY --from=package /root/rpmbuild/RPMS/noarch/postprocessing-*-1.noarch.rpm /
-
-
-RUN curl http://packages.sns.gov/distros/rhel/7/sns/sns.repo -o /etc/yum.repos.d/sns.repo
-RUN yum install -y epel-release
-RUN yum updateinfo
+RUN cd /app && ./rpmbuild.sh || exit 1
 
-RUN yum install -y /postprocessing-*-1.noarch.rpm || exit 1
+RUN dnf install -y /root/rpmbuild/RPMS/noarch/postprocessing*.noarch.rpm || exit 1
 
 # This configuration allows it to run with docker-compose from https://github.com/neutrons/data_workflow
 COPY configuration/post_process_consumer.conf.development /etc/autoreduce/post_processing.conf

diff --git a/Makefile b/Makefile
diff --git a/SPECS/postprocessing.spec b/SPECS/postprocessing.spec
@@ -1,48 +1,50 @@
-Summary: postprocessing
-Name: postprocessing
-Version: 2.6
-Release: 1
+%global srcname postprocessing
+%global summary postprocessing
+%define release 1
+
+Name: %{srcname}
+Version: 2.8.0
+Release: %{release}%{?dist}
+Summary: %{summary}
+
+License: MIT
+URL: https://github.com/neutrons/post_processing_agent
+Source: %{srcname}-%{version}.tar.gz
 Group: Applications/Engineering
-prefix: /opt/postprocessing
-BuildRoot: %{_tmppath}/%{name}
+
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 BuildArch: noarch
-License: MIT
-Source: postprocessing.tgz
-Requires: libNeXus.so.0()(64bit) libc.so.6()(64bit) libc.so.6(GLIBC_2.2.5)(64bit)
-Requires: nexus-python
-Requires: python-twisted-core
-Requires: python-twisted-web
-Requires: python-twisted-words
-Requires: python-stompest
-Requires: python2-stompest-async
-Requires: python-requests
-%define debug_package %{nil}
-%define site_packages %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
+BuildRequires:  python%{python3_pkgversion}-pip
+Requires: python%{python3_pkgversion}
+Requires: python%{python3_pkgversion}-requests
+Requires: python%{python3_pkgversion}-stomppy
+Requires: python-unversioned-command
+
+prefix: /opt/postprocessing
+
+%define site_packages %(python3 -c "import site; print(site.getsitepackages()[-1])")
 
 %description
 Post-processing agent to automatically catalog and reduce neutron data
 
 %prep
-%setup -q -n %{name}
+%autosetup -p1 -n %{srcname}-%{version}
 
 %build
 
 %install
-rm -rf %{buildroot}
-mkdir -p %{buildroot}%{_sysconfdir}
-mkdir -p %{buildroot}%{_bindir}
-mkdir -p %{buildroot}%{prefix}
+%{python3} -m pip install --target %{buildroot}%{prefix} --no-deps .
+mv %{buildroot}%{prefix}/postprocessing/queueProcessor.py %{buildroot}%{prefix}
+mkdir -p %{buildroot}%{prefix}/log
 mkdir -p %{buildroot}%{site_packages}
+echo %{prefix} > %{buildroot}%{site_packages}/postprocessing.pth
 mkdir -p %{buildroot}/var/log/SNS_applications
-make prefix="%{buildroot}%{prefix}" installed_prefix="%{prefix}" site_packages="%{buildroot}%{site_packages}" sysconfig="%{buildroot}%{_sysconfdir}/autoreduce" bindir="%{buildroot}/%{_bindir}" install
-
-%post
-chgrp snswheel %{_sysconfdir}/autoreduce/post_processing.conf
-chown -R snsdata %{_sysconfdir}/autoreduce
-chown -R snsdata %{prefix}
-chown -R snsdata /var/log/SNS_applications
 
 %files
-%attr(755, -, -) %{prefix}/
-%attr(755, -, -) /var/log/SNS_applications
-%attr(755, -, -) %{site_packages}/postprocessing.pth
+%doc README.md
+%license LICENSE.rst
+%{prefix}/*
+%attr(755, -, -) %{prefix}/scripts
+%attr(755, -, -) %{prefix}/queueProcessor.py
+%{site_packages}/postprocessing.pth
+/var/log/SNS_applications
diff --git a/configuration/post_process_consumer.conf.development b/configuration/post_process_consumer.conf.development
@@ -6,7 +6,7 @@
     "amq_pwd": "icat",
     "sw_dir": "/opt/postprocessing",
     "python_dir": "/opt/postprocessing/postprocessing",
-    "start_script": "python",
+    "start_script": "python3",
     "task_script": "PostProcessAdmin.py",
     "task_script_queue_arg": "-q",
     "task_script_data_arg": "-d",
@@ -20,5 +20,6 @@
     "dev_output_dir": "",
     "reduction_data_ready": "REDUCTION.DATA_READY",
     "communication_only": 0,
-    "max_procs": 5
+    "max_procs": 5,
+    "processors": ["reduction_processor.ReductionProcessor"]
 }
diff --git a/configuration/post_process_consumer.conf.local b/configuration/post_process_consumer.conf.local
@@ -6,7 +6,7 @@
     "amq_pwd": "",
     "sw_dir": "/opt/postprocessing",
     "python_dir": "/opt/postprocessing/postprocessing",
-    "start_script": "python",
+    "start_script": "python3",
     "task_script": "PostProcessAdmin.py",
     "task_script_queue_arg": "-q",
     "task_script_data_arg": "-d",

diff --git a/postprocessing/__init__.py b/postprocessing/__init__.py
@@ -1 +1 @@
-__version__ = "2.1.0"
+__version__ = "2.8.0"
diff --git a/postprocessing/processors/create_reduction_script_processor.py b/postprocessing/processors/create_reduction_script_processor.py
@@ -16,7 +16,7 @@ def __init__(self, data, conf, send_function):
         @param conf: configuration object
         @param send_function: function to call to send AMQ messages
         """
-        super(CreateReductionScriptProcessor, self).__init__(data, conf, send_function)
+        super().__init__(data, conf, send_function)
 
     def __call__(self):
         """

diff --git a/postprocessing/processors/reduction_processor.py b/postprocessing/processors/reduction_processor.py
@@ -23,7 +23,7 @@ def __init__(self, data, conf, send_function):
         @param conf: configuration object
         @param send_function: function to call to send AMQ messages
         """
-        super(ReductionProcessor, self).__init__(data, conf, send_function)
+        super().__init__(data, conf, send_function)
 
     def __call__(self):
         """

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,3 +1,27 @@
+[project]
+name = "postprocessing"
+description = "Post-processing agent to automatically catalog and reduce neutron data"
+version = "2.8.0"
+requires-python = ">=3.9"
+dependencies = [
+    "requests",
+    "plotly",
+    "stomp.py"
+]
+license = { text = "MIT" }
+
+[project.urls]
+homepage = "https://github.com/neutrons/post_processing_agent"
+
+[build-system]
+requires = [
+  "setuptools",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["postprocessing", "postprocessing.processors", "scripts"]
+
 [tool.ruff]
 line-length = 120
 # https://beta.ruff.rs/docs/rules/