Update packaging

.github/workflows/actions.yml

@@ -3,8 +3,6 @@ name: CI
 on:
   pull_request:
   push:
-    branches: main
-    tags: ['v*']
   workflow_dispatch:
 
 
@@ -13,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Set up MicroMamba
       uses: mamba-org/setup-micromamba@v1
@@ -39,7 +37,7 @@ jobs:
     needs: tests
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Set up MicroMamba
       uses: mamba-org/setup-micromamba@v1
@@ -86,42 +84,44 @@ jobs:
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-# disable until spec file is updated to python 3
-#  rpm:
-#    runs-on: ubuntu-latest
-#
-#    steps:
-#    - name: Checkout Repository
-#      uses: actions/checkout@v2
-#
-#    - name: Build RPM inside Docker
-#      run: |
-#        docker build --tag postprocess --target=package -f Dockerfile .
-#        fname=`docker run -v $(pwd):/store postprocess ls /root/rpmbuild/RPMS/noarch`
-#        docker run -v `pwd`:/store postprocess mv /root/rpmbuild/RPMS/noarch/$fname /store
-#        one=${fname#*postprocessing-}
-#        two=${one%.noarch*}
-#        echo "::set-output name=version::$two"
-#        echo "::set-output name=fname::$fname"
-#      continue-on-error: false
-
-# TODO: uncomment this once we have switched to python3 and can conda install the module build
-#  wheel:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - name: Checkout Repository
-#      uses: actions/checkout@v2
-#
-#    - name: Set up MicroMamba
-#      uses: mamba-org/setup-micromamba@v1
-#      with:
-#        environment-file: environment.yml
-#        condarc: |
-#          channels:
-#            - conda-forge
-#            - default
-#        cache-environment: true
-#
-#    - name: Build wheel
-#      shell: bash -l {0}
-#      run: python -m build --wheel
+  rpm:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+
+    - name: Build RPM inside Docker
+      run: |
+        docker build --tag postprocess -f Dockerfile .
+        fname=`docker run --name postprocess  postprocess ls /root/rpmbuild/RPMS/noarch`
+        docker cp postprocess:/root/rpmbuild/RPMS/noarch/$fname .
+        one=${fname#*postprocessing-}
+        two=${one%.noarch*}
+        echo "::set-output name=version::$two"
+        echo "::set-output name=fname::$fname"
+      continue-on-error: false
+
+    # now run a couple integration test with RPM installed package
+    - name: Set up MicroMamba
+      uses: mamba-org/setup-micromamba@v1
+      with:
+        environment-file: environment.yml
+        condarc: |
+          channels:
+            - conda-forge
+            - default
+        cache-environment: true
+
+    - name: Start docker containers
+      run: docker-compose -f tests/integration/docker-compose-rpm.yml up --build -d
+
+    - name: Sleep, wait for containers to start up
+      run: sleep 2
+
+    - name: Run integration tests
+      shell: bash -l {0}
+      run: python -m pytest -k "test_heartbeat or test_missing_data"
+
+    - name: Bring down docker containers
+      run: docker-compose -f tests/integration/docker-compose-rpm.yml down

Dockerfile

@@ -1,31 +1,22 @@
-FROM --platform=linux/amd64 centos:7 as package
+FROM registry.access.redhat.com/ubi9/ubi
 
-RUN yum install -y make rpm-build
-# lots of packages don't exist in rhel7
-# the Makefile will print lots of warnings as a result
-
-WORKDIR /app
+RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN dnf install -y make rpm-build python3-build python3-pip
 
 COPY scripts /app/scripts
 COPY configuration /app/configuration
 COPY postprocessing /app/postprocessing
+COPY pyproject.toml /app/
+COPY rpmbuild.sh /app/
+COPY README.md /app/
+COPY LICENSE.rst /app/
 COPY SPECS /app/SPECS
-COPY Makefile /app/
-
 
 RUN mkdir -p /root/rpmbuild/SOURCES
 
-RUN make rpm || exit 1
-
-FROM --platform=linux/amd64 centos:7 as app
-COPY --from=package /root/rpmbuild/RPMS/noarch/postprocessing-*-1.noarch.rpm /
-
-
-RUN curl http://packages.sns.gov/distros/rhel/7/sns/sns.repo -o /etc/yum.repos.d/sns.repo
-RUN yum install -y epel-release
-RUN yum updateinfo
+RUN cd /app && ./rpmbuild.sh || exit 1
 
-RUN yum install -y /postprocessing-*-1.noarch.rpm || exit 1
+RUN dnf install -y /root/rpmbuild/RPMS/noarch/postprocessing*.noarch.rpm || exit 1
 
 # This configuration allows it to run with docker-compose from https://github.com/neutrons/data_workflow
 COPY configuration/post_process_consumer.conf.development /etc/autoreduce/post_processing.conf

SPECS/postprocessing.spec

@@ -1,48 +1,49 @@
-Summary: postprocessing
-Name: postprocessing
-Version: 2.6
-Release: 1
+%global srcname postprocessing
+%global summary postprocessing
+%define release 1
+
+Name: %{srcname}
+Version: 2.8.0
+Release: %{release}%{?dist}
+Summary: %{summary}
+
+License: MIT
+URL: https://github.com/neutrons/post_processing_agent
+Source: %{srcname}-%{version}.tar.gz
 Group: Applications/Engineering
-prefix: /opt/postprocessing
-BuildRoot: %{_tmppath}/%{name}
+
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 BuildArch: noarch
-License: MIT
-Source: postprocessing.tgz
-Requires: libNeXus.so.0()(64bit) libc.so.6()(64bit) libc.so.6(GLIBC_2.2.5)(64bit)
-Requires: nexus-python
-Requires: python-twisted-core
-Requires: python-twisted-web
-Requires: python-twisted-words
-Requires: python-stompest
-Requires: python2-stompest-async
-Requires: python-requests
-%define debug_package %{nil}
-%define site_packages %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
+BuildRequires:  python%{python3_pkgversion}-pip
+Requires: python%{python3_pkgversion}
+Requires: python%{python3_pkgversion}-requests
+Requires: python%{python3_pkgversion}-stomppy
+
+prefix: /opt/postprocessing
+
+%define site_packages %(python3 -c "import site; print(site.getsitepackages()[-1])")
 
 %description
 Post-processing agent to automatically catalog and reduce neutron data
 
 %prep
-%setup -q -n %{name}
+%autosetup -p1 -n %{srcname}-%{version}
 
 %build
 
 %install
-rm -rf %{buildroot}
-mkdir -p %{buildroot}%{_sysconfdir}
-mkdir -p %{buildroot}%{_bindir}
-mkdir -p %{buildroot}%{prefix}
+%{python3} -m pip install --target %{buildroot}%{prefix} --no-deps .
+mv %{buildroot}%{prefix}/postprocessing/queueProcessor.py %{buildroot}%{prefix}
+mkdir -p %{buildroot}%{prefix}/log
 mkdir -p %{buildroot}%{site_packages}
+echo %{prefix} > %{buildroot}%{site_packages}/postprocessing.pth
 mkdir -p %{buildroot}/var/log/SNS_applications
-make prefix="%{buildroot}%{prefix}" installed_prefix="%{prefix}" site_packages="%{buildroot}%{site_packages}" sysconfig="%{buildroot}%{_sysconfdir}/autoreduce" bindir="%{buildroot}/%{_bindir}" install
-
-%post
-chgrp snswheel %{_sysconfdir}/autoreduce/post_processing.conf
-chown -R snsdata %{_sysconfdir}/autoreduce
-chown -R snsdata %{prefix}
-chown -R snsdata /var/log/SNS_applications
 
 %files
-%attr(755, -, -) %{prefix}/
-%attr(755, -, -) /var/log/SNS_applications
-%attr(755, -, -) %{site_packages}/postprocessing.pth
+%doc README.md
+%license LICENSE.rst
+%{prefix}/*
+%attr(755, -, -) %{prefix}/scripts
+%attr(755, -, -) %{prefix}/queueProcessor.py
+%{site_packages}/postprocessing.pth
+/var/log/SNS_applications

configuration/post_process_consumer.conf.development

@@ -6,7 +6,7 @@
     "amq_pwd": "icat",
     "sw_dir": "/opt/postprocessing",
     "python_dir": "/opt/postprocessing/postprocessing",
-    "start_script": "python",
+    "start_script": "python3",
     "task_script": "PostProcessAdmin.py",
     "task_script_queue_arg": "-q",
     "task_script_data_arg": "-d",
@@ -20,5 +20,6 @@
     "dev_output_dir": "",
     "reduction_data_ready": "REDUCTION.DATA_READY",
     "communication_only": 0,
-    "max_procs": 5
+    "max_procs": 5,
+    "processors": ["reduction_processor.ReductionProcessor"]
 }

configuration/post_process_consumer.conf.local

@@ -6,7 +6,7 @@
     "amq_pwd": "",
     "sw_dir": "/opt/postprocessing",
     "python_dir": "/opt/postprocessing/postprocessing",
-    "start_script": "python",
+    "start_script": "python3",
     "task_script": "PostProcessAdmin.py",
     "task_script_queue_arg": "-q",
     "task_script_data_arg": "-d",

environment.yml

@@ -11,5 +11,4 @@ dependencies:
   - pytest-mock
   - requests=2.25
   - stomp.py=7
-# needed for wheel - add when switching to python3
-#  - build
+  - build

postprocessing/__init__.py

@@ -1 +1 @@
-__version__ = "2.1.0"
+__version__ = "2.8.0"

postprocessing/processors/create_reduction_script_processor.py

@@ -16,7 +16,7 @@ def __init__(self, data, conf, send_function):
         @param conf: configuration object
         @param send_function: function to call to send AMQ messages
         """
-        super(CreateReductionScriptProcessor, self).__init__(data, conf, send_function)
+        super().__init__(data, conf, send_function)
 
     def __call__(self):
         """

postprocessing/processors/reduction_processor.py

@@ -23,7 +23,7 @@ def __init__(self, data, conf, send_function):
         @param conf: configuration object
         @param send_function: function to call to send AMQ messages
         """
-        super(ReductionProcessor, self).__init__(data, conf, send_function)
+        super().__init__(data, conf, send_function)
 
     def __call__(self):
         """