diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 474a5b6..d3257cd 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -3,8 +3,6 @@ name: CI on: pull_request: push: - branches: main - tags: ['v*'] workflow_dispatch: @@ -13,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up MicroMamba uses: mamba-org/setup-micromamba@v1 @@ -39,7 +37,7 @@ jobs: needs: tests steps: - name: Checkout Repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up MicroMamba uses: mamba-org/setup-micromamba@v1 @@ -86,42 +84,44 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} -# disable until spec file is updated to python 3 -# rpm: -# runs-on: ubuntu-latest -# -# steps: -# - name: Checkout Repository -# uses: actions/checkout@v2 -# -# - name: Build RPM inside Docker -# run: | -# docker build --tag postprocess --target=package -f Dockerfile . -# fname=`docker run -v $(pwd):/store postprocess ls /root/rpmbuild/RPMS/noarch` -# docker run -v `pwd`:/store postprocess mv /root/rpmbuild/RPMS/noarch/$fname /store -# one=${fname#*postprocessing-} -# two=${one%.noarch*} -# echo "::set-output name=version::$two" -# echo "::set-output name=fname::$fname" -# continue-on-error: false - -# TODO: uncomment this once we have switched to python3 and can conda install the module build -# wheel: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout Repository -# uses: actions/checkout@v2 -# -# - name: Set up MicroMamba -# uses: mamba-org/setup-micromamba@v1 -# with: -# environment-file: environment.yml -# condarc: | -# channels: -# - conda-forge -# - default -# cache-environment: true -# -# - name: Build wheel -# shell: bash -l {0} -# run: python -m build --wheel + rpm: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Build RPM inside Docker + run: | + docker build --tag postprocess -f Dockerfile . + fname=`docker run --name postprocess postprocess ls /root/rpmbuild/RPMS/noarch` + docker cp postprocess:/root/rpmbuild/RPMS/noarch/$fname . + one=${fname#*postprocessing-} + two=${one%.noarch*} + echo "::set-output name=version::$two" + echo "::set-output name=fname::$fname" + continue-on-error: false + + # now run a couple integration test with RPM installed package + - name: Set up MicroMamba + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: environment.yml + condarc: | + channels: + - conda-forge + - default + cache-environment: true + + - name: Start docker containers + run: docker-compose -f tests/integration/docker-compose-rpm.yml up --build -d + + - name: Sleep, wait for containers to start up + run: sleep 2 + + - name: Run integration tests + shell: bash -l {0} + run: python -m pytest -k "test_heartbeat or test_missing_data" + + - name: Bring down docker containers + run: docker-compose -f tests/integration/docker-compose-rpm.yml down diff --git a/Dockerfile b/Dockerfile index 3c6f51f..75ccdf8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,22 @@ -FROM --platform=linux/amd64 centos:7 as package +FROM registry.access.redhat.com/ubi9/ubi -RUN yum install -y make rpm-build -# lots of packages don't exist in rhel7 -# the Makefile will print lots of warnings as a result - -WORKDIR /app +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm +RUN dnf install -y make rpm-build python3-build python3-pip COPY scripts /app/scripts COPY configuration /app/configuration COPY postprocessing /app/postprocessing +COPY pyproject.toml /app/ +COPY rpmbuild.sh /app/ +COPY README.md /app/ +COPY LICENSE.rst /app/ COPY SPECS /app/SPECS -COPY Makefile /app/ - RUN mkdir -p /root/rpmbuild/SOURCES -RUN make rpm || exit 1 - -FROM --platform=linux/amd64 centos:7 as app -COPY --from=package /root/rpmbuild/RPMS/noarch/postprocessing-*-1.noarch.rpm / - - -RUN curl http://packages.sns.gov/distros/rhel/7/sns/sns.repo -o /etc/yum.repos.d/sns.repo -RUN yum install -y epel-release -RUN yum updateinfo +RUN cd /app && ./rpmbuild.sh || exit 1 -RUN yum install -y /postprocessing-*-1.noarch.rpm || exit 1 +RUN dnf install -y /root/rpmbuild/RPMS/noarch/postprocessing*.noarch.rpm || exit 1 # This configuration allows it to run with docker-compose from https://github.com/neutrons/data_workflow COPY configuration/post_process_consumer.conf.development /etc/autoreduce/post_processing.conf diff --git a/Makefile b/Makefile deleted file mode 100644 index 80f4b6a..0000000 --- a/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -prefix := /opt/postprocessing -#prefix := /sw/fermi/autoreduce/postprocessing -sysconfig := /etc/autoreduce -bindir := /usr/bin -site_packages := `python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"` - -# The installed prefix may be different from the prefix above while building an RPM -installed_prefix = $(prefix) - -all: postproc - -check: - # Check dependencies - @python -c "import requests" || echo "ERROR: Need requests: easy_install requests" - @python -c "import stompest" || echo "ERROR: Need stompest: easy_install stompest" - @python -c "import stompest.async" || echo "ERROR: Need stompest.async: easy_install stompest.async" - @python -c "import suds" || echo "ERROR: Need suds: easy_install suds" - @python -c "import nxs" || echo "ERROR: Need nexus: http://download.nexusformat.org/kits/" - @python -c "import twisted" || echo "ERROR: Need twisted: easy_install twisted" - - @test -f configuration/post_process_consumer.conf || echo -e "\n===> SET UP configuration/post_process_consumer.conf BEFORE INSTALLATION\n"; - -install: config postproc - -install/isolated: config/isolated postproc - -config: - # Make sure the directories exist - @test -d $(sysconfig) || mkdir -m 0755 -p $(sysconfig) - echo "$(installed_prefix)" > configuration/postprocessing.pth - install -m 664 configuration/postprocessing.pth $(site_packages)/postprocessing.pth - -config/isolated: - # Simplified configuration for isolated installation - usually remote systems - @test -d $(prefix)/configuration || mkdir -m 0755 $(prefix)/configuration - install -m 664 configuration/post_process_consumer.conf $(prefix)/configuration/post_processing.conf - -postproc: check - # Make sure the directories exist - @test -d $(prefix) || mkdir -m 0755 $(prefix) - @test -d $(prefix)/postprocessing || mkdir -m 0755 $(prefix)/postprocessing - @test -d $(prefix)/postprocessing/processors || mkdir -m 0755 $(prefix)/postprocessing/processors - @test -d $(prefix)/log || mkdir -m 0775 $(prefix)/log - @test -d $(prefix)/scripts || mkdir -m 0755 $(prefix)/scripts - - # Install application code - install -m 755 postprocessing/__init__.py $(prefix)/postprocessing/__init__.py - install -m 755 postprocessing/Consumer.py $(prefix)/postprocessing/Consumer.py - install -m 755 postprocessing/Configuration.py $(prefix)/postprocessing/Configuration.py - install -m 755 postprocessing/PostProcessAdmin.py $(prefix)/postprocessing/PostProcessAdmin.py - install -m 755 postprocessing/reduction_script_writer.py $(prefix)/postprocessing/reduction_script_writer.py - install -m 755 postprocessing/publish_plot.py $(prefix)/postprocessing/publish_plot.py - install -m 755 scripts/mantidpython.py $(prefix)/scripts/mantidpython.py - install -m 755 scripts/ar-report.py $(prefix)/scripts/ar-report.py - install -m 755 postprocessing/queueProcessor.py $(prefix)/queueProcessor.py - install -m 755 postprocessing/processors/__init__.py $(prefix)/postprocessing/processors - install -m 755 postprocessing/processors/base_processor.py $(prefix)/postprocessing/processors - install -m 755 postprocessing/processors/test_processor.py $(prefix)/postprocessing/processors - install -m 755 postprocessing/processors/oncat_processor.py $(prefix)/postprocessing/processors - install -m 755 postprocessing/processors/oncat_reduced_processor.py $(prefix)/postprocessing/processors - install -m 755 postprocessing/processors/calvera_processor.py $(prefix)/postprocessing/processors - install -m 755 postprocessing/processors/job_handling.py $(prefix)/postprocessing/processors -rpm: - @echo "Creating RPMs" - @rm -rf build - mkdir build - mkdir build/postprocessing - cp -pr Makefile build/postprocessing - cp -pr scripts build/postprocessing - cp -pr configuration build/postprocessing - cp -pr postprocessing build/postprocessing - - cd build;tar -czf ~/rpmbuild/SOURCES/postprocessing.tgz postprocessing - rpmbuild -ba ./SPECS/postprocessing.spec - -wheel: - @echo "Creating wheel" - @rm -rf build dist - python -m build --wheel - -links: - ln -sf /usr/local/sis/DataArchiving/scripts/oncat_datafile_ingest.py /opt/postprocessing/scripts/oncat_ingest.py - ln -sf /usr/local/sis/DataArchiving/scripts/oncat_reduction_ingest.py /opt/postprocessing/scripts/oncat_reduced_ingest.py - -.PHONY: check -.PHONY: install -.PHONY: postproc -.PHONY: wheel diff --git a/SPECS/postprocessing.spec b/SPECS/postprocessing.spec index 4b40b7a..70d691f 100644 --- a/SPECS/postprocessing.spec +++ b/SPECS/postprocessing.spec @@ -1,48 +1,49 @@ -Summary: postprocessing -Name: postprocessing -Version: 2.6 -Release: 1 +%global srcname postprocessing +%global summary postprocessing +%define release 1 + +Name: %{srcname} +Version: 2.8.0 +Release: %{release}%{?dist} +Summary: %{summary} + +License: MIT +URL: https://github.com/neutrons/post_processing_agent +Source: %{srcname}-%{version}.tar.gz Group: Applications/Engineering -prefix: /opt/postprocessing -BuildRoot: %{_tmppath}/%{name} + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot BuildArch: noarch -License: MIT -Source: postprocessing.tgz -Requires: libNeXus.so.0()(64bit) libc.so.6()(64bit) libc.so.6(GLIBC_2.2.5)(64bit) -Requires: nexus-python -Requires: python-twisted-core -Requires: python-twisted-web -Requires: python-twisted-words -Requires: python-stompest -Requires: python2-stompest-async -Requires: python-requests -%define debug_package %{nil} -%define site_packages %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") +BuildRequires: python%{python3_pkgversion}-pip +Requires: python%{python3_pkgversion} +Requires: python%{python3_pkgversion}-requests +Requires: python%{python3_pkgversion}-stomppy + +prefix: /opt/postprocessing + +%define site_packages %(python3 -c "import site; print(site.getsitepackages()[-1])") %description Post-processing agent to automatically catalog and reduce neutron data %prep -%setup -q -n %{name} +%autosetup -p1 -n %{srcname}-%{version} %build %install -rm -rf %{buildroot} -mkdir -p %{buildroot}%{_sysconfdir} -mkdir -p %{buildroot}%{_bindir} -mkdir -p %{buildroot}%{prefix} +%{python3} -m pip install --target %{buildroot}%{prefix} --no-deps . +mv %{buildroot}%{prefix}/postprocessing/queueProcessor.py %{buildroot}%{prefix} +mkdir -p %{buildroot}%{prefix}/log mkdir -p %{buildroot}%{site_packages} +echo %{prefix} > %{buildroot}%{site_packages}/postprocessing.pth mkdir -p %{buildroot}/var/log/SNS_applications -make prefix="%{buildroot}%{prefix}" installed_prefix="%{prefix}" site_packages="%{buildroot}%{site_packages}" sysconfig="%{buildroot}%{_sysconfdir}/autoreduce" bindir="%{buildroot}/%{_bindir}" install - -%post -chgrp snswheel %{_sysconfdir}/autoreduce/post_processing.conf -chown -R snsdata %{_sysconfdir}/autoreduce -chown -R snsdata %{prefix} -chown -R snsdata /var/log/SNS_applications %files -%attr(755, -, -) %{prefix}/ -%attr(755, -, -) /var/log/SNS_applications -%attr(755, -, -) %{site_packages}/postprocessing.pth +%doc README.md +%license LICENSE.rst +%{prefix}/* +%attr(755, -, -) %{prefix}/scripts +%attr(755, -, -) %{prefix}/queueProcessor.py +%{site_packages}/postprocessing.pth +/var/log/SNS_applications diff --git a/configuration/post_process_consumer.conf.development b/configuration/post_process_consumer.conf.development index 50e0e01..6c77512 100644 --- a/configuration/post_process_consumer.conf.development +++ b/configuration/post_process_consumer.conf.development @@ -6,7 +6,7 @@ "amq_pwd": "icat", "sw_dir": "/opt/postprocessing", "python_dir": "/opt/postprocessing/postprocessing", - "start_script": "python", + "start_script": "python3", "task_script": "PostProcessAdmin.py", "task_script_queue_arg": "-q", "task_script_data_arg": "-d", @@ -20,5 +20,6 @@ "dev_output_dir": "", "reduction_data_ready": "REDUCTION.DATA_READY", "communication_only": 0, - "max_procs": 5 + "max_procs": 5, + "processors": ["reduction_processor.ReductionProcessor"] } \ No newline at end of file diff --git a/configuration/post_process_consumer.conf.local b/configuration/post_process_consumer.conf.local index 4d95f1b..426c2fd 100644 --- a/configuration/post_process_consumer.conf.local +++ b/configuration/post_process_consumer.conf.local @@ -6,7 +6,7 @@ "amq_pwd": "", "sw_dir": "/opt/postprocessing", "python_dir": "/opt/postprocessing/postprocessing", - "start_script": "python", + "start_script": "python3", "task_script": "PostProcessAdmin.py", "task_script_queue_arg": "-q", "task_script_data_arg": "-d", diff --git a/environment.yml b/environment.yml index 323f6bd..0184112 100644 --- a/environment.yml +++ b/environment.yml @@ -11,5 +11,4 @@ dependencies: - pytest-mock - requests=2.25 - stomp.py=7 -# needed for wheel - add when switching to python3 -# - build + - build diff --git a/postprocessing/__init__.py b/postprocessing/__init__.py index 9aa3f90..892994a 100644 --- a/postprocessing/__init__.py +++ b/postprocessing/__init__.py @@ -1 +1 @@ -__version__ = "2.1.0" +__version__ = "2.8.0" diff --git a/postprocessing/processors/create_reduction_script_processor.py b/postprocessing/processors/create_reduction_script_processor.py index 6a38c10..349a797 100644 --- a/postprocessing/processors/create_reduction_script_processor.py +++ b/postprocessing/processors/create_reduction_script_processor.py @@ -16,7 +16,7 @@ def __init__(self, data, conf, send_function): @param conf: configuration object @param send_function: function to call to send AMQ messages """ - super(CreateReductionScriptProcessor, self).__init__(data, conf, send_function) + super().__init__(data, conf, send_function) def __call__(self): """ diff --git a/postprocessing/processors/reduction_processor.py b/postprocessing/processors/reduction_processor.py index 9f84271..8ade85b 100644 --- a/postprocessing/processors/reduction_processor.py +++ b/postprocessing/processors/reduction_processor.py @@ -23,7 +23,7 @@ def __init__(self, data, conf, send_function): @param conf: configuration object @param send_function: function to call to send AMQ messages """ - super(ReductionProcessor, self).__init__(data, conf, send_function) + super().__init__(data, conf, send_function) def __call__(self): """ diff --git a/pyproject.toml b/pyproject.toml index 2fde7c9..cc4b416 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,27 @@ +[project] +name = "postprocessing" +description = "Post-processing agent to automatically catalog and reduce neutron data" +version = "2.8.0" +requires-python = ">=3.9" +dependencies = [ + "requests", + "plotly", + "stomp.py" +] +license = { text = "MIT" } + +[project.urls] +homepage = "https://github.com/neutrons/post_processing_agent" + +[build-system] +requires = [ + "setuptools", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["postprocessing", "postprocessing.processors", "scripts"] + [tool.ruff] line-length = 120 # https://beta.ruff.rs/docs/rules/ diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8663b3e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -h5py -# mantidframework # TODO: find the appropriate version -git+https://github.com/nexusformat/python-nxs.git@master -plotly -stompest -twisted -requests - diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index adc04b4..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -build -flake8 -numpy -pytest diff --git a/rpmbuild.sh b/rpmbuild.sh new file mode 100755 index 0000000..66994c9 --- /dev/null +++ b/rpmbuild.sh @@ -0,0 +1,7 @@ +#!/bin/bash +rm -rf dist postprocessing.egg-info +python3 -m build --sdist +cp dist/postprocessing-*.tar.gz ~/rpmbuild/SOURCES/ +rpmbuild -ba SPECS/postprocessing.spec +cp ~/rpmbuild/RPMS/noarch/postprocessing-*-*.*.noarch.rpm dist/ +cp ~/rpmbuild/SRPMS/postprocessing-*-*.*.src.rpm dist/ diff --git a/scripts/mantidpython.py b/scripts/mantidpython.py index d6ebc07..8615273 100644 --- a/scripts/mantidpython.py +++ b/scripts/mantidpython.py @@ -1,4 +1,4 @@ -#!/bin/env python3 +#!/usr/bin/env python3 import os import re import subprocess diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a42209d..0000000 --- a/setup.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[metadata] -name = postprocessing -version = 2.6 -description = Post-processing agent to automatically catalog and reduce neutron data -long_description = file: README.md, LICENSE.rst -url = https://github.com/neutrons/post_processing_agent -license = MIT -classifiers = - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 - Topic :: Software Development :: Libraries :: Python Modules - Topic :: System :: Distributed Computing - Topic :: System :: Networking - -[options] -python_requires = >= 2.7 -packages = find: -install_requires = - requests - stompest - twisted -include_package_data = true -scripts = - scripts/ar-report.py - scripts/mantidpython.py - -[options.packages.find] -exclude = - tests* - -[options.extras_require] -tests = pytest - -[aliases] -test = pytest diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/integration/Dockerfile b/tests/integration/Dockerfile index d0846c0..5b5dbaf 100644 --- a/tests/integration/Dockerfile +++ b/tests/integration/Dockerfile @@ -3,7 +3,6 @@ FROM registry.access.redhat.com/ubi9/ubi RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm RUN dnf update RUN dnf install -y python3-requests python3-stomppy python3-pip python3-coverage -RUN python3 -m pip install plotly COPY postprocessing /opt/postprocessing/postprocessing COPY tests/integration/post_processing.conf /etc/autoreduce/post_processing.conf diff --git a/tests/integration/docker-compose-rpm.yml b/tests/integration/docker-compose-rpm.yml new file mode 100644 index 0000000..403783b --- /dev/null +++ b/tests/integration/docker-compose-rpm.yml @@ -0,0 +1,15 @@ +version: '3.8' + +services: + + post_processing_agent: + build: + context: ../.. + dockerfile: ./Dockerfile + + activemq: + image: rmohr/activemq + hostname: activemq + ports: + - 8161:8161 + - 61613:61613 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/unit/postprocessing/__init__.py b/tests/unit/postprocessing/__init__.py deleted file mode 100644 index e69de29..0000000