Skip to content

Commit

Permalink
Update
Browse files Browse the repository at this point in the history
  • Loading branch information
ces committed Jan 12, 2024
2 parents 9a377ca + 0521f96 commit 3f40f7c
Show file tree
Hide file tree
Showing 44 changed files with 11,824 additions and 406 deletions.
1 change: 1 addition & 0 deletions .github/workflows/perlbrew.sha256
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c3996e4fae37a0ae01839cdd73752fb7b17e81bac2a8b39712463a7d518c4945 perlbrew.sh
123 changes: 123 additions & 0 deletions .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
name: "Unit tests"

on: [push, pull_request]

jobs:
build:
runs-on: ubuntu-latest

defaults:
run:
shell: bash -l -e -o pipefail {0}

env:
PERL_CACHE: ~/perl5 # Perlbrew and CPAN modules installed here, cached
NPG_LIB: ~/perl5npg # NPG modules installed here, not cached
WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic"
CONDA_TEST_ENV: test-environment
WTSI_NPG_GITHUB_URL: https://github.com/wtsi-npg
WTSI_NPG_BUILD_BRANCH: ${{ github.base_ref || github.ref }}

strategy:
matrix:
perl: ["5.26.3", "5.34.1"]

steps:
- uses: actions/checkout@v3

- name: "Install OS dependencies"
run: |
sudo apt-get update
# https://github.com/actions/runner-images/issues/2139
sudo apt-get remove -y nginx libgd3
sudo apt-get install -y libgd-dev uuid-dev libgd-text-perl
- name: "Initialize Miniconda"
run: |
echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile"
- name: "Install Conda packages"
run: |
conda config --prepend pkgs_dirs ~/conda/pkgs
conda config --prepend envs_dirs ~/conda/envs
conda config --set auto_update_conda False
conda config --prepend channels "$WSI_CONDA_CHANNEL"
conda info
conda create -y -n "$CONDA_TEST_ENV"
conda install -y -n "$CONDA_TEST_ENV" baton
conda install -y -n "$CONDA_TEST_ENV" samtools
- name: "Cache Perl"
id: cache-perl
uses: actions/cache@v3
with:
path: ${{ env.PERL_CACHE }}
key: ${{ runner.os }}-${{ matrix.perl }}-perl

- name: "Install Perlbrew"
if: steps.cache-perl.outputs.cache-hit != 'true'
run: |
curl -sSL https://install.perlbrew.pl -o perlbrew.sh
sha256sum -c .github/workflows/perlbrew.sha256
export PERLBREW_ROOT=${{ env.PERL_CACHE }}
sh perlbrew.sh
source ${{ env.PERL_CACHE }}/etc/bashrc
perlbrew available
perlbrew install --notest perl-${{ matrix.perl }}
perlbrew use perl-${{ matrix.perl }}
perlbrew install-cpanm
- name: "Initialize Perlbrew"
run: |
echo "source ${{ env.PERL_CACHE }}/etc/bashrc" >> "$HOME/.bash_profile"
- name: "Install Perl dependencies"
run: |
cpanm --local-lib=${{ env.PERL_CACHE }} local::lib
eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB")
eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib)
cpanm --quiet --notest Module::Build
cpanm --quiet --notest Alien::Tidyp
cpanm --quiet --notest Net::SSLeay
cpanm --quiet --notest https://github.com/chapmanb/vcftools-cpan/archive/v0.953.tar.gz
./scripts/install_wsi_dependencies.sh "$NPG_LIB" \
perl-dnap-utilities \
perl-irods-wrap \
ml_warehouse \
npg_tracking \
npg_seq_common \
npg_qc \
npg_irods
cpanm --installdeps --notest .
- name: "Log install failure"
if: ${{ failure() }}
run: |
find ~/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20 {} \;
- name: "Archive CPAN logs on failure"
if: ${{ failure() }}
uses: actions/upload-artifact@v2
with:
name: cpan_log
path: ~/.cpanm/work/*/build.log
retention-days: 5

- name: "Run tests"
run: |
conda activate "$CONDA_TEST_ENV"
conda info --envs
eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib)
eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB")
export TEST_AUTHOR=1
perl Build.PL
./Build test --verbose
./Build install
74 changes: 0 additions & 74 deletions .github/workflows/testing_and_building_repo.yml

This file was deleted.

1 change: 0 additions & 1 deletion Build.PL
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ my $builder = $class->new(
'npg_qc::illumina::interop::parser' => 0,
'st::api::lims' => 0,
'st::api::lims::ml_warehouse' => 0,
'npg::api::request' => 0,
'npg::samplesheet' => 0,
'WTSI::DNAP::Utilities::Loggable' => 0,
'WTSI::DNAP::Warehouse::Schema' => 0,
Expand Down
57 changes: 57 additions & 0 deletions Changes
Original file line number Diff line number Diff line change
@@ -1,9 +1,66 @@
LIST OF CHANGES
---------------


- allow samtools ampliconclip to run for GBS products where primer panel
has primer bed file in the repository

release 67.1.0
- Fix typo in analysis specific overrides for bwa_als_se mapping to bwa0_6
- Add in use of autosome target regions for BGE libraries in seq_alignment
- Add 'merge_by_library' pipeline boolean option. This options is automatially
activated for NovaSeqX platform. It triggers a discovery of sets of data
that belong to the same libraries. If cases like this are found, the pipeline
is instructed, at the secondary analysis stage, to process this data as a
single entity. In practice, if the same pool is sequenced in more than one
lane of the run, sample data for the pool are merged across these lanes.
The 'discovery' part of the algorithm is implemented in
https://github.com/wtsi-npg/npg_tracking/pull/772
- Removed provisions for inline indexes
- Removed a check for rapid runs when deciding whether to merge
- Stop warnings about an undefined value when writing to the log from
npg_pipeline::function::seq_alignment
- Some tests were creating test data in the package's source tree. These
activities are redirected to temporary files and directories in /tmp
- Removed listing of non-existing files from MANIFEST
- Removed superfluous dependency on now removed st::api::request
- Added a test to expose a problem with ref cache, which is resolved by
https://github.com/wtsi-npg/npg_tracking/pull/761

release 67.0.0
- Turn off spatial filter QC check for NovaSeqX
- Switch to Perlbrew to obtain multiple Perl versions
- Remove npg_ml_warehouse dependency
- Enhance README with more context
- Improve Markdown format consistency
- Add images of DAGs, add links, fix a typo
- Add info on data intensive P4 based steps

release 66.0.0
- small tweak to seq_alignment so GbS samples with no study ref do not fail
- switch off spatial filter for NovaSeqX
- for NovaSeqX, default RNA analysis should be STAR

release 65.1.0
- ensure per-product archival for NovaSeqX data
- runs with data analysed on-board are not deletable
- ensure per product archival impacts hierarchy
- per product publish doc and variable name fixup
- when platform_NovaSeqX is detected, set p4 parameter i2b_nocall_qual_switch
to "on"
- avoid archiving "Analysis" hierarchy with XML and InterOp

release 65.0.0
- remove wr limit of p4s1 to specific flavor

release 64.0.1
- set p4 parameter to fix bug in bwa-mem2 + non-consented human split

release 64.0.0
- add bwa_mem2 flag to options.pm to allow override of default bwa analyses at pipeline invocation
- update seq_alignment to recognise the bwa_mem2 flag and also default to bwa_mem2 for NovaSeqX platform
>>>>>>> upstream/devel

release 63.3.0
- force no_target_alignment for haplotag libraries

Expand Down
10 changes: 8 additions & 2 deletions MANIFEST
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
.github/workflows/testing_and_building_repo.yml
bin/npg_pipeline_analysis_runner
bin/npg_pipeline_archival_runner
bin/npg_pipeline_central
Expand Down Expand Up @@ -78,7 +77,6 @@ lib/npg_pipeline/validation/s3.pm
MANIFEST This list of files
README
README.md
scripts/install_npg_perl_dependencies.sh
scripts/jgf2gml
t/00-critic.t
t/00-distribution.t
Expand Down Expand Up @@ -1042,6 +1040,13 @@ t/data/novaseq/210415_A00971_0162_AHNNTMDSXY/Data/Intensities/BAM_basecalls_2021
t/data/novaseq/210415_A00971_0162_AHNNTMDSXY/Data/Intensities/BAM_basecalls_20210417-080715/metadata_cache_37416/lane_3.taglist
t/data/novaseq/210415_A00971_0162_AHNNTMDSXY/Data/Intensities/BAM_basecalls_20210417-080715/metadata_cache_37416/lane_4.taglist
t/data/novaseq/210415_A00971_0162_AHNNTMDSXY/Data/Intensities/BAM_basecalls_20210417-080715/metadata_cache_37416/samplesheet_37416.csv
t/data/novaseqx/47539/README.md
t/data/novaseqx/47539/RunInfo.xml
t/data/novaseqx/47539/RunParameters.xml
t/data/novaseqx/47539/samplesheet_47539.csv
t/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3/RunInfo.xml
t/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3/RunParameters.xml
t/data/novaseqx/20231017_LH00210_0012_B22FCNFLT3/samplesheet_47995.csv
t/data/p4_stage1_analysis/1234_samplesheet.csv
t/data/p4_stage1_analysis/TileMetricsOut.bin
t/data/portable_pipelines/samplesheet4archival_all_controls.csv
Expand Down Expand Up @@ -1110,6 +1115,7 @@ t/data/run_params/runParameters.hiseq.xml
t/data/run_params/runParameters.hiseqx.upgraded.xml
t/data/run_params/runParameters.miseq.xml
t/data/run_params/runParameters.novaseq.xml
t/data/run_params/RunParameters.novaseqx.xml
t/data/runfolder/archive/1234_2.bam
t/data/runfolder/archive/1234_2_human.bam
t/data/runfolder/archive/1234_3.bam
Expand Down
Loading

0 comments on commit 3f40f7c

Please sign in to comment.