diff --git a/.circleci/config.yml b/.circleci/config.yml index c2e46d072..e20d67762 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ common_jobs: &common_jobs - integration-tests: matrix: parameters: - testing_profile: [ "singularity-tests", "bitbucket-tests" ] + testing_profile: [ "singularity-tests", "bitbucket-tests", "toil-integration-tests" ] <<: *common_filters <<: *slack_context requires: @@ -198,7 +198,8 @@ jobs: - run: name: findDevelopSnapshot command: | - ./mvnw versions:update-properties -Dincludes=io.dockstore:* -DallowSnapshots + # grab the latest snapshot version that doesn't seem to correspond to a feature branch or similar + ./mvnw versions:update-properties -Dincludes="io.dockstore:*" -Dmaven.version.ignore=\.*-\.*-SNAPSHOT -DallowSnapshots - run: name: force update to latest snapshot versions, # review this step to see what versions were used, also see https://stackoverflow.com/questions/29020716/maven-what-does-u-update-snapshots-really-do command: | diff --git a/.github/collab-mvn-settings.xml b/.github/collab-mvn-settings.xml deleted file mode 100644 index 711bbdc0d..000000000 --- a/.github/collab-mvn-settings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - central - dockstore-bot - ${env.COLLAB_DEPLOY_TOKEN} - - - diff --git a/.github/snapshot-mvn-settings.xml b/.github/snapshot-mvn-settings.xml deleted file mode 100644 index ce26d73be..000000000 --- a/.github/snapshot-mvn-settings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - snapshots - dockstore-snapshot-bot - ${env.SNAPSHOT_DEPLOY_TOKEN} - - - diff --git a/.github/workflows/deploy_artifacts.yml b/.github/workflows/deploy_artifacts.yml new file mode 100644 index 000000000..13390ac1b --- /dev/null +++ b/.github/workflows/deploy_artifacts.yml @@ -0,0 +1,16 @@ +name: Deploy artifacts + +on: + push: + tags: + - '**' + branches-ignore: + - 'master' + - 'release/**' + - 'hotfix/**' + - 'dependabot/**' + +jobs: + build: + uses: dockstore/workflow-actions/.github/workflows/deploy_artifacts.yaml@main + secrets: inherit diff --git a/.github/workflows/deploy_snapshot.yml b/.github/workflows/deploy_snapshot.yml deleted file mode 100644 index e7faceaef..000000000 --- a/.github/workflows/deploy_snapshot.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Deploy develop snapshot - -on: - push: - branches: - - develop - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v4 - # Step that does that actual cache save and restore - - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up JDK - uses: actions/setup-java@v4 - with: - java-version: '17.0.4+8' - distribution: 'adopt' - - name: install git secrets - run: | - wget --no-verbose -O git-secrets-1.3.0.tar.gz https://github.com/awslabs/git-secrets/archive/1.3.0.tar.gz - tar -zxf git-secrets-1.3.0.tar.gz - cd git-secrets-1.3.0 - sudo make install - - - name: Deploy with mvnw - run: ./mvnw --batch-mode deploy -ntp -s .github/snapshot-mvn-settings.xml -DskipTests - env: - SNAPSHOT_DEPLOY_TOKEN: ${{ secrets.SNAPSHOT_DEPLOY_TOKEN }} diff --git a/.github/workflows/deploy_tagged.yml b/.github/workflows/deploy_tagged.yml deleted file mode 100644 index 614cd5d6b..000000000 --- a/.github/workflows/deploy_tagged.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Tagged release deploy - -on: - workflow_dispatch: - inputs: - changelist: - description: 'patch and pre-release metadata' - required: true - default: '.0-alpha.1' - - -jobs: - call-reusable-tagged-release: - uses: dockstore/workflow-actions/.github/workflows/deploy_tagged.yaml@main - with: - changelist: ${{ inputs.changelist }} - secrets: inherit diff --git a/.gitignore b/.gitignore index 5bc1f379d..9be685cad 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .idea *.class *.swp +.DS_Store # Contains SQL statements to initialize testing DB with a confidential user and tokens # Contains SQL statements to initialize testing DB with a confidential user, assuming they followed the dockstore setup tutorial diff --git a/README.md b/README.md index 3fc2b0dc5..8aca30fa3 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The following section is useful for users of Dockstore (e.g. those that want to launch tools). After registering at [dockstore.org](https://dockstore.org), you will be able to download the Dockstore -CLI at https://dockstore.org/onboarding +CLI at [onboarding](https://dockstore.org/onboarding). The Dockstore command line script should be installed in a location in your path. ### Configuration File @@ -41,12 +41,13 @@ token = server-url = https://www.dockstore.org/api ``` +For developers, if you are working with a custom-built or updated Dockstore client you will need to update the jar in: `~/.dockstore/config/self-installs`. + ### File Provisioning By default, cwltool reads input files from the local filesystem. Dockstore also adds support for additional file systems such as http, https, and ftp. Through a plug-in system, Dockstore also supports -the Amazon S3, [Synapse](http://docs.synapse.org/articles/downloading_data.html), and -[ICGC Score Client](https://docs.icgc.org/download/guide/#score-client-usage) via [plugins](https://github.com/dockstore). +Amazon S3 and [Synapse](https://help.synapse.org/docs/Downloading-Data-Programmatically.2003796248.html) via [plugins](https://github.com/dockstore). Download the above set of default plugins via: ``` @@ -68,9 +69,6 @@ cache-dir = /home//.dockstore/cache #set this to determine where input f [dockstore-file-s3-plugin] endpoint = #set this to point at a non AWS S3 endpoint - -[dockstore-file-icgc-storage-client-plugin] -client = /media/large_volume/icgc-storage-client-1.0.23/bin/icgc-storage-client ``` Additional plugins can be created by taking one of the repos in [plugins](https://github.com/dockstore) as a model and diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index fb32b47fc..d7541bd3e 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -1,5 +1,5 @@ -Lists of 394 third-party dependencies. +Lists of 403 third-party dependencies. (Apache License, Version 2.0) akka-actor (com.typesafe.akka:akka-actor_2.13:2.5.32 - https://akka.io/) (Apache License, Version 2.0) akka-protobuf (com.typesafe.akka:akka-protobuf_2.13:2.5.32 - https://akka.io/) (Apache License, Version 2.0) akka-slf4j (com.typesafe.akka:akka-slf4j_2.13:2.5.32 - https://akka.io/) @@ -9,16 +9,16 @@ Lists of 394 third-party dependencies. (Apache License 2.0) Annotations for Metrics (io.dropwizard.metrics:metrics-annotation:4.2.19 - https://metrics.dropwizard.io/metrics-annotation) (The BSD License) ANTLR 4 Runtime (org.antlr:antlr4-runtime:4.10.1 - http://www.antlr.org/antlr4-runtime) (EPL 2.0) (GPL2 w/ CPE) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:3.0.4 - https://github.com/eclipse-ee4j/glassfish-hk2/external/aopalliance-repackaged) - (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.3 - https://avro.apache.org) + (Apache-2.0) Apache Avro (org.apache.avro:avro:1.11.4 - https://avro.apache.org) (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/) - (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.15 - https://commons.apache.org/proper/commons-codec/) + (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.1 - https://commons.apache.org/proper/commons-codec/) (Apache License, Version 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/) (Apache License, Version 2.0) Apache Commons Collections (org.apache.commons:commons-collections4:4.4 - https://commons.apache.org/proper/commons-collections/) - (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.0 - https://commons.apache.org/proper/commons-compress/) + (Apache-2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.26.2 - https://commons.apache.org/proper/commons-compress/) (Apache-2.0) Apache Commons Configuration (org.apache.commons:commons-configuration2:2.10.1 - https://commons.apache.org/proper/commons-configuration/) (Apache License, Version 2.0) Apache Commons CSV (org.apache.commons:commons-csv:1.5 - http://commons.apache.org/proper/commons-csv/) (Apache License, Version 2.0) Apache Commons Exec (org.apache.commons:commons-exec:1.3 - http://commons.apache.org/proper/commons-exec/) - (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.15.1 - https://commons.apache.org/proper/commons-io/) + (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.16.1 - https://commons.apache.org/proper/commons-io/) (Apache-2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.14.0 - https://commons.apache.org/proper/commons-lang/) (Apache-2.0) Apache Commons Logging (commons-logging:commons-logging:1.3.0 - https://commons.apache.org/proper/commons-logging/) (Apache License, Version 2.0) Apache Commons Net (commons-net:commons-net:3.9.0 - https://commons.apache.org/proper/commons-net/) @@ -43,33 +43,44 @@ Lists of 394 third-party dependencies. (Apache License 2.0) Asynchronous Http Client Netty Utils (org.asynchttpclient:async-http-client-netty-utils:2.10.3 - http://github.com/AsyncHttpClient/async-http-client/async-http-client-netty-utils) (Apache 2.0) AutoValue Annotations (com.google.auto.value:auto-value-annotations:1.9 - https://github.com/google/auto/tree/master/value) (Apache License, Version 2.0) AWS Event Stream (software.amazon.eventstream:eventstream:1.0.1 - https://github.com/awslabs/aws-eventstream-java) - (Apache License, Version 2.0) AWS Java SDK :: Annotations (software.amazon.awssdk:annotations:2.17.186 - https://aws.amazon.com/sdkforjava/core/annotations) - (Apache License, Version 2.0) AWS Java SDK :: Arns (software.amazon.awssdk:arns:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Auth (software.amazon.awssdk:auth:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: AWS Core (software.amazon.awssdk:aws-core:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: AWS Json Protocol (software.amazon.awssdk:aws-json-protocol:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: AWS Query Protocol (software.amazon.awssdk:aws-query-protocol:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: AWS Xml Protocol (software.amazon.awssdk:aws-xml-protocol:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: Json Utils (software.amazon.awssdk:json-utils:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: Protocol Core (software.amazon.awssdk:protocol-core:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: HTTP Client Interface (software.amazon.awssdk:http-client-spi:2.17.186 - https://aws.amazon.com/sdkforjava/http-client-spi) - (Apache License, Version 2.0) AWS Java SDK :: HTTP Clients :: Apache (software.amazon.awssdk:apache-client:2.17.186 - https://aws.amazon.com/sdkforjava/http-clients/apache-client) - (Apache License, Version 2.0) AWS Java SDK :: HTTP Clients :: Netty Non-Blocking I/O (software.amazon.awssdk:netty-nio-client:2.17.186 - https://aws.amazon.com/sdkforjava/http-clients/netty-nio-client) - (Apache License, Version 2.0) AWS Java SDK :: Metrics SPI (software.amazon.awssdk:metrics-spi:2.17.186 - https://aws.amazon.com/sdkforjava/core/metrics-spi) - (Apache License, Version 2.0) AWS Java SDK :: Profiles (software.amazon.awssdk:profiles:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Regions (software.amazon.awssdk:regions:2.17.186 - https://aws.amazon.com/sdkforjava/core/regions) - (Apache License, Version 2.0) AWS Java SDK :: SDK Core (software.amazon.awssdk:sdk-core:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon CloudWatch (software.amazon.awssdk:cloudwatch:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon EC2 (software.amazon.awssdk:ec2:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon RDS (software.amazon.awssdk:rds:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon S3 (software.amazon.awssdk:s3:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon SQS (software.amazon.awssdk:sqs:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS CloudFormation (software.amazon.awssdk:cloudformation:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS Config (software.amazon.awssdk:config:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS IAM (software.amazon.awssdk:iam:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS Lambda (software.amazon.awssdk:lambda:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Third Party :: Jackson-core (software.amazon.awssdk:third-party-jackson-core:2.17.186 - https://aws.amazon.com/sdkforjava) - (Apache License, Version 2.0) AWS Java SDK :: Utilities (software.amazon.awssdk:utils:2.17.186 - https://aws.amazon.com/sdkforjava/utils) + (Apache License, Version 2.0) AWS Java SDK :: Annotations (software.amazon.awssdk:annotations:2.28.7 - https://aws.amazon.com/sdkforjava/core/annotations) + (Apache License, Version 2.0) AWS Java SDK :: Arns (software.amazon.awssdk:arns:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Auth (software.amazon.awssdk:auth:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: AWS Core (software.amazon.awssdk:aws-core:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: AWS CRT Core (software.amazon.awssdk:crt-core:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Checksums (software.amazon.awssdk:checksums:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Checksums SPI (software.amazon.awssdk:checksums-spi:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: AWS Json Protocol (software.amazon.awssdk:aws-json-protocol:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: AWS Query Protocol (software.amazon.awssdk:aws-query-protocol:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: AWS Xml Protocol (software.amazon.awssdk:aws-xml-protocol:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: Json Utils (software.amazon.awssdk:json-utils:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Core :: Protocols :: Protocol Core (software.amazon.awssdk:protocol-core:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Endpoints SPI (software.amazon.awssdk:endpoints-spi:2.28.7 - https://aws.amazon.com/sdkforjava/core/endpoints-spi) + (Apache License, Version 2.0) AWS Java SDK :: HTTP Auth (software.amazon.awssdk:http-auth:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: HTTP Auth AWS (software.amazon.awssdk:http-auth-aws:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: HTTP Auth Event Stream (software.amazon.awssdk:http-auth-aws-eventstream:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: HTTP Auth SPI (software.amazon.awssdk:http-auth-spi:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: HTTP Client Interface (software.amazon.awssdk:http-client-spi:2.28.7 - https://aws.amazon.com/sdkforjava/http-client-spi) + (Apache License, Version 2.0) AWS Java SDK :: HTTP Clients :: Apache (software.amazon.awssdk:apache-client:2.28.7 - https://aws.amazon.com/sdkforjava/http-clients/apache-client) + (Apache License, Version 2.0) AWS Java SDK :: HTTP Clients :: Netty Non-Blocking I/O (software.amazon.awssdk:netty-nio-client:2.28.7 - https://aws.amazon.com/sdkforjava/http-clients/netty-nio-client) + (Apache License, Version 2.0) AWS Java SDK :: Identity SPI (software.amazon.awssdk:identity-spi:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Metrics SPI (software.amazon.awssdk:metrics-spi:2.28.7 - https://aws.amazon.com/sdkforjava/core/metrics-spi) + (Apache License, Version 2.0) AWS Java SDK :: Profiles (software.amazon.awssdk:profiles:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Regions (software.amazon.awssdk:regions:2.28.7 - https://aws.amazon.com/sdkforjava/core/regions) + (Apache License, Version 2.0) AWS Java SDK :: Retries (software.amazon.awssdk:retries:2.28.7 - https://aws.amazon.com/sdkforjava/core/retries) + (Apache License, Version 2.0) AWS Java SDK :: Retries API (software.amazon.awssdk:retries-spi:2.28.7 - https://aws.amazon.com/sdkforjava/core/retries-spi) + (Apache License, Version 2.0) AWS Java SDK :: SDK Core (software.amazon.awssdk:sdk-core:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon CloudWatch (software.amazon.awssdk:cloudwatch:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon EC2 (software.amazon.awssdk:ec2:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon RDS (software.amazon.awssdk:rds:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon S3 (software.amazon.awssdk:s3:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: Amazon SQS (software.amazon.awssdk:sqs:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS CloudFormation (software.amazon.awssdk:cloudformation:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS Config (software.amazon.awssdk:config:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS IAM (software.amazon.awssdk:iam:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Services :: AWS Lambda (software.amazon.awssdk:lambda:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Third Party :: Jackson-core (software.amazon.awssdk:third-party-jackson-core:2.28.7 - https://aws.amazon.com/sdkforjava) + (Apache License, Version 2.0) AWS Java SDK :: Utilities (software.amazon.awssdk:utils:2.28.7 - https://aws.amazon.com/sdkforjava/utils) (The Apache Software License, Version 2.0) AWS V4 Signer for Java (uk.co.lucasweb:aws-v4-signer-java:1.3 - https://github.com/lucasweb78/aws-v4-signer-java) (The Apache Software License, Version 2.0) Bean Validation API (javax.validation:validation-api:1.1.0.Final - http://beanvalidation.org) (MIT) better-files (com.github.pathikrit:better-files_2.13:3.9.1 - https://github.com/pathikrit/better-files) @@ -121,8 +132,8 @@ Lists of 394 third-party dependencies. (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wdl-transforms-new-base (org.broadinstitute:cromwell-wdl-transforms-new-base_2.13:85 - no url defined) (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wdl-transforms-shared (org.broadinstitute:cromwell-wdl-transforms-shared_2.13:85 - no url defined) (Cromwell License https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt) cromwell-wom (org.broadinstitute:cromwell-wom_2.13:85 - no url defined) - (Apache License, Version 2.0) cwlavro-generated (io.cwl:cwlavro-generated:2.0.4.9 - no url defined) - (Apache License, Version 2.0) cwlavro-tools (io.cwl:cwlavro-tools:2.0.4.9 - no url defined) + (Apache License, Version 2.0) cwlavro-generated (io.cwl:cwlavro-generated:2.1.0 - no url defined) + (Apache License, Version 2.0) cwlavro-tools (io.cwl:cwlavro-tools:2.1.0 - no url defined) (The Apache Software License, Version 2.0) docker-java-api (com.github.docker-java:docker-java-api:3.3.6 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-core (com.github.docker-java:docker-java-core:3.3.6 - https://github.com/docker-java/docker-java) (The Apache Software License, Version 2.0) docker-java-transport (com.github.docker-java:docker-java-transport:3.3.6 - https://github.com/docker-java/docker-java) @@ -158,7 +169,7 @@ Lists of 394 third-party dependencies. (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) (The Apache Software License, Version 2.0) GeantyRef (io.leangen.geantyref:geantyref:1.3.14 - https://github.com/leangen/geantyref) (Apache 2.0) genericExtras (io.circe:circe-generic-extras_2.13:0.14.1 - https://github.com/circe/circe-generic-extras) - (The MIT license) GitHub API for Java (org.kohsuke:github-api:1.313 - https://github-api.kohsuke.org/) + (The MIT license) GitHub API for Java (org.kohsuke:github-api:1.322 - https://github-api.kohsuke.org/) (The Apache Software License, Version 2.0) Gitlab Java API Wrapper (org.gitlab:java-gitlab-api:4.0.0 - http://nexus.sonatype.org/oss-repository-hosting.html/java-gitlab-api) (The Apache Software License, Version 2.0) Google APIs Client Library for Java (com.google.api-client:google-api-client:1.35.0 - https://github.com/googleapis/google-api-java-client/google-api-client) (BSD New license) Google Auth Library for Java - Credentials (com.google.auth:google-auth-library-credentials:1.5.3 - https://github.com/googleapis/google-auth-library-java/google-auth-library-credentials) @@ -167,7 +178,7 @@ Lists of 394 third-party dependencies. (The Apache Software License, Version 2.0) Google OAuth Client Library for Java (com.google.oauth-client:google-oauth-client:1.34.0 - https://github.com/googleapis/google-oauth-java-client/google-oauth-client) (The Apache Software License, Version 2.0) Google OAuth2 API v2-rev151-1.25.0 (com.google.apis:google-api-services-oauth2:v2-rev151-1.25.0 - http://nexus.sonatype.org/oss-repository-hosting.html/google-api-services-oauth2) (The Apache Software License, Version 2.0) Graph Core (org.scala-graph:graph-core_2.13:1.13.1 - http://scala-graph.org) - (Apache-2.0) Gson (com.google.code.gson:gson:2.9.0 - https://github.com/google/gson/gson) + (Apache-2.0) Gson (com.google.code.gson:gson:2.10.1 - https://github.com/google/gson/gson) (The Apache Software License, Version 2.0) GSON extensions to the Google HTTP Client Library for Java. (com.google.http-client:google-http-client-gson:1.41.8 - https://github.com/googleapis/google-http-java-client/google-http-client-gson) (The Apache Software License, Version 2.0) Guava InternalFutureFailureAccess and InternalFutures (com.google.guava:failureaccess:1.0.1 - https://github.com/google/guava/failureaccess) (The Apache Software License, Version 2.0) Guava ListenableFuture only (com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - https://github.com/google/guava/listenablefuture) @@ -310,24 +321,22 @@ Lists of 394 third-party dependencies. (The MIT License) mockito-inline (org.mockito:mockito-inline:3.12.4 - https://github.com/mockito/mockito) (Apache 2 License) Moneta Core (org.javamoney.moneta:moneta-core:1.4.2 - http://javamoney.org) (MIT) mouse (org.typelevel:mouse_2.13:1.0.11 - https://typelevel.org/mouse) - (Apache License, Version 2.0) Netty Reactive Streams HTTP support (com.typesafe.netty:netty-reactive-streams-http:2.0.5 - https://github.com/playframework/netty-reactive-streams/netty-reactive-streams-http) (Apache License, Version 2.0) Netty Reactive Streams Implementation (com.typesafe.netty:netty-reactive-streams:2.0.5 - https://github.com/playframework/netty-reactive-streams/netty-reactive-streams) - (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.72.Final - https://netty.io/netty-buffer/) - (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.72.Final - https://netty.io/netty-codec/) - (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.72.Final - https://netty.io/netty-codec-dns/) - (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.72.Final - https://netty.io/netty-codec-http/) - (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.72.Final - https://netty.io/netty-codec-http2/) - (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.72.Final - https://netty.io/netty-codec-socks/) - (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.72.Final - https://netty.io/netty-common/) - (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.72.Final - https://netty.io/netty-handler/) - (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.72.Final - https://netty.io/netty-handler-proxy/) - (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.72.Final - https://netty.io/netty-resolver/) - (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.72.Final - https://netty.io/netty-resolver-dns/) - (https://github.com/netty/netty/blob/4.1/LICENSE.txt) Netty/TomcatNative [OpenSSL - Classes] (io.netty:netty-tcnative-classes:2.0.46.Final - https://github.com/netty/netty-tcnative/netty-tcnative-classes/) - (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.72.Final - https://netty.io/netty-transport/) - (Apache License, Version 2.0) Netty/Transport/Classes/Epoll (io.netty:netty-transport-classes-epoll:4.1.72.Final - https://netty.io/netty-transport-classes-epoll/) - (Apache License, Version 2.0) Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.72.Final - https://netty.io/netty-transport-native-epoll/) - (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.72.Final - https://netty.io/netty-transport-native-unix-common/) + (Apache License, Version 2.0) Netty/Buffer (io.netty:netty-buffer:4.1.112.Final - https://netty.io/netty-buffer/) + (Apache License, Version 2.0) Netty/Codec (io.netty:netty-codec:4.1.112.Final - https://netty.io/netty-codec/) + (Apache License, Version 2.0) Netty/Codec/DNS (io.netty:netty-codec-dns:4.1.112.Final - https://netty.io/netty-codec-dns/) + (Apache License, Version 2.0) Netty/Codec/HTTP (io.netty:netty-codec-http:4.1.112.Final - https://netty.io/netty-codec-http/) + (Apache License, Version 2.0) Netty/Codec/HTTP2 (io.netty:netty-codec-http2:4.1.112.Final - https://netty.io/netty-codec-http2/) + (Apache License, Version 2.0) Netty/Codec/Socks (io.netty:netty-codec-socks:4.1.112.Final - https://netty.io/netty-codec-socks/) + (Apache License, Version 2.0) Netty/Common (io.netty:netty-common:4.1.112.Final - https://netty.io/netty-common/) + (Apache License, Version 2.0) Netty/Handler (io.netty:netty-handler:4.1.112.Final - https://netty.io/netty-handler/) + (Apache License, Version 2.0) Netty/Handler/Proxy (io.netty:netty-handler-proxy:4.1.112.Final - https://netty.io/netty-handler-proxy/) + (Apache License, Version 2.0) Netty/Resolver (io.netty:netty-resolver:4.1.112.Final - https://netty.io/netty-resolver/) + (Apache License, Version 2.0) Netty/Resolver/DNS (io.netty:netty-resolver-dns:4.1.112.Final - https://netty.io/netty-resolver-dns/) + (Apache License, Version 2.0) Netty/Transport (io.netty:netty-transport:4.1.112.Final - https://netty.io/netty-transport/) + (Apache License, Version 2.0) Netty/Transport/Classes/Epoll (io.netty:netty-transport-classes-epoll:4.1.112.Final - https://netty.io/netty-transport-classes-epoll/) + (Apache License, Version 2.0) Netty/Transport/Native/Epoll (io.netty:netty-transport-native-epoll:4.1.112.Final - https://netty.io/netty-transport-native-epoll/) + (Apache License, Version 2.0) Netty/Transport/Native/Unix/Common (io.netty:netty-transport-native-unix-common:4.1.112.Final - https://netty.io/netty-transport-native-unix-common/) (Apache License, Version 2.0) Objenesis (org.objenesis:objenesis:3.2 - http://objenesis.org/objenesis) (The Apache Software License, Version 2.0) okhttp (com.squareup.okhttp3:okhttp:4.10.0 - https://square.github.io/okhttp/) (The Apache Software License, Version 2.0) okio (com.squareup.okio:okio-jvm:3.0.0 - https://github.com/square/okio/) @@ -352,7 +361,7 @@ Lists of 394 third-party dependencies. (The Apache Software License, Version 2.0) PowerMock (org.powermock:powermock-reflect:2.0.9 - http://www.powermock.org) (MIT) pprint_2.13 (com.lihaoyi:pprint_2.13:0.7.3 - https://github.com/lihaoyi/PPrint) (The Apache Software License, Version 2.0) rank-eval (org.elasticsearch.plugin:rank-eval-client:7.10.2 - https://github.com/elastic/elasticsearch) - (CC0) reactive-streams (org.reactivestreams:reactive-streams:1.0.3 - http://www.reactive-streams.org/) + (MIT-0) reactive-streams (org.reactivestreams:reactive-streams:1.0.4 - http://www.reactive-streams.org/) (MIT) refined (eu.timepit:refined_2.13:0.10.1 - https://github.com/fthomas/refined) (The Apache Software License, Version 2.0) rest (org.elasticsearch.client:elasticsearch-rest-client:7.10.2 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) rest-high-level (org.elasticsearch.client:elasticsearch-rest-high-level-client:7.10.2 - https://github.com/elastic/elasticsearch) diff --git a/dockstore-cli-integration-testing/generated/src/main/resources/pom.xml b/dockstore-cli-integration-testing/generated/src/main/resources/pom.xml index 056f80ece..0a7292ee5 100644 --- a/dockstore-cli-integration-testing/generated/src/main/resources/pom.xml +++ b/dockstore-cli-integration-testing/generated/src/main/resources/pom.xml @@ -19,7 +19,7 @@ 4.0.0 io.dockstore dockstore-cli-integration-testing - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT Apache Software License, Version 2.0 @@ -30,13 +30,13 @@ io.dockstore dockstore-client - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT compile io.dockstore dockstore-integration-testing - 1.16.0-alpha.11 + 1.17.0-alpha.0 tests compile @@ -49,13 +49,13 @@ io.dockstore dockstore-common - 1.16.0-alpha.11 + 1.17.0-alpha.0 compile io.dockstore dockstore-webservice - 1.16.0-alpha.11 + 1.17.0-alpha.0 compile @@ -67,7 +67,7 @@ io.dockstore openapi-java-client - 1.16.0-alpha.11 + 1.17.0-alpha.0 compile @@ -79,7 +79,7 @@ commons-io commons-io - 2.15.1 + 2.16.1 compile diff --git a/dockstore-cli-integration-testing/pom.xml b/dockstore-cli-integration-testing/pom.xml index 3a1feeb55..802fb313b 100644 --- a/dockstore-cli-integration-testing/pom.xml +++ b/dockstore-cli-integration-testing/pom.xml @@ -293,7 +293,7 @@ - + net.alchim31.maven scala-maven-plugin diff --git a/dockstore-cli-integration-testing/src/test/java/io/dockstore/client/cli/ClientIT.java b/dockstore-cli-integration-testing/src/test/java/io/dockstore/client/cli/ClientIT.java index 44b78b758..3abb4d790 100644 --- a/dockstore-cli-integration-testing/src/test/java/io/dockstore/client/cli/ClientIT.java +++ b/dockstore-cli-integration-testing/src/test/java/io/dockstore/client/cli/ClientIT.java @@ -158,7 +158,6 @@ void testPluginEnable() { Client.main(new String[] { CONFIG, ResourceHelpers.resourceFilePath("pluginsTest1/configWithPlugins"), PLUGIN, LIST }); assertTrue(systemOutRule.getText().contains("dockstore-file-synapse-plugin")); assertTrue(systemOutRule.getText().contains("dockstore-file-s3-plugin")); - assertFalse(systemOutRule.getText().contains("dockstore-icgc-storage-client-plugin")); } @Test @@ -168,7 +167,6 @@ void testPluginDisable() { Client.main(new String[] { CONFIG, ResourceHelpers.resourceFilePath("pluginsTest2/configWithPlugins"), PLUGIN, LIST }); assertFalse(systemOutRule.getText().contains("dockstore-file-synapse-plugin")); assertFalse(systemOutRule.getText().contains("dockstore-file-s3-plugin")); - assertTrue(systemOutRule.getText().contains("dockstore-file-icgc-storage-client-plugin")); } @Disabled("seems to have been disabled for ages") diff --git a/dockstore-cli-integration-testing/src/test/resources/pluginsTest1/configWithPlugins b/dockstore-cli-integration-testing/src/test/resources/pluginsTest1/configWithPlugins index c613451db..64327480e 100644 --- a/dockstore-cli-integration-testing/src/test/resources/pluginsTest1/configWithPlugins +++ b/dockstore-cli-integration-testing/src/test/resources/pluginsTest1/configWithPlugins @@ -2,17 +2,10 @@ token: test server-url: https://www.dockstore.org/api file-plugins-location: src/test/resources/pluginsTest1/plugins -[dockstore-icgc-get-plugin] -client = /home/gluu/icgc-get/icgc-get -config-file-location = /home/gluu/.icgc-get/config.yaml - [dockstore-s3cmd-plugin] client = /usr/bin/s3cmd config-file-location = /home/gluu/.s3cfg -[dockstore-file-icgc-storage-client-plugin] -client = asdf - [dockstore-file-s3-plugin] client = asdf diff --git a/dockstore-cli-integration-testing/src/test/resources/pluginsTest2/configWithPlugins b/dockstore-cli-integration-testing/src/test/resources/pluginsTest2/configWithPlugins index 9033e2fca..8439068fc 100644 --- a/dockstore-cli-integration-testing/src/test/resources/pluginsTest2/configWithPlugins +++ b/dockstore-cli-integration-testing/src/test/resources/pluginsTest2/configWithPlugins @@ -2,17 +2,10 @@ token: test server-url: https://www.dockstore.org/api file-plugins-location: src/test/resources/pluginsTest2/plugins -[dockstore-icgc-get-plugin] -client = /home/gluu/icgc-get/icgc-get -config-file-location = /home/gluu/.icgc-get/config.yaml - [dockstore-s3cmd-plugin] client = /usr/bin/s3cmd config-file-location = /home/gluu/.s3cfg -[dockstore-file-icgc-storage-client-plugin] -client = asdf - [dockstore-file-s3-plugin] client = asdf diff --git a/dockstore-cli-reports/generated/src/main/resources/pom.xml b/dockstore-cli-reports/generated/src/main/resources/pom.xml index 5cabf5766..7907e92dd 100644 --- a/dockstore-cli-reports/generated/src/main/resources/pom.xml +++ b/dockstore-cli-reports/generated/src/main/resources/pom.xml @@ -19,7 +19,7 @@ 4.0.0 io.dockstore dockstore-cli-reports - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT Apache Software License, Version 2.0 @@ -30,25 +30,25 @@ io.dockstore dockstore-webservice - 1.16.0-alpha.11 + 1.17.0-alpha.0 compile io.dockstore dockstore-common - 1.16.0-alpha.11 + 1.17.0-alpha.0 compile io.dockstore dockstore-client - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT compile io.dockstore dockstore-cli-integration-testing - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT compile diff --git a/dockstore-client/README.md b/dockstore-client/README.md index 65c6558d6..9e6830c28 100644 --- a/dockstore-client/README.md +++ b/dockstore-client/README.md @@ -30,7 +30,7 @@ This takes a CWL-based descriptor along with a JSON file that specifies paramete 0. pulls over 0 or more files that were associated with this workflow order to `/datastore/launcher-/configs`, these will be used by the workflow itself. These will come from a web service endpoint in Consonance rather than external sources like inputs below. This is how we get a SeqWare INI file for example. For this demo launcher this functionality will be skipped since it lacks a queue/web service to talk to. 0. make `/datastore/launcher-/working` to be used as the working directory for the command, `/datastore/launcher-/inputs` for all the file inputs, and `/datastore/launcher-/logs` for logs 0. start services referenced in the descriptor, this functionality does not yet exist -0. download all the inputs, these will come from S3, HTTP/S, SFTP, FTP, ICGCObjectStore, etc, put them in locations within `/datastore/launcher-/inputs` +0. download all the inputs, these will come from S3, HTTP/S, SFTP, FTP, etc, put them in locations within `/datastore/launcher-/inputs` 0. produces a new JSON parameterization document that has the URLs replaced with file paths that are local to the container host 0. hands the updated JSON parameterization document and CWL descriptor to the CWL runner tool, this causes the command to be constructed, docker containers to be pulled and the command to be run correctly 0. collect and provision output files to their destination referenced in `~/.consonance/launcher.config` @@ -43,10 +43,6 @@ working-directory=./datastore/ [s3] # optional: the S3 endpoint can be overridden to use custom implementations of the S3 API rather than AWS S3 # endpoint = https://www.cancercollaboratory.org:9080 - -[dcc_storage] -# optional: the location of the DCC storage client can be customized -# client = /icgc/dcc-storage/bin/dcc-storage-client ``` This tells the system what the local working directory should be. Files will be written here. diff --git a/dockstore-client/generated/src/main/resources/pom.xml b/dockstore-client/generated/src/main/resources/pom.xml index 7c119af0e..d43a6fd69 100644 --- a/dockstore-client/generated/src/main/resources/pom.xml +++ b/dockstore-client/generated/src/main/resources/pom.xml @@ -19,7 +19,7 @@ 4.0.0 io.dockstore dockstore-client - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT Apache Software License, Version 2.0 @@ -66,7 +66,7 @@ io.dockstore dockstore-common - 1.16.0-alpha.11 + 1.17.0-alpha.0 compile @@ -78,19 +78,19 @@ io.dockstore openapi-java-client - 1.16.0-alpha.11 + 1.17.0-alpha.0 compile io.dockstore openapi-java-wes-client - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT compile io.dockstore dockstore-file-plugin-parent - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT compile @@ -108,7 +108,7 @@ com.google.code.gson gson - 2.9.0 + 2.10.1 compile @@ -126,7 +126,7 @@ commons-io commons-io - 2.15.1 + 2.16.1 compile @@ -162,7 +162,7 @@ io.cwl cwlavro-tools - 2.0.4.9 + 2.1.0 compile @@ -178,7 +178,7 @@ io.cwl cwlavro-generated - 2.0.4.9 + 2.1.0 compile @@ -267,7 +267,7 @@ commons-codec commons-codec - 1.15 + 1.17.1 compile @@ -313,25 +313,25 @@ software.amazon.awssdk regions - 2.17.186 + 2.28.7 compile software.amazon.awssdk sdk-core - 2.17.186 + 2.28.7 compile software.amazon.awssdk auth - 2.17.186 + 2.28.7 compile software.amazon.awssdk profiles - 2.17.186 + 2.28.7 compile diff --git a/dockstore-client/src/main/java/io/dockstore/client/cli/nested/AbstractEntryClient.java b/dockstore-client/src/main/java/io/dockstore/client/cli/nested/AbstractEntryClient.java index b63587c58..649e07960 100644 --- a/dockstore-client/src/main/java/io/dockstore/client/cli/nested/AbstractEntryClient.java +++ b/dockstore-client/src/main/java/io/dockstore/client/cli/nested/AbstractEntryClient.java @@ -221,7 +221,7 @@ boolean getIgnoreChecksums() { public CWL getCwlUtil() { // TODO: may be reactivated if we find a different way to read CWL into Java // String cwlrunner = CWLRunnerFactory.getCWLRunner(); - return new CWL(false, Utilities.parseConfig(getConfigFile())); + return new CWL(Utilities.parseConfig(getConfigFile())); } public abstract String getConfigFile(); diff --git a/dockstore-client/src/main/java/io/dockstore/common/ToilOnlyTest.java b/dockstore-client/src/main/java/io/dockstore/common/ToilOnlyTest.java new file mode 100644 index 000000000..495f5c07d --- /dev/null +++ b/dockstore-client/src/main/java/io/dockstore/common/ToilOnlyTest.java @@ -0,0 +1,10 @@ +package io.dockstore.common; + +public interface ToilOnlyTest { + + String NAME = "io.dockstore.common.ToilOnlyTest"; + + default String getName() { + return NAME; + } +} diff --git a/dockstore-client/src/main/java/io/github/collaboratory/cwl/CWLClient.java b/dockstore-client/src/main/java/io/github/collaboratory/cwl/CWLClient.java index a71380899..8c5a454b2 100644 --- a/dockstore-client/src/main/java/io/github/collaboratory/cwl/CWLClient.java +++ b/dockstore-client/src/main/java/io/github/collaboratory/cwl/CWLClient.java @@ -177,7 +177,7 @@ public File provisionInputFiles() { Class cwlClassTarget = getCWLClassTarget(); // Load CWL from JSON to object - CWL cwlUtil = new CWL(false, config); + CWL cwlUtil = new CWL(config); // This won't work since I am using zip files, it is expecting files to be unzipped final String imageDescriptorContent = cwlUtil.parseCWL(localPrimaryDescriptorFile.getAbsolutePath()).getLeft(); Object cwlObject = null; diff --git a/dockstore-client/src/main/java/io/github/collaboratory/cwl/cwlrunner/ToilWrapper.java b/dockstore-client/src/main/java/io/github/collaboratory/cwl/cwlrunner/ToilWrapper.java index 3dc47b7e5..b993442ce 100644 --- a/dockstore-client/src/main/java/io/github/collaboratory/cwl/cwlrunner/ToilWrapper.java +++ b/dockstore-client/src/main/java/io/github/collaboratory/cwl/cwlrunner/ToilWrapper.java @@ -22,6 +22,8 @@ import io.dockstore.client.cli.ArgumentUtility; import io.dockstore.client.cli.Client; import io.dockstore.openapi.client.api.MetadataApi; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -35,9 +37,9 @@ public void checkForCWLDependencies(MetadataApi metadataApi) { final ImmutablePair pair1 = io.cwl.avro.Utilities .executeCommand(Joiner.on(" ").join(Arrays.asList(s1)), false, com.google.common.base.Optional.absent(), com.google.common.base.Optional.absent()); - final String toilVersion = pair1.getValue().trim(); + final String toilVersion = pair1.getKey().trim(); - final String expectedToilVersion = "3.15.0"; + final String expectedToilVersion = "7.0.0"; if (!toilVersion.equals(expectedToilVersion)) { ArgumentUtility.errorMessage("toil version is " + toilVersion + " , Dockstore is tested with " + expectedToilVersion + "\nOverride and run with `" + SCRIPT_FLAG + "`", Client.COMMAND_ERROR); @@ -49,9 +51,15 @@ public List getExecutionCommand(String outputDir, String tmpDir, String //TODO: this doesn't quite work yet, seeing "toil.batchSystems.abstractBatchSystem.InsufficientSystemResources: Requesting more disk // than either physically available, or enforced by --maxDisk. Requested: 537944653824, Available: 134853001216" on trivial // workflows like md5sum + + //try normalizing paths + Path currentRelativePath = Path.of(""); + Path tmpPath = Paths.get(currentRelativePath.toAbsolutePath().toString(), tmpDir); + tmpPath = tmpPath.normalize(); + ArrayList command = new ArrayList<>( - Arrays.asList("toil-cwl-runner", "--logError", "--outdir", outputDir, "--tmpdir-prefix", tmpDir, "--tmp-outdir-prefix", - workingDir, cwlFile)); + Arrays.asList("toil-cwl-runner", "--bypass-file-store", "--logError", "--outdir", outputDir, "--tmpdir-prefix", tmpPath.toString(), "--tmp-outdir-prefix", + workingDir, cwlFile)); jsonSettings.ifPresent(command::add); return command; } diff --git a/dockstore-client/src/main/resources/plugins.json b/dockstore-client/src/main/resources/plugins.json index 61d8f9143..726b321a3 100644 --- a/dockstore-client/src/main/resources/plugins.json +++ b/dockstore-client/src/main/resources/plugins.json @@ -1,7 +1,4 @@ [{ - "name": "dockstore-file-icgc-storage-client-plugin", - "version": "0.0.6" - },{ "name": "dockstore-file-s3-plugin", "version": "0.0.3" },{ diff --git a/dockstore-client/src/test/java/io/dockstore/client/cli/YamlValidatorTest.java b/dockstore-client/src/test/java/io/dockstore/client/cli/YamlValidatorTest.java index 1217ec963..c25d9bb85 100644 --- a/dockstore-client/src/test/java/io/dockstore/client/cli/YamlValidatorTest.java +++ b/dockstore-client/src/test/java/io/dockstore/client/cli/YamlValidatorTest.java @@ -90,7 +90,7 @@ void relativePrimaryPathYml() { YamlVerifyUtility.dockstoreValidate(relativePathDockstoreYmlDirectory); fail("Invalid YAML not caught"); } catch (YamlVerifyUtility.ValidateYamlException ex) { - assertTrue(ex.getMessage().contains("the path must be an absolute path to be valid")); + assertTrue(ex.getMessage().contains("must be an absolute path to be valid")); } } @@ -103,7 +103,7 @@ void relativeTestDescriptorPathYml() { YamlVerifyUtility.dockstoreValidate(relativePathDockstoreYmlDirectory); fail("Invalid YAML not caught"); } catch (YamlVerifyUtility.ValidateYamlException ex) { - assertTrue(ex.getMessage().contains("the path must be an absolute path to be valid")); + assertTrue(ex.getMessage().contains("must be an absolute path to be valid")); } } diff --git a/dockstore-client/src/test/java/io/github/collaboratory/cwl/ToilLauncherIT.java b/dockstore-client/src/test/java/io/github/collaboratory/cwl/ToilLauncherIT.java index 5f996f75e..7c28118b3 100644 --- a/dockstore-client/src/test/java/io/github/collaboratory/cwl/ToilLauncherIT.java +++ b/dockstore-client/src/test/java/io/github/collaboratory/cwl/ToilLauncherIT.java @@ -15,14 +15,16 @@ */ package io.github.collaboratory.cwl; +import io.dockstore.common.ToilOnlyTest; import org.apache.commons.io.FileUtils; -import org.junit.Ignore; +import org.junit.experimental.categories.Category; +import org.junit.jupiter.api.Tag; /** * @author dyuen */ -@Ignore -// @Category(ToilOnlyTest.class) +@Tag(ToilOnlyTest.NAME) +@Category(ToilOnlyTest.class) public class ToilLauncherIT extends LauncherIT { public String getConfigFile() { diff --git a/dockstore-file-plugin-parent/README.md b/dockstore-file-plugin-parent/README.md index 3c45e2550..cbb653373 100644 --- a/dockstore-file-plugin-parent/README.md +++ b/dockstore-file-plugin-parent/README.md @@ -21,7 +21,7 @@ Use the following command to list currently installed plugins dockstore plugin list ``` -This will list the plugins along with their versions, their install locations, and the schemes that they handle (ex: `s3:\\test.txt` , `icgc:\\123-456` , `syn:\\syn12345` ). +This will list the plugins along with their versions, their install locations, and the schemes that they handle (ex: `s3:\\test.txt`, `syn:\\syn12345` ). Note that dockstore will automatically delete old versions of plugins when you install new plugins. @@ -34,9 +34,6 @@ synapse-api-key = dummy-key synapse-user-name = my.user.name [dockstore-file-s3-plugin] - -[dockstore-file-icgc-storage-client-plugin] -client = /media/large_volume/icgc-storage-client-1.0.23/bin/icgc-storage-client ``` @@ -44,7 +41,7 @@ client = /media/large_volume/icgc-storage-client-1.0.23/bin/icgc-storage-client When developing new plugins, we recommend using the [s3-plugin](https://github.com/dockstore/s3-plugin) as a model for plugins where [a Java library is available](https://aws.amazon.com/sdk-for-java/). -We recommend using the [icgc-storage-client-plugin](https://github.com/dockstore/icgc-storage-client-plugin) as a model for +Although it is no longer directly supported, we recommend using the [icgc-storage-client-plugin](https://github.com/dockstore/icgc-storage-client-plugin) as a model for plugins where a Java library is not available and the plugin needs to call out to an external binary. This was developed in an environment with Java 8 and Maven 3.3.9. diff --git a/dockstore-file-plugin-parent/generated/src/main/resources/pom.xml b/dockstore-file-plugin-parent/generated/src/main/resources/pom.xml index 67df75bf7..1f3f7f0c9 100644 --- a/dockstore-file-plugin-parent/generated/src/main/resources/pom.xml +++ b/dockstore-file-plugin-parent/generated/src/main/resources/pom.xml @@ -19,7 +19,7 @@ 4.0.0 io.dockstore dockstore-file-plugin-parent - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT Apache Software License, Version 2.0 diff --git a/dockstore-file-plugin-parent/src/main/java/io/dockstore/provision/ProvisionInterface.java b/dockstore-file-plugin-parent/src/main/java/io/dockstore/provision/ProvisionInterface.java index 3466c92a5..a55d1662c 100644 --- a/dockstore-file-plugin-parent/src/main/java/io/dockstore/provision/ProvisionInterface.java +++ b/dockstore-file-plugin-parent/src/main/java/io/dockstore/provision/ProvisionInterface.java @@ -29,7 +29,7 @@ public interface ProvisionInterface extends ExtensionPoint { /** * Returns whether a particular file path should be handled by this plugin - * @return return schemes that this provision interface handles (ex: http, https, ftp, syn, icgc) + * @return return schemes that this provision interface handles (ex: http, https, ftp, syn) */ Set schemesHandled(); diff --git a/openapi-java-wes-client/generated/src/main/resources/pom.xml b/openapi-java-wes-client/generated/src/main/resources/pom.xml index f200d38c4..65505c398 100644 --- a/openapi-java-wes-client/generated/src/main/resources/pom.xml +++ b/openapi-java-wes-client/generated/src/main/resources/pom.xml @@ -19,7 +19,7 @@ 4.0.0 io.dockstore openapi-java-wes-client - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT Apache Software License, Version 2.0 diff --git a/pom.xml b/pom.xml index 788697b9f..f4817c284 100644 --- a/pom.xml +++ b/pom.xml @@ -33,12 +33,12 @@ --> - 1.15 + 1.17 .0-SNAPSHOT scm:git:git@github.com:dockstore/dockstore-cli.git UTF-8 - 1.16.0-alpha.11 + 1.17.0-alpha.0 2.22.2 2.22.2 false @@ -53,7 +53,7 @@ ${project.basedir}/target/jacoco.xml,${project.basedir}/target/jacoco-it.xml,${project.basedir}/../dockstore-cli-reports/target/site/jacoco-aggregate/jacoco.xml - 2.0.4.9 + 2.1.0 @@ -200,28 +200,6 @@ - - - org.apache.commons - commons-compress - 1.26.0 - - - org.apache.commons - commons-text - 1.11.0 - - - commons-io - commons-io - 2.15.1 - - - org.apache.commons - commons-lang3 - 3.14.0 - - @@ -399,7 +377,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.8.3.1 + 4.8.6.2 org.apache.maven.plugins @@ -432,12 +410,12 @@ maven-javadoc-plugin 3.3.2 - + net.alchim31.maven scala-maven-plugin 4.8.1 - 17 + ${maven-java.version} @@ -454,7 +432,7 @@ org.codehaus.mojo versions-maven-plugin - 2.10.0 + 2.18.0 org.codehaus.mojo diff --git a/scripts/install-tests.sh b/scripts/install-tests.sh index 22b3037fe..3ae383be5 100755 --- a/scripts/install-tests.sh +++ b/scripts/install-tests.sh @@ -12,9 +12,9 @@ if [ "${TESTING_PROFILE}" = "unit-tests" ] || [ "${TESTING_PROFILE}" == "automat fi if [ "${TESTING_PROFILE}" = "toil-integration-tests" ]; then - pip3 install --user toil[cwl]==3.15.0 + pip3 install --user toil[cwl]==7.0.0 else - # depending on https://github.com/dockstore/dockstore/pull/5958 we may want to match where we go with the cwltool install, for now apt seems to work well + pip3 install --user -r https://raw.githubusercontent.com/dockstore/dockstore/develop/dockstore-webservice/src/main/resources/requirements/1.15.0/requirements3.txt sudo apt-get update # https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive needed by cwltool DEBIAN_FRONTEND=noninteractive sudo apt-get -qq --yes --force-yes install tzdata diff --git a/support/generated/src/main/resources/pom.xml b/support/generated/src/main/resources/pom.xml index 5c55f6aed..9f7589209 100644 --- a/support/generated/src/main/resources/pom.xml +++ b/support/generated/src/main/resources/pom.xml @@ -19,7 +19,7 @@ 4.0.0 io.dockstore support - 1.15.0-SNAPSHOT + 1.17.0-SNAPSHOT Apache Software License, Version 2.0 @@ -30,43 +30,43 @@ software.amazon.awssdk ec2 - 2.17.186 + 2.28.7 compile software.amazon.awssdk s3 - 2.17.186 + 2.28.7 compile software.amazon.awssdk iam - 2.17.186 + 2.28.7 compile software.amazon.awssdk lambda - 2.17.186 + 2.28.7 compile software.amazon.awssdk regions - 2.17.186 + 2.28.7 compile software.amazon.awssdk cloudformation - 2.17.186 + 2.28.7 compile software.amazon.awssdk rds - 2.17.186 + 2.28.7 compile @@ -78,7 +78,7 @@ software.amazon.awssdk config - 2.17.186 + 2.28.7 compile