diff --git a/.circleci/config.yml b/.circleci/config.yml index b09758f..38d3cd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,12 +31,20 @@ commands: steps: - run: command: | - sudo apt-get update && sudo apt-get install software-properties-common git wget curl default-jre -y + sudo apt-get update && sudo apt-get install software-properties-common git wget curl -y + + # install java 11 + sudo add-apt-repository ppa:openjdk-r/ppa -y + sudo apt-get update && sudo apt-get install openjdk-11-jdk -y + # automatically set 11 as default java + sudo update-java-alternatives -a + sudo add-apt-repository ppa:deadsnakes/ppa -y sudo apt-get update && sudo apt-get install python3.6 -y sudo wget --no-check-certificate https://bootstrap.pypa.io/get-pip.py sudo python3.6 get-pip.py sudo ln -s /usr/bin/python3.6 /usr/local/bin/python3 + pip3 install --upgrade pip pip3 install caper google-cloud-storage @@ -51,11 +59,17 @@ commands: echo ${GCLOUD_SERVICE_ACCOUNT_SECRET_JSON} > tmp_secret_key.json export GOOGLE_APPLICATION_CREDENTIALS=$PWD/tmp_secret_key.json + + # add docker image to input JSON + cat ${INPUT} | jq ".+{\"chip.docker\": \"${TAG}\"}" > input_with_docker.json + caper run ../../../chip.wdl \ --backend gcp --gcp-prj ${GOOGLE_PROJECT_ID} \ --gcp-service-account-key-json $PWD/tmp_secret_key.json \ --out-gcs-bucket ${CAPER_OUT_DIR} --tmp-gcs-bucket ${CAPER_TMP_DIR} \ - -i ${INPUT} -m metadata.json --docker ${TAG} + -i input_with_docker.json -m metadata.json --docker ${TAG} + + rm -f input_with_docker.json res=$(jq '.outputs["chip.qc_json_ref_match"]' metadata.json) [[ "$res" != true ]] && exit 100 diff --git a/README.md b/README.md index 52c324a..9db41d6 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,33 @@ ## Download new Caper>=2.0 -New Caper is out. You need to update your Caper to work with the latest ENCODE ATAC-seq pipeline. +New Caper is out. You need to update your Caper to work with the latest ENCODE ChIP-seq pipeline. ```bash $ pip install caper --upgrade ``` +## Local/HPC users and new Caper>=2.0 + +There are tons of changes for local/HPC backends: `local`, `slurm`, `sge`, `pbs` and `lsf`(added). Make a backup of your current Caper configuration file `~/.caper/default.conf` and run `caper init`. Local/HPC users need to reset/initialize Caper's configuration file according to your chosen backend. Edit the configuration file and follow instructions in there. +```bash +$ cd ~/.caper +$ cp default.conf default.conf.bak +$ caper init [YOUR_BACKEND] +``` + +In order to run a pipeline, you need to add one of the following flags to specify the environment to run each task within. i.e. `--conda`, `--singularity` and `--docker`. These flags are not required for cloud backend users (`aws` and `gcp`). +```bash +# for example +$ caper run ... --singularity +``` + +For Conda users, **RE-INSTALL PIPELINE'S CONDA ENVIRONMENT AND DO NOT ACTIVATE CONDA ENVIRONMENT BEFORE RUNNING PIPELINES**. Caper will internally call `conda run -n ENV_NAME CROMWELL_JOB_SCRIPT`. Just make sure that pipeline's new Conda environments are correctly installed. +```bash +$ scripts/uninstall_conda_env.sh +$ scripts/install_conda_env.sh +``` + + ## Introduction This ChIP-Seq pipeline is based off the ENCODE (phase-3) transcription factor and histone ChIP-seq pipeline specifications (by Anshul Kundaje) in [this google doc](https://docs.google.com/document/d/1lG_Rd7fnYgRpSIqrIfuVlAz2dW1VaSQThzk836Db99c/edit#). @@ -21,34 +43,35 @@ This ChIP-Seq pipeline is based off the ENCODE (phase-3) transcription factor an ## Installation - 1) Make sure that you have Python>=3.6. Caper does not work with Python2. Install Caper and check its version >=2.0. ```bash $ python --version $ pip install caper - $ caper -v ``` +2) Make a backup of your Caper configuration file `~/.caper/default.conf` if you are upgrading from old Caper(<2.0.0). Reset/initialize Caper's configuration file. Read Caper's [README](https://github.com/ENCODE-DCC/caper/blob/master/README.md) carefully to choose a backend for your system. Follow the instruction in the configuration file. + ```bash + # make a backup of ~/.caper/default.conf if you already have it + $ caper init [YOUR_BACKEND] -2) Git clone this pipeline. - > **IMPORTANT**: use `~/chip-seq-pipeline2/chip.wdl` as `[WDL]` in Caper's documentation. + # then edit ~/.caper/default.conf + $ vi ~/.caper/default.conf + ``` +3) Git clone this pipeline. + > **IMPORTANT**: use `~/chip-seq-pipeline2/chip.wdl` as `[WDL]` in Caper's documentation. ```bash $ cd $ git clone https://github.com/ENCODE-DCC/chip-seq-pipeline2 ``` - -3) (Optional for Conda) Install pipeline's Conda environments if you don't have Singularity or Docker installed on your system. We recommend to use Singularity instead of Conda. If you don't have Conda on your system, install [Miniconda3](https://docs.conda.io/en/latest/miniconda.html). +4) (Optional for Conda) Install pipeline's Conda environments if you don't have Singularity or Docker installed on your system. We recommend to use Singularity instead of Conda. If you don't have Conda on your system, install [Miniconda3](https://docs.conda.io/en/latest/miniconda.html). ```bash $ cd chip-seq-pipeline2 + # uninstall old environments (<2.0.0) + $ bash scripts/uninstall_conda_env.sh $ bash scripts/install_conda_env.sh ``` -4) Follow [Caper's README](https://github.com/ENCODE-DCC/caper) carefully. Find an instruction for your platform and run `caper init`. Edit the initialized Caper's configuration file (`~/.caper/default.conf`). - ```bash - $ caper init [YOUR_PLATFORM] - $ vi ~/.caper/default.conf - ``` ## Test run @@ -63,10 +86,35 @@ The followings are just examples. Please read [Caper's README](https://github.co # Or submit it as a leader job (with long/enough resources) to SLURM (Stanford Sherlock) with Singularity # It will fail if you directly run the leader job on login nodes - $ sbatch -p [SLURM_PARTITON] -J [WORKFLOW_NAME] --export=ALL --mem 4G -t 4-0 --wrap "caper chip atac.wdl -i https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only.json --singularity" + $ sbatch -p [SLURM_PARTITON] -J [WORKFLOW_NAME] --export=ALL --mem 4G -t 4-0 --wrap "caper chip chip.wdl -i https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only.json --singularity" ``` +## Running a pipeline on Terra/Anvil (using Dockstore) + +Visit our pipeline repo on [Dockstore](https://dockstore.org/workflows/github.com/ENCODE-DCC/chip-seq-pipeline2). Click on `Terra` or `Anvil`. Follow Terra's instruction to create a workspace on Terra and add Terra's billing bot to your Google Cloud account. + +Download this [test input JSON for Terra](https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only.terra.json) and upload it to Terra's UI and then run analysis. + +If you want to use your own input JSON file, then make sure that all files in the input JSON are on a Google Cloud Storage bucket (`gs://`). URLs will not work. + + +## Running a pipeline on DNAnexus (using Dockstore) + +Sign up for a new account on [DNAnexus](https://platform.dnanexus.com/) and create a new project on either AWS or Azure. Visit our pipeline repo on [Dockstore](https://dockstore.org/workflows/github.com/ENCODE-DCC/chip-seq-pipeline2). Click on `DNAnexus`. Choose a destination directory on your DNAnexus project. Click on `Submit` and visit DNAnexus. This will submit a conversion job so that you can check status of it on `Monitor` on DNAnexus UI. + +Once conversion is done download one of the following input JSON files according to your chosen platform (AWS or Azure) for your DNAnexus project: +- AWS: https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only_dx.json +- Azure: https://storage.googleapis.com/encode-pipeline-test-samples/encode-chip-seq-pipeline/ENCSR000DYI_subsampled_chr19_only_dx_azure.json + +You cannot use these input JSON files directly. Go to the destination directory on DNAnexus and click on the converted workflow `chip`. You will see input file boxes in the left-hand side of the task graph. Expand it and define FASTQs (`fastq_repX_R1`) and `genome_tsv` as in the downloaded input JSON file. Click on the `common` task box and define other non-file pipeline parameters. + + +## Running a pipeline on DNAnexus (using our pre-built workflows) + +See [this](docs/tutorial_dx_web.md) for details. + + ## Input JSON file @@ -82,13 +130,6 @@ You can run this pipeline on [truwl.com](https://truwl.com/). This provides a we If you do not run the pipeline on Truwl, you can still share your use-case/job on the platform by getting in touch at [info@truwl.com](mailto:info@truwl.com) and providing your inputs.json file. -## Running a pipeline on DNAnexus - -You can also run this pipeline on DNAnexus without using Caper or Cromwell. There are two ways to build a workflow on DNAnexus based on our WDL. - -1) [dxWDL CLI](docs/tutorial_dx_cli.md) -2) [DNAnexus Web UI](docs/tutorial_dx_web.md) - ## How to organize outputs Install [Croo](https://github.com/ENCODE-DCC/croo#installation). **You can skip this installation if you have installed pipeline's Conda environment and activated it**. Make sure that you have python3(> 3.4.1) installed on your system. Find a `metadata.json` on Caper's output directory. diff --git a/chip.wdl b/chip.wdl index c5e8e2a..1e1e652 100644 --- a/chip.wdl +++ b/chip.wdl @@ -7,16 +7,20 @@ struct RuntimeEnvironment { } workflow chip { - String pipeline_ver = 'v2.0.0' + String pipeline_ver = 'v2.0.1' meta { - version: 'v2.0.0' - author: 'Jin wook Lee (leepc12@gmail.com) at ENCODE-DCC' - description: 'ENCODE TF/Histone ChIP-Seq pipeline' + version: 'v2.0.1' + + author: 'Jin wook Lee' + email: 'leepc12@gmail.com' + description: 'ENCODE TF/Histone ChIP-Seq pipeline. See https://github.com/ENCODE-DCC/chip-seq-pipeline2 for more details. e.g. example input JSON for Terra/Anvil.' + organization: 'ENCODE DCC' + specification_document: 'https://docs.google.com/document/d/1lG_Rd7fnYgRpSIqrIfuVlAz2dW1VaSQThzk836Db99c/edit?usp=sharing' - default_docker: 'encodedcc/chip-seq-pipeline:v2.0.0' - default_singularity: 'library://leepc12/default/chip-seq-pipeline:v2.0.0' + default_docker: 'encodedcc/chip-seq-pipeline:v2.0.1' + default_singularity: 'library://leepc12/default/chip-seq-pipeline:v2.0.1' croo_out_def: 'https://storage.googleapis.com/encode-pipeline-output-definition/chip.croo.v5.json' parameter_group: { @@ -67,8 +71,8 @@ workflow chip { } input { # group: runtime_environment - String docker = 'encodedcc/chip-seq-pipeline:v2.0.0' - String singularity = 'library://leepc12/default/chip-seq-pipeline:v2.0.0' + String docker = 'encodedcc/chip-seq-pipeline:v2.0.1' + String singularity = 'library://leepc12/default/chip-seq-pipeline:v2.0.1' String conda = 'encode-chip-seq-pipeline' String conda_macs2 = 'encode-chip-seq-pipeline-macs2' String conda_spp = 'encode-chip-seq-pipeline-spp' @@ -117,9 +121,9 @@ workflow chip { Array[File] bams = [] Array[File] nodup_bams = [] Array[File] tas = [] - Array[File?] peaks = [] - Array[File?] peaks_pr1 = [] - Array[File?] peaks_pr2 = [] + Array[File] peaks = [] + Array[File] peaks_pr1 = [] + Array[File] peaks_pr2 = [] File? peak_ppr1 File? peak_ppr2 File? peak_pooled @@ -1703,7 +1707,7 @@ workflow chip { # we have all tas and ctl_tas (optional for histone chipseq) ready, let's call peaks scatter(i in range(num_rep)) { Boolean has_input_of_call_peak = defined(ta_[i]) - Boolean has_output_of_call_peak = i