Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ora reference for tn workflows #761

Merged
merged 2 commits into from
Dec 9, 2024
Merged

Add ora reference for tn workflows #761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a01ecd
Add ora reference for tn workflows
alexiswl Nov 14, 2024
1b023c7
Add ora step is a json path
alexiswl Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions config/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,13 @@ export const dragenIcav2ReferenceUriMappingSSMParameterPath =
// }
// ]
// '
export const dragenIcav2OraReferenceUriSSMParameterPath =
'/icav2/umccr-prod/dragen_ora_reference_uri';

// Deployed under dev // FIXME
// 'icav2://reference-data/dragen-ora/v2/ora_reference_v2.tar.gz
//

export const icav2GencodeAnnotationUriMappingSSMParameterPath =
'/icav2/umccr-prod/wts_qc_annotation_mapping';

Expand Down
2 changes: 2 additions & 0 deletions config/stacks/tumorNormalPipelineManager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
icaEventPipeStackName,
icav2AccessTokenSecretName,
dragenIcav2ReferenceUriMappingSSMParameterPath,
dragenIcav2OraReferenceUriSSMParameterPath,
tnIcav2PipelineIdSSMParameterPath,
tnIcav2PipelineManagerDynamodbTableName,
tnIcav2PipelineWorkflowType,
Expand Down Expand Up @@ -54,5 +55,6 @@ export const getTnIcav2PipelineManagerStackProps = (
/* SSM Workflow Parameters */
defaultReferenceVersion: tnDefaultReferenceVersion,
referenceUriSsmPath: dragenIcav2ReferenceUriMappingSSMParameterPath,
oraReferenceUriSsmPath: dragenIcav2OraReferenceUriSSMParameterPath,
};
};
27 changes: 26 additions & 1 deletion lib/workload/stateless/stacks/tumor-normal-pipeline-manager/deploy/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export interface TnIcav2PipelineManagerConfig {
icav2TokenSecretId: string; // "/icav2/umccr-prod/service-production-jwt-token-secret-arn"
pipelineIdSsmPath: string; // List of parameters the workflow session state machine will need access to
referenceUriSsmPath: string; // "/icav2/umccr-prod/reference-genome-uri"
oraReferenceUriSsmPath: string; // "/icav2/umccr-prod/ora-reference-uri-ssm-path"
defaultReferenceVersion: string;
/* Table to store analyis metadata */
dynamodbTableName: string;
Expand Down Expand Up @@ -79,6 +80,11 @@ export class TnIcav2PipelineManagerStack extends cdk.Stack {
props.referenceUriSsmPath,
props.referenceUriSsmPath
);
const oraReferenceSsmObj = ssm.StringParameter.fromStringParameterName(
this,
props.oraReferenceUriSsmPath,
props.oraReferenceUriSsmPath
);

// Get event bus object
this.eventBusObj = events.EventBus.fromEventBusName(this, 'event_bus', props.eventBusName);
Expand Down Expand Up @@ -107,6 +113,19 @@ export class TnIcav2PipelineManagerStack extends cdk.Stack {
timeout: Duration.seconds(60),
}
);
const addOraReferenceLambdaObj = new PythonFunction(
this,
'add_ora_reference_lambda_python_function',
{
entry: path.join(__dirname, '../lambdas/add_ora_reference_py'),
runtime: lambda.Runtime.PYTHON_3_12,
architecture: lambda.Architecture.ARM_64,
index: 'add_ora_reference.py',
handler: 'handler',
memorySize: 1024,
timeout: Duration.seconds(60),
}
);

// Specify the statemachine and replace the arn placeholders with the lambda arns defined above
const configureInputsSfn = new sfn.StateMachine(
Expand All @@ -124,13 +143,16 @@ export class TnIcav2PipelineManagerStack extends cdk.Stack {
__table_name__: this.dynamodbTableObj.tableName,
/* SSM Parameters */
__reference_version_uri_ssm_parameter_name__: referenceSsmObj.parameterName,
__ora_reference_uri_ssm_parameter_path__: oraReferenceSsmObj.parameterName,
__default_reference_version__: props.defaultReferenceVersion,
// We collect the reference version AND the pipeline versions
/* Lambdas */
__convert_fastq_list_rows_lambda_function_arn__:
convertFastqListRowsToCwlInputObjectsLambdaObj.currentVersion.functionArn,
__get_boolean_parameters_lambda_function_arn__:
getBooleanParametersFromEventInputLambdaObj.currentVersion.functionArn,
__add_ora_reference_lambda_function_arn__:
addOraReferenceLambdaObj.currentVersion.functionArn,
},
}
);
Expand All @@ -139,6 +161,7 @@ export class TnIcav2PipelineManagerStack extends cdk.Stack {
[
convertFastqListRowsToCwlInputObjectsLambdaObj,
getBooleanParametersFromEventInputLambdaObj,
addOraReferenceLambdaObj,
].forEach((lambdaObj) => {
lambdaObj.currentVersion.grantInvoke(configureInputsSfn);
});
Expand All @@ -147,7 +170,9 @@ export class TnIcav2PipelineManagerStack extends cdk.Stack {
this.dynamodbTableObj.grantReadWriteData(configureInputsSfn);

// Allow state machine to read ssm parameters
referenceSsmObj.grantRead(configureInputsSfn);
[referenceSsmObj, oraReferenceSsmObj].forEach((ssmObj) => {
ssmObj.grantRead(configureInputsSfn);
});

/*
Part 2: Configure the lambdas and outputs step function
Expand Down
148 changes: 148 additions & 0 deletions ...ss/stacks/tumor-normal-pipeline-manager/lambdas/add_ora_reference_py/add_ora_reference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
#!/usr/bin/env python3

"""
Add ora reference
"""
from typing import Dict, Optional, List


def handler(event, context) -> Dict[str, bool]:
"""
Get the boolean parameters from the event input
:param event:
:param context:
:return: Dictionary of boolean parameters
"""

# Collect the event data input
event_data_input: Dict = event['event_data_input']

# Get the fastq list rows
tumor_fastq_list_rows: Optional[List] = event_data_input.get('tumorFastqListRows', None)
normal_fastq_list_rows: Optional[List] = event_data_input.get('fastqListRows', None)

# If tumorFastqListRows is None and fastqListRows is None, return false
if tumor_fastq_list_rows is None and normal_fastq_list_rows is None:
return {
"add_ora_step": False
}

for fastq_list_row_iter in [tumor_fastq_list_rows, normal_fastq_list_rows]:
if fastq_list_row_iter is not None:
# If fastqListRows is not None, return true
# Iterate over each of the fastq list rows, if one of the read1FileUri or read2FileUri end with .fastq.ora
# return true
if any(
[
row.get('read1FileUri', '').endswith('.fastq.ora') or
row.get('read2FileUri', '').endswith('.fastq.ora')
for row in fastq_list_row_iter
]
):
return {
"add_ora_step": True
}

# Got to here? Return false
return {
"add_ora_step": False
}


# if __name__ == "__main__":
# import json
#
# print(
# json.dumps(
# handler(
# {
# "event_data_input": {
# "tumorFastqListRows": [
# {
# "rgid": "ATGAGGCC.CAATTAAC.2",
# "rgsm": "L2400195",
# "rglb": "L2400195",
# "lane": 2,
# "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz",
# "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz"
# },
# {
# "rgid": "ATGAGGCC.CAATTAAC.3",
# "rgsm": "L2400195",
# "rglb": "L2400195",
# "lane": 3,
# "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz",
# "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz"
# }
# ],
# "fastqListRows": [
# {
# "rgid": "GCACGGAC.TGCGAGAC.4",
# "rgsm": "L2400191",
# "rglb": "L2400191",
# "lane": 4,
# "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.gz",
# "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.gz"
# }
# ],
# "dragenReferenceVersion": "v9-r3"
# }
# },
# None
# ),
# indent=4
# )
# )
#
# # {
# # "add_ora_step": false
# # }

# if __name__ == "__main__":
# import json
#
# print(
# json.dumps(
# handler(
# {
# "event_data_input": {
# "tumorFastqListRows": [
# {
# "rgid": "ATGAGGCC.CAATTAAC.2",
# "rgsm": "L2400195",
# "rglb": "L2400195",
# "lane": 2,
# "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R1_001.fastq.gz",
# "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_2/L2400195/L2400195_S9_L002_R2_001.fastq.gz"
# },
# {
# "rgid": "ATGAGGCC.CAATTAAC.3",
# "rgsm": "L2400195",
# "rglb": "L2400195",
# "lane": 3,
# "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R1_001.fastq.gz",
# "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_3/L2400195/L2400195_S9_L003_R2_001.fastq.gz"
# }
# ],
# "fastqListRows": [
# {
# "rgid": "GCACGGAC.TGCGAGAC.4",
# "rgsm": "L2400191",
# "rglb": "L2400191",
# "lane": 4,
# "read1FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R1_001.fastq.ora",
# "read2FileUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/primary/240229_A00130_0288_BH5HM2DSXC/2024071110689063/Samples/Lane_4/L2400191/L2400191_S17_L004_R2_001.fastq.ora"
# }
# ],
# "dragenReferenceVersion": "v9-r3"
# }
# },
# None
# ),
# indent=4
# )
# )
#
# # {
# # "add_ora_step": true
# # }
Loading
Loading