From 432cfc5d80a1fa04ab7241b13a9330dd9b72e609 Mon Sep 17 00:00:00 2001 From: amygutierrez Date: Thu, 20 Jul 2023 10:22:56 -0400 Subject: [PATCH 1/3] added correlations as separate workflow --- .github/workflows/regression_correlations.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/regression_correlations.yml diff --git a/.github/workflows/regression_correlations.yml b/.github/workflows/regression_correlations.yml new file mode 100644 index 0000000000..f81cf1bf5b --- /dev/null +++ b/.github/workflows/regression_correlations.yml @@ -0,0 +1,35 @@ +name: Run Regression Test Correlations + +on: + workflow_call: + secrets: + SSH_PRIVATE_KEY: + required: true + SSH_USER: + required: true + SSH_HOST: + required: true + SSH_WORK_DIR: + required: true + +jobs: + test: + name: Regression Correlations + runs-on: ubuntu-latest + steps: + - name: Install SSH Keys + run: | + mkdir -p ~/.ssh/ + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -H -t rsa ${{ secrets.SSH_HOST }} > ~/.ssh/known_hosts + + - name: Cleanup SSH + run: | + rm -rf ~/.ssh + + #- uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: logs + # path: output/*/*/log/ \ No newline at end of file From 4055443ffeaebbce4b884ccb5b85a3aa61edf04b Mon Sep 17 00:00:00 2001 From: amygutierrez Date: Thu, 20 Jul 2023 10:24:00 -0400 Subject: [PATCH 2/3] removed commented out code --- .github/workflows/regression_test_lite.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/regression_test_lite.yml b/.github/workflows/regression_test_lite.yml index f6c0b49ac3..097ddbc27f 100644 --- a/.github/workflows/regression_test_lite.yml +++ b/.github/workflows/regression_test_lite.yml @@ -1,9 +1,6 @@ name: Run Regression Lite Test - on: - #pull_request: - # types: [ready_for_review] workflow_call: secrets: SSH_PRIVATE_KEY: @@ -37,12 +34,6 @@ jobs: echo DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}" | tr '[:upper:]' '[:lower:]'):$TAG >> $GITHUB_ENV cat $GITHUB_ENV - #- name: Checkout Code - # uses: actions/checkout@v2 - #- name: Clone slurm_testing - # run: | - # git clone https://github.com/cmi-dair/slurm_testing - - name: Install SSH Keys run: | mkdir -p ~/.ssh/ @@ -68,12 +59,6 @@ jobs: fi bash slurm_testing/regression_run_scripts/regtest_lite.sh --home_dir ${{ secrets.SSH_WORK_DIR }} --image ${{ env.DOCKER_TAG }} - #- name: Correlating - # run: | - # echo "Correlating lite regression runs" - # echo "(ﾉ◕ヮ◕)ﾉ*:･ﾟ✧" - # python3 reg-suite/main.py correlate --pipeline1 $GITHUB_WORKSPACE/output/default/HNU_1 --pipeline2 $GITHUB_WORKSPACE/lite/default/HNU_1--workspace $GITHUB_WORKSPACE - - name: Cleanup SSH run: | rm -rf ~/.ssh From 686f0a86053de9c32481b3a698266a7e0883b30c Mon Sep 17 00:00:00 2001 From: amygutierrez Date: Fri, 21 Jul 2023 13:05:49 -0400 Subject: [PATCH 3/3] added instructions/template for the corerlations scripts --- .github/workflows/regression_correlations.yml | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/regression_correlations.yml b/.github/workflows/regression_correlations.yml index f81cf1bf5b..b41be6b536 100644 --- a/.github/workflows/regression_correlations.yml +++ b/.github/workflows/regression_correlations.yml @@ -17,6 +17,10 @@ jobs: name: Regression Correlations runs-on: ubuntu-latest steps: + - name: Get C-PAC branch + run: | + GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-) + - name: Install SSH Keys run: | mkdir -p ~/.ssh/ @@ -24,9 +28,34 @@ jobs: chmod 600 ~/.ssh/id_rsa ssh-keyscan -H -t rsa ${{ secrets.SSH_HOST }} > ~/.ssh/known_hosts + - name: Connect and Run Regression Test Lite + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USER }} + key: ${{ secrets.SSH_PRIVATE_KEY }} + command_timeout: 200m + script: | + cd ${{ secrets.SSH_WORK_DIR }} + if [ ! -d regression_dashboard ] ; then + git clone https://github.com/amygutierrez/regression_dashboard.git + else + cd regression_dashboard + git pull origin main + cd .. + fi + #find where the cpac run outputs are saved. Repeat this for all data sources (may need to loop) + DATA_SOURCES="HNU_1 Site-CBIC Site-SI" + python3 regression_dashboard/create_yaml.py --pipeline1 --pipeline2 --workspace ${{ secrets.SSH_WORK_DIR }} --branch $GITHUB_BRANCH --data_source + python3 regression_dashboard/calculate_correlations.py output.yaml --data_source + + #this one doesn't need to be in the data_sorces loop + python3 build_dashboard.py --json_files HNU_1_regression_reg-suite.json,KKI_regression_reg-suite.json --branch $GITHUB_BRANCH + + - name: Cleanup SSH run: | - rm -rf ~/.ssh + rm -rf ~/.ssh #- uses: actions/upload-artifact@v3 # if: always()