Merge pull request #2004 from splunk/develop #18837
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: detection-testing | |
| on: | |
| push: | |
| pull_request: | |
| types: [opened, reopened] | |
| schedule: | |
| - cron: "44 4 * * *" | |
| jobs: | |
| validate-tag-if-present: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: TAGGED, Validate that the tag is in the correct format | |
| run: | | |
| echo "The GITHUB_REF: $GITHUB_REF" | |
| #First check to see if the release is a tag | |
| if [[ $GITHUB_REF =~ refs/tags/* ]]; then | |
| #Yes, this is a tag, so we need to test to make sure that the tag | |
| #is in the correct format (like v1.10.20) | |
| if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then | |
| echo "PASS: Tagged release with good format" | |
| exit 0 | |
| else | |
| echo "FAIL: Tagged release with bad format" | |
| exit 1 | |
| fi | |
| else | |
| echo "PASS: Not a tagged release" | |
| exit 0 | |
| fi | |
| quit-for-dependabot: | |
| runs-on: ubuntu-latest | |
| if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' | |
| steps: | |
| - name: "Placeholder" | |
| run: | | |
| echo "yes it ran" | |
| docker-detection-testing-setup: | |
| runs-on: ubuntu-latest | |
| needs: [validate-tag-if-present, quit-for-dependabot] | |
| steps: | |
| - name: Get branch and PR required for detection testing main.py | |
| id: vars | |
| run: | | |
| echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| - name: Checkout Repo | |
| uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy | |
| architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified | |
| - name: Install Python Dependencies | |
| run: | | |
| cd bin/automated_detection_testing/ci/detection_testing_batch | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python3 -m pip install wheel | |
| python3 -m pip install -r requirements.txt | |
| - name: Run the CI | |
| run: | | |
| cd bin/automated_detection_testing/ci/detection_testing_batch | |
| source .venv/bin/activate | |
| echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]" | |
| echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]" | |
| echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]" | |
| echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]" | |
| echo "github.event_name : [${{ github.event_name }}]" | |
| if [[ ${{ github.event_name }} == schedule ]]; then | |
| # Note that scheduled actions ONLY run on the default branch, so it won't run on all other branches! | |
| echo "Running a nightly test on all detections OR a commit was made directly to develop" | |
| python detection_testing_execution.py run --branch develop --mode all --mock --config_file test_config_github_actions.json | |
| elif [[ ! -z "${{ github.event.pull_request.head.ref }}" && ! -z "${{ github.event.pull_request.number }}" ]]; then | |
| echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]" | |
| python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }} --mode changes --mock --config_file test_config_github_actions.json | |
| else | |
| echo "Push from branch [${{ steps.vars.outputs.branch }}]" | |
| python detection_testing_execution.py run --branch ${{ steps.vars.outputs.branch }} --mode changes --mock --config_file test_config_github_actions.json | |
| fi | |
| mv *-test-run.json replicate_test.json | |
| - name: Upload Test Results Files | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: testing-results-config | |
| path: | | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_0.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_1.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_2.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_3.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_4.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_5.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_6.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_7.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_8.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/prior_config/config_tests_9.json | |
| - name: Upload File to Enable Replication of the Test at a Different Time or Place | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: replicate_test | |
| path: | | |
| bin/automated_detection_testing/ci/detection_testing_batch/replicate_test.json | |
| docker-detection-testing-execution: | |
| runs-on: ubuntu-latest | |
| needs: [docker-detection-testing-setup] | |
| strategy: | |
| matrix: | |
| manifest_filename: ["config_tests_0.json", | |
| "config_tests_1.json", | |
| "config_tests_2.json", | |
| "config_tests_3.json", | |
| "config_tests_4.json", | |
| "config_tests_5.json", | |
| "config_tests_6.json", | |
| "config_tests_7.json", | |
| "config_tests_8.json", | |
| "config_tests_9.json"] | |
| steps: | |
| - name: Get branch and PR required for detection testing main.py | |
| id: vars | |
| run: | | |
| echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| - name: Checkout Repo | |
| uses: actions/checkout@v2 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: testing-results-config | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/prior_config | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy | |
| architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified | |
| - name: Install Python Dependencies | |
| run: | | |
| cd bin/automated_detection_testing/ci/detection_testing_batch | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python3 -m pip install wheel | |
| python3 -m pip install -r requirements.txt | |
| - name: Run the CI | |
| run: | | |
| cd bin/automated_detection_testing/ci/detection_testing_batch | |
| source .venv/bin/activate | |
| python3 detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} --splunkbase_username ${{ secrets.SPLUNKBASE_TESTING_USERNAME }} --splunkbase_password ${{ secrets.SPLUNKBASE_TESTING_KEY }} | |
| - name: Upload Test Results Files | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: ${{ matrix.manifest_filename}}.results | |
| path: | | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/success.csv | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/error.csv | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/failure.csv | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.csv | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/success.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/error.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/failure.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/combined.json | |
| bin/automated_detection_testing/ci/detection_testing_batch/test_results/summary.json | |
| docker-detection-testing-execution-merge-results: | |
| runs-on: ubuntu-latest | |
| needs: [docker-detection-testing-setup, docker-detection-testing-execution] | |
| steps: | |
| - name: Get branch and PR required for detection testing main.py | |
| id: vars | |
| run: | | |
| echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| - name: Checkout Repo | |
| uses: actions/checkout@v2 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_0.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_0 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_1.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_1 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_2.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_2 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_3.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_3 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_4.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_4 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_5.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_5 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_6.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_6 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_7.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_7 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_8.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_8 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_9.json.results | |
| path: bin/automated_detection_testing/ci/detection_testing_batch/results_9 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy | |
| architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified | |
| - name: Install Python Dependencies | |
| run: | | |
| cd bin/automated_detection_testing/ci/detection_testing_batch | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| python3 -m pip install wheel | |
| python3 -m pip install -r requirements.txt | |
| - name: Merge Detections into single File | |
| run: | | |
| cd bin/automated_detection_testing/ci/detection_testing_batch | |
| source .venv/bin/activate | |
| python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json | |
| - name: Upload Summary Test Results JSON | |
| uses: actions/upload-artifact@v2 | |
| if: always() | |
| with: | |
| name: SummaryTestResults | |
| path: | | |
| bin/automated_detection_testing/ci/detection_testing_batch/summary_test_results.json | |
| - name: Upload Failures Manifest on Failure | |
| uses: actions/upload-artifact@v2 | |
| if: failure() | |
| with: | |
| name: DetectionFailureManifest | |
| path: | | |
| bin/automated_detection_testing/ci/detection_testing_batch/detection_failure_manifest.json | |
| #Always clean these up, they make the output messy | |
| - name: Clean up intermediate Files | |
| uses: geekyeggo/delete-artifact@v1 | |
| if: always() | |
| with: | |
| name: | | |
| config_tests_0.json.results | |
| config_tests_1.json.results | |
| config_tests_2.json.results | |
| config_tests_3.json.results | |
| config_tests_4.json.results | |
| config_tests_5.json.results | |
| config_tests_6.json.results | |
| config_tests_7.json.results | |
| config_tests_8.json.results | |
| config_tests_9.json.results | |