From 1c038e3db911260a29e60bd58bbfc7038782b719 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:59:56 -0500 Subject: [PATCH 01/28] Create codeql_scan.yml --- .gitlab/codeql_scan.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .gitlab/codeql_scan.yml diff --git a/.gitlab/codeql_scan.yml b/.gitlab/codeql_scan.yml new file mode 100644 index 00000000000000..1f2031956d3d04 --- /dev/null +++ b/.gitlab/codeql_scan.yml @@ -0,0 +1,31 @@ +--- +# codeql_scan stage +# Contains CodeQL scan job to perform security static analysis + +run_codeql_scan: + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES + tags: ["arch:arm64"] + stage: codeql_scan + # rules: + # - if: $CI_PIPELINE_SOURCE == "schedule" && $CODE_SCANNING_PIPELINE == "true" + # when: always + # - when: never + variables: + ARCH: arm64 + BASE_REF: main + GOMAXPROCS: 10 + KUBERNETES_CPU_REQUEST: 10 + KUBERNETES_CPU_LIMIT: 10 + KUBERNETES_MEMORY_REQUEST: 64Gi + KUBERNETES_MEMORY_LIMIT: 64Gi + GITHUB_APP_PRIVATE_KEY_NAME: csec.codescanning.githubapp.privatekey + CODEQL: /usr/local/codeql/codeql + CODEQL_DB: /tmp/dd-source.codeql + PYTHON_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/python/codeql-suites/custom.qls + GO_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/golang/codeql-suites/dd-source.qls + DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript --quiet + SCAN_CONFIGS: --format sarifv2.1.0 --threads 8 --ram 96000 --no-tuple-counting --quiet + UPLOAD_CONFIGS: -upload_sarif=true + script: + - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning + - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS From 9b41bed8bc9d3ea7182d3cf5a0a848cc517a3522 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:01:31 -0500 Subject: [PATCH 02/28] testing codeql scan on gitlab --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 290f3484f3bfd5..7ae477de3e68f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ include: - .gitlab/.pre/include.yml - .gitlab/benchmarks/include.yml - .gitlab/binary_build/include.yml + - /.gitlab/codeql_scan.yml - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml @@ -58,6 +59,7 @@ stages: - .pre - setup - maintenance_jobs + - codeql_scan - deps_build - deps_fetch - lint From da6921659d9c221f5119ca41edb0394c42c8df36 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:38:18 -0500 Subject: [PATCH 03/28] Update .gitlab-ci.yml --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ae477de3e68f4..2a3158745f1712 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,7 +59,6 @@ stages: - .pre - setup - maintenance_jobs - - codeql_scan - deps_build - deps_fetch - lint From 363ef14ad734ce89eb82a00bdc9ce01c08c775bb Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:38:57 -0500 Subject: [PATCH 04/28] Update codeql_scan.yml --- .gitlab/codeql_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/codeql_scan.yml b/.gitlab/codeql_scan.yml index 1f2031956d3d04..7b6260eaeaf8d3 100644 --- a/.gitlab/codeql_scan.yml +++ b/.gitlab/codeql_scan.yml @@ -5,7 +5,7 @@ run_codeql_scan: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES tags: ["arch:arm64"] - stage: codeql_scan + stage: source_test # rules: # - if: $CI_PIPELINE_SOURCE == "schedule" && $CODE_SCANNING_PIPELINE == "true" # when: always From 756349543645a6e7f4ec0f4bcd3a0e449c5fff17 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv Date: Wed, 5 Feb 2025 18:49:09 -0500 Subject: [PATCH 05/28] adding codeowners --- .github/CODEOWNERS | 2 ++ .gitlab/{ => codeql_scan}/codeql_scan.yml | 0 2 files changed, 2 insertions(+) rename .gitlab/{ => codeql_scan}/codeql_scan.yml (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 930956b2669903..1a7868c8f8a7b7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -121,6 +121,8 @@ /.gitlab/binary_build/system_probe.yml @DataDog/ebpf-platform @DataDog/agent-delivery /.gitlab/binary_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/codeql_scan/ @DataDog/sdlc-security + /.gitlab/benchmarks/ @DataDog/agent-devx-infra @DataDog/apm-ecosystems-performance @DataDog/agent-apm /.gitlab/deploy_containers/ @DataDog/container-integrations @DataDog/agent-delivery diff --git a/.gitlab/codeql_scan.yml b/.gitlab/codeql_scan/codeql_scan.yml similarity index 100% rename from .gitlab/codeql_scan.yml rename to .gitlab/codeql_scan/codeql_scan.yml From 3397d967a02312e1fd2f89f2080481c395c72d7a Mon Sep 17 00:00:00 2001 From: ganeshkumarsv Date: Wed, 5 Feb 2025 19:26:51 -0500 Subject: [PATCH 06/28] just yoloing to run the scan --- .gitlab/source_test/notify.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/source_test/notify.yml b/.gitlab/source_test/notify.yml index 05fc53bc0829ef..81176b0556849b 100644 --- a/.gitlab/source_test/notify.yml +++ b/.gitlab/source_test/notify.yml @@ -22,4 +22,5 @@ unit_tests_notify: - tests_flavor_iot_deb-x64 - tests_flavor_dogstatsd_deb-x64 - tests_flavor_heroku_deb-x64 + - run_codeql_scan allow_failure: true From a9e42fba4c732fe93c7e9ff4b5551b12a5d39b99 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv Date: Wed, 5 Feb 2025 19:28:16 -0500 Subject: [PATCH 07/28] fix codeql_scan yml path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a3158745f1712..7ea52c276bed06 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ include: - .gitlab/.pre/include.yml - .gitlab/benchmarks/include.yml - .gitlab/binary_build/include.yml - - /.gitlab/codeql_scan.yml + - /.gitlab/codeql_scan/codeql_scan.yml - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml From 605fa44c23f647d7b3709853895cc5755ff1dcca Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 5 Feb 2025 19:31:14 -0500 Subject: [PATCH 08/28] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ea52c276bed06..a6b9ca4861f224 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ include: - .gitlab/.pre/include.yml - .gitlab/benchmarks/include.yml - .gitlab/binary_build/include.yml - - /.gitlab/codeql_scan/codeql_scan.yml + - .gitlab/codeql_scan/codeql_scan.yml - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml From becd25fddead70d8b53b47d598ef398b72634466 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv Date: Wed, 5 Feb 2025 19:38:52 -0500 Subject: [PATCH 09/28] fix rules --- .gitlab/codeql_scan/codeql_scan.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/codeql_scan/codeql_scan.yml b/.gitlab/codeql_scan/codeql_scan.yml index 7b6260eaeaf8d3..111c3e04faa3be 100644 --- a/.gitlab/codeql_scan/codeql_scan.yml +++ b/.gitlab/codeql_scan/codeql_scan.yml @@ -6,10 +6,8 @@ run_codeql_scan: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES tags: ["arch:arm64"] stage: source_test - # rules: - # - if: $CI_PIPELINE_SOURCE == "schedule" && $CODE_SCANNING_PIPELINE == "true" - # when: always - # - when: never + rules: + - when: on_success variables: ARCH: arm64 BASE_REF: main From b67c9a3d255bc0a7c9ae87716c565d9b1336ee81 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 6 Feb 2025 10:10:04 +0100 Subject: [PATCH 10/28] Fix needs and build the agent binary --- .gitlab-ci.yml | 1 - .gitlab/{codeql_scan => source_test}/codeql_scan.yml | 3 +++ .gitlab/source_test/include.yml | 1 + .gitlab/source_test/notify.yml | 1 - 4 files changed, 4 insertions(+), 2 deletions(-) rename .gitlab/{codeql_scan => source_test}/codeql_scan.yml (93%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6b9ca4861f224..290f3484f3bfd5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ include: - .gitlab/.pre/include.yml - .gitlab/benchmarks/include.yml - .gitlab/binary_build/include.yml - - .gitlab/codeql_scan/codeql_scan.yml - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml diff --git a/.gitlab/codeql_scan/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml similarity index 93% rename from .gitlab/codeql_scan/codeql_scan.yml rename to .gitlab/source_test/codeql_scan.yml index 111c3e04faa3be..eabc318dff21f1 100644 --- a/.gitlab/codeql_scan/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -8,6 +8,7 @@ run_codeql_scan: stage: source_test rules: - when: on_success + needs: ["go_deps"] variables: ARCH: arm64 BASE_REF: main @@ -25,5 +26,7 @@ run_codeql_scan: SCAN_CONFIGS: --format sarifv2.1.0 --threads 8 --ram 96000 --no-tuple-counting --quiet UPLOAD_CONFIGS: -upload_sarif=true script: + - !reference [.retrieve_linux_go_deps] + - inv -e agent.build - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS diff --git a/.gitlab/source_test/include.yml b/.gitlab/source_test/include.yml index 3a7f051b856c1b..83d092c55574cc 100644 --- a/.gitlab/source_test/include.yml +++ b/.gitlab/source_test/include.yml @@ -15,3 +15,4 @@ include: - .gitlab/source_test/notify.yml - .gitlab/source_test/protobuf.yml - .gitlab/source_test/tooling_unit_tests.yml + - .gitlab/source_test/codeql_scan.yml diff --git a/.gitlab/source_test/notify.yml b/.gitlab/source_test/notify.yml index 81176b0556849b..05fc53bc0829ef 100644 --- a/.gitlab/source_test/notify.yml +++ b/.gitlab/source_test/notify.yml @@ -22,5 +22,4 @@ unit_tests_notify: - tests_flavor_iot_deb-x64 - tests_flavor_dogstatsd_deb-x64 - tests_flavor_heroku_deb-x64 - - run_codeql_scan allow_failure: true From e90f05d640e6951ced2ad8f7cde3bfd097e7f1cd Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 6 Feb 2025 10:28:46 +0100 Subject: [PATCH 11/28] Change base image --- .gitlab/source_test/codeql_scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index eabc318dff21f1..3ee40b5db0bb1f 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -3,8 +3,8 @@ # Contains CodeQL scan job to perform security static analysis run_codeql_scan: - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + tags: ["arch:amd64"] stage: source_test rules: - when: on_success From a54254f3ff4e0e1d87b57bc769c5e1c5210ac9ea Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 6 Feb 2025 10:34:27 +0100 Subject: [PATCH 12/28] Add JOBOWNER --- .gitlab/JOBOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index f6e39584e9e040..76ebb6865d05b7 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -167,3 +167,6 @@ single_machine_performance* @DataDog/single-machine-performance # Dependency Security software_composition_analysis* @DataDog/sdlc-security + +# CodeQL +run_codeql_scan @DataDog/sdlc-security From 9a6025652b6c0e67b6bd17d9e6b4d73386806cfa Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 6 Feb 2025 10:38:29 +0100 Subject: [PATCH 13/28] Fix codeowner + allow to fail --- .github/CODEOWNERS | 2 +- .gitlab/source_test/codeql_scan.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1a7868c8f8a7b7..b471822076a3b1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -121,7 +121,7 @@ /.gitlab/binary_build/system_probe.yml @DataDog/ebpf-platform @DataDog/agent-delivery /.gitlab/binary_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent -/.gitlab/codeql_scan/ @DataDog/sdlc-security +/.gitlab/source_test/codeql_scan.yml @DataDog/sdlc-security /.gitlab/benchmarks/ @DataDog/agent-devx-infra @DataDog/apm-ecosystems-performance @DataDog/agent-apm diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index 3ee40b5db0bb1f..f5f921b10757a8 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -9,6 +9,7 @@ run_codeql_scan: rules: - when: on_success needs: ["go_deps"] + allow_failure: true # This job should not impact the overall status of the pipeline variables: ARCH: arm64 BASE_REF: main From d707b0cca2179e94e5e5006bddc40a8e5c759f33 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Thu, 6 Feb 2025 11:12:51 +0100 Subject: [PATCH 14/28] Exclude systemd --- .gitlab/source_test/codeql_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index f5f921b10757a8..b948ff9437238d 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -28,6 +28,6 @@ run_codeql_scan: UPLOAD_CONFIGS: -upload_sarif=true script: - !reference [.retrieve_linux_go_deps] - - inv -e agent.build + - inv -e agent.build --build-exclude=systemd - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS From e1fb85be398ab7f22c2b6b5b7bc353413e647e3b Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:57:45 -0500 Subject: [PATCH 15/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index b948ff9437238d..96ad0e8fb19d32 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -26,8 +26,11 @@ run_codeql_scan: DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript --quiet SCAN_CONFIGS: --format sarifv2.1.0 --threads 8 --ram 96000 --no-tuple-counting --quiet UPLOAD_CONFIGS: -upload_sarif=true + CODEQL_BUNDLE_URL: https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.17.5/codeql-bundle-linux64.tar.gz script: - !reference [.retrieve_linux_go_deps] - inv -e agent.build --build-exclude=systemd + - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning + - curl -L $CODEQL_BUNDLE_URL > /tmp/codeql.tar.gz && tar -zxf /tmp/codeql.tar.gz -C /usr/local \ && rm /tmp/codeql.tar.gz - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS From 118e6cd41af6e833a18beb6fec5c985cc9022a11 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Fri, 7 Feb 2025 14:13:48 -0500 Subject: [PATCH 16/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index 96ad0e8fb19d32..cb05ae3a9a9d29 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -32,5 +32,5 @@ run_codeql_scan: - inv -e agent.build --build-exclude=systemd - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning - - curl -L $CODEQL_BUNDLE_URL > /tmp/codeql.tar.gz && tar -zxf /tmp/codeql.tar.gz -C /usr/local \ && rm /tmp/codeql.tar.gz + - curl -L $CODEQL_BUNDLE_URL -o /tmp/codeql.tar.gz && tar -zxf /tmp/codeql.tar.gz -C /usr/local && rm /tmp/codeql.tar.gz - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS From 1c6a26357730e6576f63395e4c7824a608d61a79 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:05:29 -0500 Subject: [PATCH 17/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index cb05ae3a9a9d29..d31e201dd2a111 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -33,4 +33,10 @@ run_codeql_scan: - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning - curl -L $CODEQL_BUNDLE_URL -o /tmp/codeql.tar.gz && tar -zxf /tmp/codeql.tar.gz -C /usr/local && rm /tmp/codeql.tar.gz - - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS + - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" + - echo "CGO_CFLAGS=${CGO_CFLAGS}" + - export CGO_LDFLAGS="-L${PWD}/rtloader/build/rtloader -ldl" + - export CGO_CFLAGS="-I${PWD}/rtloader/include -I${PWD}/rtloader/common" + - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" + - echo "CGO_CFLAGS=${CGO_CFLAGS}" + - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="invoke install-tools && invoke deps && invoke agent.build --build-exclude=systemd" From 2c9b578df2a7006b56086c48ee64dc3f422dc9cf Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Fri, 7 Feb 2025 18:33:47 -0500 Subject: [PATCH 18/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index d31e201dd2a111..9e64e332c114e1 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -39,4 +39,7 @@ run_codeql_scan: - export CGO_CFLAGS="-I${PWD}/rtloader/include -I${PWD}/rtloader/common" - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" - echo "CGO_CFLAGS=${CGO_CFLAGS}" - - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="invoke install-tools && invoke deps && invoke agent.build --build-exclude=systemd" + - invoke install-tools + - invoke deps + - invoke agent.build --build-exclude=systemd + - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS From 5ed4cc021059b63e879d65e62d2ff550460498fa Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Mon, 10 Feb 2025 07:10:06 -0500 Subject: [PATCH 19/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index 9e64e332c114e1..af7529069142ef 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -29,7 +29,6 @@ run_codeql_scan: CODEQL_BUNDLE_URL: https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.17.5/codeql-bundle-linux64.tar.gz script: - !reference [.retrieve_linux_go_deps] - - inv -e agent.build --build-exclude=systemd - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning - curl -L $CODEQL_BUNDLE_URL -o /tmp/codeql.tar.gz && tar -zxf /tmp/codeql.tar.gz -C /usr/local && rm /tmp/codeql.tar.gz @@ -39,7 +38,4 @@ run_codeql_scan: - export CGO_CFLAGS="-I${PWD}/rtloader/include -I${PWD}/rtloader/common" - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" - echo "CGO_CFLAGS=${CGO_CFLAGS}" - - invoke install-tools - - invoke deps - - invoke agent.build --build-exclude=systemd - - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS + - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="inv -e agent.build --build-exclude=systemd" From 268be662d02efbef4bc2841819da1db173b5f7d6 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Mon, 10 Feb 2025 08:53:57 -0500 Subject: [PATCH 20/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index af7529069142ef..f6d4f0251a717e 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -20,7 +20,7 @@ run_codeql_scan: KUBERNETES_MEMORY_LIMIT: 64Gi GITHUB_APP_PRIVATE_KEY_NAME: csec.codescanning.githubapp.privatekey CODEQL: /usr/local/codeql/codeql - CODEQL_DB: /tmp/dd-source.codeql + CODEQL_DB: /tmp/datadog-agent.codeql PYTHON_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/python/codeql-suites/custom.qls GO_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/golang/codeql-suites/dd-source.qls DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript --quiet @@ -38,4 +38,14 @@ run_codeql_scan: - export CGO_CFLAGS="-I${PWD}/rtloader/include -I${PWD}/rtloader/common" - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" - echo "CGO_CFLAGS=${CGO_CFLAGS}" + # Todo: Add CPP - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="inv -e agent.build --build-exclude=systemd" + - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" + - $CODEQL database analyze "$CODEQL_DB"/javascript codeql/javascript-queries $SCAN_CONFIGS --sarif-category="javascript" --output="/tmp/javascript.sarif" + - $CODEQL database analyze "$CODEQL_DB"/python "$PYTHON_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="python" --output="/tmp/python.sarif" + artifacts: + paths: + - /tmp/go.sarif + - /tmp/javascript.sarif + - /tmp/python.sarif + From dc023be72867ad1bdd76f18d1ead6df618729828 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:34:53 -0500 Subject: [PATCH 21/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index f6d4f0251a717e..7b8ffb4d0243c8 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -23,7 +23,7 @@ run_codeql_scan: CODEQL_DB: /tmp/datadog-agent.codeql PYTHON_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/python/codeql-suites/custom.qls GO_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/golang/codeql-suites/dd-source.qls - DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript --quiet + DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript,cpp --quiet SCAN_CONFIGS: --format sarifv2.1.0 --threads 8 --ram 96000 --no-tuple-counting --quiet UPLOAD_CONFIGS: -upload_sarif=true CODEQL_BUNDLE_URL: https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.17.5/codeql-bundle-linux64.tar.gz @@ -43,9 +43,15 @@ run_codeql_scan: - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" - $CODEQL database analyze "$CODEQL_DB"/javascript codeql/javascript-queries $SCAN_CONFIGS --sarif-category="javascript" --output="/tmp/javascript.sarif" - $CODEQL database analyze "$CODEQL_DB"/python "$PYTHON_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="python" --output="/tmp/python.sarif" + - $CODEQL database analyze "$CODEQL_DB"/cpp codeql/cpp-queries $SCAN_CONFIGS --sarif-category="cpp" --output="/tmp/cpp.sarif" + - mv /tmp/go.sarif . + - mv /tmp/javascript.sarif . + - mv /tmp/python.sarif . + - mv /tmp/cpp.sarif . artifacts: paths: - - /tmp/go.sarif - - /tmp/javascript.sarif - - /tmp/python.sarif + - go.sarif + - javascript.sarif + - python.sarif + - cpp.sarif From 6c76340e1ad6799aeb69b06a0e8f8e0ef5b10eb6 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Mon, 10 Feb 2025 12:42:11 -0500 Subject: [PATCH 22/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index 7b8ffb4d0243c8..a4210de2136da1 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -26,7 +26,7 @@ run_codeql_scan: DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript,cpp --quiet SCAN_CONFIGS: --format sarifv2.1.0 --threads 8 --ram 96000 --no-tuple-counting --quiet UPLOAD_CONFIGS: -upload_sarif=true - CODEQL_BUNDLE_URL: https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.17.5/codeql-bundle-linux64.tar.gz + CODEQL_BUNDLE_URL: https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.20.4/codeql-bundle-linux64.tar.gz script: - !reference [.retrieve_linux_go_deps] - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" From e0760810d626e948090b0f6bf8a80610a0a0591b Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:34:21 -0500 Subject: [PATCH 23/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index a4210de2136da1..d4a9117dbcd3ca 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -38,7 +38,7 @@ run_codeql_scan: - export CGO_CFLAGS="-I${PWD}/rtloader/include -I${PWD}/rtloader/common" - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" - echo "CGO_CFLAGS=${CGO_CFLAGS}" - # Todo: Add CPP + - GOPROXY=https://proxy.golang.org,https://goproxy.io,direct invoke install-tools - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="inv -e agent.build --build-exclude=systemd" - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" - $CODEQL database analyze "$CODEQL_DB"/javascript codeql/javascript-queries $SCAN_CONFIGS --sarif-category="javascript" --output="/tmp/javascript.sarif" From 51197fdf9f753acb6d332c30116b18f21be30305 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Tue, 11 Feb 2025 18:06:52 -0500 Subject: [PATCH 24/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index d4a9117dbcd3ca..1b194e5cf5fbb0 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -38,6 +38,7 @@ run_codeql_scan: - export CGO_CFLAGS="-I${PWD}/rtloader/include -I${PWD}/rtloader/common" - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" - echo "CGO_CFLAGS=${CGO_CFLAGS}" + - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest - GOPROXY=https://proxy.golang.org,https://goproxy.io,direct invoke install-tools - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="inv -e agent.build --build-exclude=systemd" - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" From 582aaeb7e54e3e6735279eb715b6e673db0822a4 Mon Sep 17 00:00:00 2001 From: Kevin Fairise Date: Wed, 12 Feb 2025 13:52:46 +0100 Subject: [PATCH 25/28] Retrieve tooling deps --- .gitlab/source_test/codeql_scan.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index 1b194e5cf5fbb0..017ba13ba2234e 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -3,12 +3,12 @@ # Contains CodeQL scan job to perform security static analysis run_codeql_scan: - image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES + image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] stage: source_test rules: - when: on_success - needs: ["go_deps"] + needs: ["go_deps", "go_tools_deps"] allow_failure: true # This job should not impact the overall status of the pipeline variables: ARCH: arm64 @@ -29,6 +29,7 @@ run_codeql_scan: CODEQL_BUNDLE_URL: https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.20.4/codeql-bundle-linux64.tar.gz script: - !reference [.retrieve_linux_go_deps] + - !reference [.retrieve_linux_go_tools_deps] - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - git clone https://github.com/DataDog/codescanning.git --depth 1 --single-branch --branch=main /tmp/codescanning - curl -L $CODEQL_BUNDLE_URL -o /tmp/codeql.tar.gz && tar -zxf /tmp/codeql.tar.gz -C /usr/local && rm /tmp/codeql.tar.gz @@ -38,8 +39,7 @@ run_codeql_scan: - export CGO_CFLAGS="-I${PWD}/rtloader/include -I${PWD}/rtloader/common" - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" - echo "CGO_CFLAGS=${CGO_CFLAGS}" - - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest - - GOPROXY=https://proxy.golang.org,https://goproxy.io,direct invoke install-tools + - invoke install-tools - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="inv -e agent.build --build-exclude=systemd" - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" - $CODEQL database analyze "$CODEQL_DB"/javascript codeql/javascript-queries $SCAN_CONFIGS --sarif-category="javascript" --output="/tmp/javascript.sarif" From 66924541a4ceaf5a9da9cdd391f92f2998c38e6a Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 12 Feb 2025 12:40:38 -0500 Subject: [PATCH 26/28] remove quiet flag to show more logs --- .gitlab/source_test/codeql_scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index 017ba13ba2234e..e0e10e0d46c4bf 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -23,8 +23,8 @@ run_codeql_scan: CODEQL_DB: /tmp/datadog-agent.codeql PYTHON_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/python/codeql-suites/custom.qls GO_CUSTOM_QLPACK: /tmp/codescanning/qlpacks/golang/codeql-suites/dd-source.qls - DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript,cpp --quiet - SCAN_CONFIGS: --format sarifv2.1.0 --threads 8 --ram 96000 --no-tuple-counting --quiet + DB_CONFIGS: --threads 8 --ram 96000 --db-cluster --language=go,python,javascript,cpp + SCAN_CONFIGS: --format sarifv2.1.0 --threads 8 --ram 96000 --no-tuple-counting UPLOAD_CONFIGS: -upload_sarif=true CODEQL_BUNDLE_URL: https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.20.4/codeql-bundle-linux64.tar.gz script: From 84407b990c9625fb1e1d60cbf7057759c103c4b8 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 12 Feb 2025 14:41:49 -0500 Subject: [PATCH 27/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index e0e10e0d46c4bf..20cfea9be4d32f 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -40,6 +40,9 @@ run_codeql_scan: - echo "CGO_LDFLAGS=${CGO_LDFLAGS}" - echo "CGO_CFLAGS=${CGO_CFLAGS}" - invoke install-tools + - rm -rf /tmp/codeql.* + - df -h /tmp + - ls -ld /tmp - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="inv -e agent.build --build-exclude=systemd" - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" - $CODEQL database analyze "$CODEQL_DB"/javascript codeql/javascript-queries $SCAN_CONFIGS --sarif-category="javascript" --output="/tmp/javascript.sarif" From 97a7c64f8d0fb9cfee09b9beb1071d37d5aa2a34 Mon Sep 17 00:00:00 2001 From: ganeshkumarsv <53483484+ganeshkumarsv@users.noreply.github.com> Date: Wed, 12 Feb 2025 20:30:52 -0500 Subject: [PATCH 28/28] Update codeql_scan.yml --- .gitlab/source_test/codeql_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/source_test/codeql_scan.yml b/.gitlab/source_test/codeql_scan.yml index 20cfea9be4d32f..529d861452d9b2 100644 --- a/.gitlab/source_test/codeql_scan.yml +++ b/.gitlab/source_test/codeql_scan.yml @@ -44,8 +44,8 @@ run_codeql_scan: - df -h /tmp - ls -ld /tmp - $CODEQL database create "$CODEQL_DB" $DB_CONFIGS --command="inv -e agent.build --build-exclude=systemd" - - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" - $CODEQL database analyze "$CODEQL_DB"/javascript codeql/javascript-queries $SCAN_CONFIGS --sarif-category="javascript" --output="/tmp/javascript.sarif" + - $CODEQL database analyze "$CODEQL_DB"/go "$GO_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="go" --output="/tmp/go.sarif" - $CODEQL database analyze "$CODEQL_DB"/python "$PYTHON_CUSTOM_QLPACK" $SCAN_CONFIGS --sarif-category="python" --output="/tmp/python.sarif" - $CODEQL database analyze "$CODEQL_DB"/cpp codeql/cpp-queries $SCAN_CONFIGS --sarif-category="cpp" --output="/tmp/cpp.sarif" - mv /tmp/go.sarif .