usvm-python

.github/workflows/build-and-run-tests.yml

@@ -1,36 +1,24 @@
-# This is a basic workflow to help you get started with Actions
-
 name: Build and Run Tests [gradle]
 
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
   build:
     strategy:
       matrix:
-#        java: [ '11', '17' ]
         java: [ '11' ]
-
-    runs-on: [self-hosted, usvm] 
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
+    runs-on: ubuntu-20.04
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          submodules: "true"
 
-      # Runs a single command using the runners shell
       - name: Setup Java JDK
         uses: actions/[email protected]
         with:
@@ -41,22 +29,23 @@ jobs:
       - name: Setup Gradle
         uses: gradle/gradle-build-action@v2
 
-      # Runs a set of commands using the runners shell
-      - name: Build and run tests
-        run: ./gradlew build --no-daemon
+      - name: Install CPython optional dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -yq\
+            libssl-dev \
+            libffi-dev
 
-      # - name: Upload usvm test reports
-      #   uses: actions/upload-artifact@v3
-      #   if: always()
-      #   with:
-      #     name: usvm-tests-report-linux
-      #     path: ./**/build/reports/tests/test/
+      - name: Build and run tests
+        run: |
+          ./gradlew build --no-daemon -PcpythonActivated=true
 
       - name: Run Detekt
-        run: ./gradlew detektMain detektTest --no-daemon
+        run: |
+          ./gradlew detektMain detektTest --no-daemon
 
       - name: Upload SARIF to GitHub
         uses: github/codeql-action/upload-sarif@v3
         if: success() || failure()
         with:
-          sarif_file: build/reports/detekt/detekt.sarif
+          sarif_file: build/reports/detekt/detekt.sarif

.github/workflows/gradle-publish.yml

@@ -10,12 +10,10 @@ on:
 
 jobs:
   build:
-
-    runs-on: [self-hosted, usvm]
+    runs-on: ubuntu-20.04
     permissions:
       contents: read
       packages: write
-
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 17

.github/workflows/python-runner-publish.yml

@@ -0,0 +1,35 @@
+name: Publish Package `usvm-python-runner`
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Release version
+        type: string
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'corretto'
+          server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
+          settings-path: ${{ github.workspace }} # location for the settings.xml file
+
+      # The USERNAME and TOKEN need to correspond to the credentials environment variables used in
+      # the publishing section of your build.gradle
+      - name: Publish usvm-python-runner to GitHub Packages
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: :usvm-python:usvm-python-runner:publishAllPublicationsToGitHubPackagesRepository :usvm-python:usvm-python-common:publishAllPublicationsToGitHubPackagesRepository -Pversion=${{ inputs.version }}
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -8,3 +8,10 @@ buildSrc/.gradle
 
 # Ignore Idea directory
 .idea
+
+# Ignore vim cache
+*.swp
+
+# Ignore Python execution cache
+__pycache__/
+run_python_with_gdb.sh

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "usvm-python/cpythonadapter/cpython"]
+	path = usvm-python/cpythonadapter/cpython
+	url = https://github.com/tochilinak/cpython.git
+	branch = wrapper-v2

README.md

@@ -19,7 +19,7 @@ With USVM, you can achieve completely automated
 * [unit test cases generation](#usvm-for-unit-test-generation),
 * [targeted fuzzing and more symbolic execution based solutions](#using-usvm-to-confirm-sarif-reports).
 
-Right now, we have ready-to-be-used implementation for [Java](https://github.com/UnitTestBot/usvm/tree/main/usvm-jvm) and experimental implementation for [Python](https://github.com/UnitTestBot/usvm/tree/tochilinak/python/usvm-python).
+Right now, we have ready-to-be-used implementation for [Java](https://github.com/UnitTestBot/usvm/tree/main/usvm-jvm) and experimental implementation for [Python](https://github.com/UnitTestBot/usvm/tree/main/usvm-python).
 
 # Taint analysis with USVM
 

buildSrc/build.gradle.kts

@@ -4,14 +4,17 @@ plugins {
 
 val kotlinVersion = "1.9.20"
 val detektVersion = "1.23.5"
+val gjavahVersion = "0.3.1"
 
 repositories {
     mavenCentral()
     gradlePluginPortal()
+    maven("https://jitpack.io")
 }
 
 
 dependencies {
     implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
     implementation("io.gitlab.arturbosch.detekt:detekt-gradle-plugin:$detektVersion")
+    implementation("org.glavo:gjavah:$gjavahVersion")
 }

buildSrc/src/main/kotlin/Versions.kt

@@ -22,4 +22,7 @@ object Versions {
     const val ini4j = "0.5.4"
 
     const val sarif4k = "0.5.0"
+
+    const val pythonTypesAPIHash = "139b81d"
+    const val utbotMypyRunner = "0.2.17"
 }

buildSrc/src/main/kotlin/usvmpython/Names.kt

@@ -0,0 +1,35 @@
+package usvmpython
+
+/** Module names */
+const val USVM_PYTHON_MODULE = "usvm-python"
+const val CPYTHON_ADAPTER_MODULE = "usvm-python:cpythonadapter"
+const val USVM_PYTHON_ANNOTATIONS_MODULE = "usvm-python:usvm-python-annotations"
+const val USVM_PYTHON_MAIN_MODULE = "usvm-python:usvm-python-main"
+const val USVM_PYTHON_RUNNER_MODULE = "usvm-python:usvm-python-runner"
+const val USVM_PYTHON_COMMONS_MODULE = "usvm-python:usvm-python-commons"
+
+/** Task group names */
+const val CPYTHON_GROUP_NAME = "cpython"
+const val SAMPLE_GROUP_NAME = "samples"
+const val MANUAL_RUN_GROUP_NAME = "run"
+
+/** Task names */
+const val CPYTHON_BUILD_DEBUG_CONFIGURATION = "CPythonBuildConfigurationDebug"
+const val CPYTHON_BUILD_DEBUG = "CPythonBuildDebug"
+const val INSTALL_MYPY_RUNNER_TASK = "installUtbotMypyRunner"
+const val BUILD_SAMPLES_TASK = "buildSamples"
+const val MANUAL_TEST_FOR_RUNNER = "manualTestOfRunner"
+const val MANUAL_TEST_DEBUG_TASK = "manualTestDebug"
+const val MANUAL_TEST_DEBUG_NO_LOGS_TASK = "manualTestDebugNoLogs"
+
+/** Property names */
+const val PROPERTY_FOR_CPYTHON_ACTIVATION = "cpythonActivated"
+const val PROPERTY_FOR_CPYTHON_SSL_PATH = "cpython.ssl.path"
+
+/** Entry points */
+const val BUILD_SAMPLES_ENTRY_POINT = "org.usvm.runner.BuildSamplesKt"
+const val MANUAL_TEST_FOR_RUNNER_ENTRY = "org.usvm.runner.ManualTestKt"
+const val RUNNER_ENTRY_POINT = "org.usvm.runner.UtBotPythonRunnerEntryPointKt"
+const val MANUAL_TEST_ENTRY = "org.usvm.runner.manual.ManualTestKt"
+
+const val CPYTHON_ADAPTER_CLASS = "org.usvm.interpreter.CPythonAdapter"

buildSrc/src/main/kotlin/usvmpython/Utils.kt

@@ -0,0 +1,75 @@
+package usvmpython
+
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.gradle.api.Project
+import java.io.File
+
+fun Project.cpythonIsActivated(): Boolean {
+    if (!project.hasProperty(PROPERTY_FOR_CPYTHON_ACTIVATION))
+        return false
+    val prop = project.property(PROPERTY_FOR_CPYTHON_ACTIVATION) as? String
+    return prop?.lowercase() == "true"
+}
+
+fun Project.getCPythonModule() =
+    rootProject.findProject(CPYTHON_ADAPTER_MODULE)!!
+
+fun Project.getUsvmPythonModule() =
+    rootProject.findProject(USVM_PYTHON_MODULE)!!
+
+fun Project.getCPythonBuildPath(): File {
+    val cpythonModule = getCPythonModule()
+    return cpythonModule.layout.buildDirectory.file("cpython_build").get().asFile
+}
+
+fun Project.getCPythonSourcePath(): File {
+    val cpythonModule = getCPythonModule()
+    return File(cpythonModule.layout.projectDirectory.asFile, "cpython")
+}
+
+fun Project.getCPythonAdapterBuildPath(): File {
+    val cpythonModule = getCPythonModule()
+    return cpythonModule.layout.buildDirectory.file("lib/main/debug").get().asFile
+}
+
+val isWindows = Os.isFamily(Os.FAMILY_WINDOWS)
+val isMacos = Os.isFamily(Os.FAMILY_MAC)
+
+fun Project.getWidowsBuildScriptPath(): File =
+    File(getCPythonSourcePath(), "PCBuild/build.bat")
+
+fun Project.getIncludePipFile(): File {
+    val cpythonModule = getCPythonModule()
+    return File(cpythonModule.projectDir, "include_pip_in_build")
+}
+
+fun Project.includePipInBuildLine(): String {
+    val includePipFile = getIncludePipFile()
+    val includePip = includePipFile.readText().trim() != "false"
+    return if (includePip) {
+        "--with-ensurepip=yes"
+    } else {
+        "--with-ensurepip=no"
+    }
+}
+
+fun Project.getGeneratedHeadersPath(): File =
+    getCPythonModule().layout.buildDirectory.file("adapter_include").get().asFile
+
+fun Project.getSamplesSourceDir(): File =
+    File(getUsvmPythonModule().projectDir, "src/test/resources/samples")
+
+fun Project.getSamplesBuildDir(): File =
+    getUsvmPythonModule().layout.buildDirectory.file("samples_build").get().asFile
+
+fun Project.getApproximationsDir(): File =
+    File(getUsvmPythonModule().projectDir, "python_approximations")
+
+fun Project.getPythonBinaryPath(): File {
+    val cpythonBuildPath = getCPythonBuildPath()
+    return if (isWindows) {
+        File(cpythonBuildPath, "python_d.exe")
+    } else {
+        File(cpythonBuildPath, "bin/python3")
+    }
+}