Aknezevic/ci on one machine

.github/Dockerfile.ci

@@ -15,45 +15,17 @@ RUN echo "Building $PROJECT_NAME at $GIT_SHA"
 RUN mkdir -p $BUILD_DIR && \
     mkdir -p $TTMLIR_TOOLCHAIN_DIR
 
-# Clone tt-mlir
-RUN git clone [email protected]:tenstorrent/tt-mlir.git $BUILD_DIR/$PROJECT_NAME && \
-    cd $BUILD_DIR/$PROJECT_NAME && \
-    git checkout $GIT_SHA
+# Copy the project from host, cloned in build-image.yml
+COPY . $BUILD_DIR/$PROJECT_NAME
 
 # Build the toolchain
 WORKDIR $BUILD_DIR/$PROJECT_NAME
-RUN source env/activate && \
+RUN cmake -B toolchain -DTOOLCHAIN=ON third_party/ && \
+    cd third_party/tt-mlir/src/tt-mlir/ && \
+    source env/activate && \
     cmake -B env/build env && \
     cmake --build env/build
 
-# Self-test
-
-# Build project to test the container
-RUN source env/activate && \
-    cmake -G Ninja \
-    -B build \
-    -DCMAKE_BUILD_TYPE=Release \
-    -DCMAKE_C_COMPILER=clang \
-    -DCMAKE_CXX_COMPILER=clang++ \
-    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-    -DTTMLIR_ENABLE_RUNTIME=ON \
-    -DTTMLIR_ENABLE_RUNTIME_TESTS=ON \
-    -DTTMLIR_ENABLE_STABLEHLO=ON
-
-RUN source env/activate && \
-    cmake --build build --config Release
-
-RUN source env/activate && \
-    cmake --build build -- ttrt
-
-# Run clang-tidy
-RUN source env/activate && \
-    cmake --build build -- clang-tidy
-
-# Run the tests
-RUN source env/activate && \
-    cmake --build build -- check-ttmlir
-
 # Final stage
 FROM ghcr.io/tenstorrent/tt-xla/tt-xla-base-ubuntu-22-04:${FROM_TAG} AS ci
 

.github/workflows/build-and-test.yml

@@ -0,0 +1,148 @@
+name: Build and Test
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  # build-ttxla:
+
+  #   timeout-minutes: 120
+  #   strategy:
+  #     fail-fast: false
+
+  #   name: Build and test tt-xla
+  #   runs-on: ubuntu-latest
+
+  #   container:
+  #     image: ghcr.io/tenstorrent/tt-mlir/tt-mlir-ci-ubuntu-22-04:latest #TODO update this to the correct image
+  #     options: --user root
+
+  #   steps:
+  #   - uses: actions/checkout@v4
+  #     with:
+  #       submodules: recursive
+  #       lfs: true
+
+  #   - name: Set reusable strings
+  #     id: strings
+  #     shell: bash
+  #     run: |
+  #       echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+  #       echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+  #       echo "metal-lib-dir=$(pwd)/third_party/tt-mlir/src/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib" >> "$GITHUB_OUTPUT"
+  #       echo "install-output-dir=$(pwd)/install" >> "$GITHUB_OUTPUT"
+
+  #   - name: Git safe dir
+  #     run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
+
+
+  #   # Build project
+
+  #   - name: Configure CMake
+  #     shell: bash
+  #     run: |
+  #       source venv/activate
+  #       cmake -G Ninja \
+  #       -B ${{ steps.strings.outputs.build-output-dir }} \
+  #       -S ${{ steps.strings.outputs.work-dir }}
+
+  #   - name: Build
+  #     shell: bash
+  #     run: |
+  #       source venv/activate
+  #       cmake --build ${{ steps.strings.outputs.build-output-dir }}
+  #       cmake --install ${{ steps.strings.outputs.build-output-dir }}
+
+
+  #   # This is needed to preserve file permissions
+  #   # https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
+  #   - name: 'Tar install directory and metal lib directory'
+  #     shell: bash
+  #     working-directory: ${{ steps.strings.outputs.install-output-dir }}
+  #     run: | 
+  #       tar cvf artifact.tar .
+
+  #   - name: Upload install folder to archive
+  #     uses: actions/upload-artifact@v4
+  #     with:
+  #       name: install-artifacts
+  #       path: ${{ steps.strings.outputs.build-output-dir }}/artifact.tar
+
+  #   - name: Get the latest tag
+  #     shell: bash
+  #     run: |
+  #       latest_tag=$(git describe --tags --abbrev=0)
+  #       latest_tag=${latest_tag#v}
+  #       echo "latest_tag=$latest_tag" >> $GITHUB_ENV
+  #       commit_count=$(git rev-list ${{ env.latest_tag }}..HEAD --count)
+  #       echo "commit_count=$commit_count" >> $GITHUB_ENV
+  #       version="${{ env.latest_tag }}.${{ env.commit_count }}"
+  #       echo "version=$version" >> $GITHUB_ENV
+  #       echo $version
+
+
+  # # Run tests on TT hardware
+
+  build-and-run-tests:
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        build: [
+          {runs-on: n150, name: "run"},
+          # {runs-on: n300, name: "run"}, #TODO: https://github.com/tenstorrent/tt-xla/issues/9
+        ]
+
+    runs-on:
+      - in-service
+      - ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/tenstorrent/tt-mlir/tt-mlir-ci-ubuntu-22-04:latest
+      options: --user root --device /dev/tenstorrent/0
+      volumes:
+        - /dev/hugepages:/dev/hugepages
+        - /dev/hugepages-1G:/dev/hugepages-1G
+        - /etc/udev/rules.d:/etc/udev/rules.d
+        - /lib/modules:/lib/modules
+        - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        lfs: true
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+        echo "install-output-dir=$(pwd)/install" >> "$GITHUB_OUTPUT"
+
+    # Build project
+    - name: Configure CMake
+      shell: bash
+      run: |
+        source venv/activate
+        cmake -G Ninja \
+        -B ${{ steps.strings.outputs.build-output-dir }} \
+        -S ${{ steps.strings.outputs.work-dir }}
+
+    - name: Build
+      shell: bash
+      run: |
+        source venv/activate
+        cmake --build ${{ steps.strings.outputs.build-output-dir }}
+        cmake --install ${{ steps.strings.outputs.build-output-dir }}
+
+    - name: Run tests
+      shell: bash
+      run: |
+        export LD_LIBRARY_PATH="/opt/ttmlir-toolchain/lib/:${{ steps.strings.outputs.install-output-dir }}/lib:${LD_LIBRARY_PATH}"
+        source venv/activate
+        pytest -v tests/

.github/workflows/build-image.yml

@@ -21,6 +21,9 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          lfs: true
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -37,7 +40,6 @@ jobs:
       - name: Build and export base Docker image
         uses: docker/build-push-action@v6
         with:
-          context: .github
           file: .github/Dockerfile.base
           push: true
           build-args: |
@@ -48,32 +50,18 @@ jobs:
       - name: Build and export CI Docker image
         uses: docker/build-push-action@v6
         with:
-          context: .github
           file: .github/Dockerfile.ci
           push: true
           build-args: |
             GIT_SHA=${{ github.sha }}
           tags: |
             ${{ env.CI_IMAGE_NAME}}:${{ github.sha }}
 
-      - name: Build and export IRD Docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: .github
-          file: .github/Dockerfile.ird
-          push: true
-          build-args: |
-            GIT_SHA=${{ github.sha }}
-            FROM_IMAGE=ci
-          tags: |
-            ${{ env.IRD_IMAGE_NAME}}:${{ github.sha }}
-
       # Tag images as latest
 
       - name: Build and push base Docker image
         uses: docker/build-push-action@v6
         with:
-          context: .github
           file: .github/Dockerfile.base
           push: true
           build-args: |
@@ -84,7 +72,6 @@ jobs:
       - name: Build and push CI Docker image
         uses: docker/build-push-action@v6
         with:
-          context: .github
           file: .github/Dockerfile.ci
           push: true
           build-args: |

CMakeLists.txt

@@ -12,6 +12,8 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") 
 
+set(CMAKE_INSTALL_PREFIX ${TTPJRT_SOURCE_DIR}/install)
+
 # set(ASAN_FLAGS "-fsanitize=address -fno-omit-frame-pointer")
 # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ASAN_FLAGS}")
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")

src/common/CMakeLists.txt

@@ -91,7 +91,6 @@ target_link_libraries(TTPJRTCommon PUBLIC
     MLIR
     TTMLIR
     TTPJRTCommonDylibPlatform
-    TTMLIRStatic
     TTMLIRStableHLOToTTIR
     ${STABLEHLO_LIBS}
 )

src/tt/CMakeLists.txt

@@ -27,12 +27,12 @@ target_link_libraries(TTPJRTTTDylib PUBLIC TTPJRTTT)
 
 # # Output to the project wide python binary directory tree.
 set_target_properties(TTPJRTTTDylib
-  PROPERTIES
+    PROPERTIES
     PREFIX ""  # Disable "lib" prefix.
     LIBRARY_OUTPUT_NAME pjrt_plugin_tt
-    RUNTIME_OUTPUT_DIRECTORY "${_NATIVE_PYTHON_DIR}"
-    LIBRARY_OUTPUT_DIRECTORY "${_NATIVE_PYTHON_DIR}"
+    INSTALL_RPATH "$ORIGIN"
 )
+install(TARGETS TTPJRTTTDylib DESTINATION lib)
 
 target_link_options(TTPJRTTTDylib PRIVATE "-Wl,--no-undefined")
 

tests/TTIR/test_mnist.py

@@ -30,7 +30,7 @@ def module_relu(a):
 
   verify_module(module_relu, [(32, 32)])
 
-# @pytest.mark.xfail
+@pytest.mark.xfail
 def test_softmax():
   def module_softmax(a):
     return jax.nn.softmax(a)