#12349: Skipping failing tests on BH

tenstorrent · Sep 19, 2024 · 4d8b28b · 4d8b28b
1 parent 3404377
commit 4d8b28b
Show file tree

Hide file tree

Showing 62 changed files with 186 additions and 55 deletions.
diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/helper_funcs/test_linear.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/helper_funcs/test_linear.py
@@ -10,6 +10,7 @@
 
 from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test
+from models.utility_functions import is_blackhole
 
 
 @pytest.mark.parametrize(
@@ -50,6 +51,9 @@ def test_linear_no_bias(input_shapes, device):
     ),
 )
 def test_linear_with_bias(input_shapes, device):
+    if is_blackhole() and input_shapes[0] == [1, 1, 64, 128]:
+        pytest.skip("Failing case for BH, see #12349")
+
     comparison_func = partial(comparison_funcs.comp_pcc)
 
     datagen_func = [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_not.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_not.py
@@ -15,14 +15,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Mismatch on BH, see #12349")
 @pytest.mark.parametrize(
     "scalar",
     (1, 1),

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_or.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_or.py
@@ -16,14 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_xor.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_xor.py
@@ -16,14 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "scalar",
     {random.randint(-100, 100) for _ in range(10)},

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div.py
@@ -16,14 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import is_grayskull
+from models.utility_functions import is_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Only supported for WH, see #12349")
 @pytest.mark.parametrize("accurate_mode", [False, True])
 @pytest.mark.parametrize("round_mode", ["None", "trunc", "floor"])
 @pytest.mark.parametrize(

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div_unary.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div_unary.py
@@ -16,14 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import is_grayskull
+from models.utility_functions import is_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Only supported on WH, see #12349")
 @pytest.mark.parametrize("accurate_mode", [True])
 @pytest.mark.parametrize("round_mode", ["None", "trunc", "floor"])
 @pytest.mark.parametrize(

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_floor_div.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_floor_div.py
@@ -16,14 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Only supported for WH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_fmod.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_fmod.py
@@ -16,14 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Only supported on WH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_frac.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_frac.py
@@ -7,7 +7,7 @@
 import random
 from functools import partial
 import ttnn
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 from tests.tt_eager.python_api_testing.sweep_tests import (
     comparison_funcs,
@@ -23,6 +23,7 @@
 ]
 
 
+@skip_for_blackhole("Unsupported on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_matmul.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_matmul.py
@@ -15,7 +15,7 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import is_wormhole_b0
+from models.utility_functions import is_wormhole_b0, skip_for_blackhole
 
 shapes_mm = [
     # Single core (won't be hit after padding is added for multicast)
@@ -47,6 +47,7 @@
     del shapes_mm[1:]
 
 
+@skip_for_blackhole("Hanging configs on BH, see #12349")
 @pytest.mark.parametrize("input_shapes", shapes_mm)
 @pytest.mark.parametrize(
     "dtype",

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_pad.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_pad.py
@@ -9,7 +9,7 @@
 
 from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test
-from models.utility_functions import is_grayskull
+from models.utility_functions import is_grayskull, skip_for_blackhole
 import random
 
 # Seed here for fixed params
@@ -22,6 +22,7 @@
     params += [pytest.param([[5, 5, 64, 96]], pad_args) for pad_args in generation_funcs.gen_pad_args([[5, 5, 64, 96]])]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize("input_shapes, pad_args", params)
 def test_run_pad_test(input_shapes, pad_args, device, function_level_defaults):
     datagen_func = [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_permute.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_permute.py
@@ -9,6 +9,7 @@
 
 from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test
+from models.utility_functions import skip_for_blackhole
 
 params = [
     pytest.param([[1, 1, 32, 32]], permute_args) for permute_args in generation_funcs.gen_permute_args([[1, 1, 32, 32]])
@@ -19,6 +20,7 @@
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize("input_shapes, permute_args", params)
 def test_run_permute_test(input_shapes, permute_args, device, function_level_defaults):
     datagen_func = [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_prod.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_prod.py
@@ -16,13 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
+from models.utility_functions import skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "dim",
     (3, 2, 1, 0, -1, -2, -3, -4),

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_remainder.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_remainder.py
@@ -16,14 +16,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Unsupported on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_repeat_interleave.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_repeat_interleave.py
@@ -15,7 +15,7 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import is_wormhole_b0
+from models.utility_functions import is_wormhole_b0, skip_for_blackhole
 
 shapes = (
     [[1, 1, 32, 32]],  # Single core
@@ -25,6 +25,7 @@
 )
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize("input_shapes", shapes)
 @pytest.mark.parametrize("dim", [0, 2, -4, -2, 1, 3])
 @pytest.mark.parametrize("repeat", [2, 3, 4])

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_right_shift.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_right_shift.py
@@ -15,7 +15,7 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
@@ -40,6 +40,7 @@
     mem_configs,
 )
 @skip_for_grayskull("#TODO: GS implementation needs to be done")
+@skip_for_blackhole("Mismatching on BH, see #12349")
 class TestRightShift:
     def test_run_right_shift_op(
         self,

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_round.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_round.py
@@ -7,7 +7,7 @@
 import random
 from functools import partial
 import ttnn
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 from tests.tt_eager.python_api_testing.sweep_tests import (
     comparison_funcs,
@@ -23,6 +23,7 @@
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "decimals",
     [0],

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_stats.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_stats.py
@@ -9,7 +9,7 @@
 
 from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test
-from models.utility_functions import is_wormhole_b0
+from models.utility_functions import is_wormhole_b0, skip_for_blackhole
 import ttnn
 import numpy as np
 
@@ -57,6 +57,7 @@
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes_and_pcc",
     shapes,

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_sum.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_sum.py
@@ -9,8 +9,10 @@
 
 from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test
+from models.utility_functions import skip_for_blackhole
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     [

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_transpose.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_transpose.py
@@ -9,13 +9,15 @@
 
 from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 shape_wh = [
     [[1, 1, 32, 32]],  # Single core
     [[3, 1, 320, 384]],  # Multi core
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize("input_shapes", shape_wh)
 def test_run_transpose_wh_test(input_shapes, device, function_level_defaults):
     datagen_func = [
@@ -32,6 +34,7 @@ def test_run_transpose_wh_test(input_shapes, device, function_level_defaults):
     run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args)
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     (
@@ -60,6 +63,7 @@ def test_run_transpose_hc_test(input_shapes, device, function_level_defaults):
 ]
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize("input_shapes", shape_cn)
 def test_run_transpose_cn_test(input_shapes, device, function_level_defaults):
     datagen_func = [
@@ -76,6 +80,7 @@ def test_run_transpose_cn_test(input_shapes, device, function_level_defaults):
     run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args)
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     (
@@ -98,6 +103,7 @@ def test_run_transpose_nh_test(input_shapes, device, function_level_defaults):
     run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args)
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     (
@@ -120,6 +126,7 @@ def test_run_transpose_nw_test(input_shapes, device, function_level_defaults):
     run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args)
 
 
+@skip_for_blackhole("Mismatching on BH, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     (

diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_unary_floor_div.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_unary_floor_div.py
@@ -12,14 +12,15 @@
 from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import (
     run_single_pytorch_test,
 )
-from models.utility_functions import skip_for_grayskull
+from models.utility_functions import skip_for_grayskull, skip_for_blackhole
 
 mem_configs = [
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),
     ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.L1),
 ]
 
 
+@skip_for_blackhole("Unsupported on non WH arch, see #12349")
 @pytest.mark.parametrize(
     "input_shapes",
     [