diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/helper_funcs/test_linear.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/helper_funcs/test_linear.py index 5f338a2ee6c..3ac938336bb 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/helper_funcs/test_linear.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/helper_funcs/test_linear.py @@ -10,6 +10,7 @@ from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test +from models.utility_functions import is_blackhole @pytest.mark.parametrize( @@ -50,6 +51,9 @@ def test_linear_no_bias(input_shapes, device): ), ) def test_linear_with_bias(input_shapes, device): + if is_blackhole() and input_shapes[0] == [1, 1, 64, 128]: + pytest.skip("Failing case for BH, see #12349") + comparison_func = partial(comparison_funcs.comp_pcc) datagen_func = [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_not.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_not.py index 5b46869f3e8..b417a89e256 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_not.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_not.py @@ -15,7 +15,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -23,6 +23,7 @@ ] +@skip_for_blackhole("Mismatch on BH, see #12349") @pytest.mark.parametrize( "scalar", (1, 1), diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_or.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_or.py index 753385fcaba..da4c1bbeca6 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_or.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_or.py @@ -16,7 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -24,6 +24,7 @@ ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_xor.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_xor.py index ff9b4d1a8d0..9328215f7bc 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_xor.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_bitwise_xor.py @@ -16,7 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -24,6 +24,7 @@ ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "scalar", {random.randint(-100, 100) for _ in range(10)}, diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div.py index 0fc89897817..60acfe5f59b 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div.py @@ -16,7 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -24,6 +24,7 @@ ] +@skip_for_blackhole("Only supported for WH, see #12349") @pytest.mark.parametrize("accurate_mode", [False, True]) @pytest.mark.parametrize("round_mode", ["None", "trunc", "floor"]) @pytest.mark.parametrize( diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div_unary.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div_unary.py index 3d85a290137..0a85d67f471 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div_unary.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_div_unary.py @@ -16,7 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -24,6 +24,7 @@ ] +@skip_for_blackhole("Only supported on WH, see #12349") @pytest.mark.parametrize("accurate_mode", [True]) @pytest.mark.parametrize("round_mode", ["None", "trunc", "floor"]) @pytest.mark.parametrize( diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_floor_div.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_floor_div.py index 11d66737da4..cfefd41cc8d 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_floor_div.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_floor_div.py @@ -16,7 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -24,6 +24,7 @@ ] +@skip_for_blackhole("Only supported for WH, see #12349") @pytest.mark.parametrize( "input_shapes", [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_fmod.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_fmod.py index 65b45a5ba6b..28cd0590730 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_fmod.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_fmod.py @@ -16,7 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -24,6 +24,7 @@ ] +@skip_for_blackhole("Only supported on WH, see #12349") @pytest.mark.parametrize( "input_shapes", [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_frac.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_frac.py index 9c6907d1fac..25e92cbf3fd 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_frac.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_frac.py @@ -7,7 +7,7 @@ import random from functools import partial import ttnn -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole from tests.tt_eager.python_api_testing.sweep_tests import ( comparison_funcs, @@ -23,6 +23,7 @@ ] +@skip_for_blackhole("Unsupported on BH, see #12349") @pytest.mark.parametrize( "input_shapes", [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_matmul.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_matmul.py index 797f2e99ff8..bb41e5b6ad0 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_matmul.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_matmul.py @@ -15,7 +15,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import is_wormhole_b0 +from models.utility_functions import is_wormhole_b0, skip_for_blackhole shapes_mm = [ # Single core (won't be hit after padding is added for multicast) @@ -47,6 +47,7 @@ del shapes_mm[1:] +@skip_for_blackhole("Hanging configs on BH, see #12349") @pytest.mark.parametrize("input_shapes", shapes_mm) @pytest.mark.parametrize( "dtype", diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_pad.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_pad.py index fe0e4d4bd22..e0104e6e3a6 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_pad.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_pad.py @@ -9,7 +9,7 @@ from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole import random # Seed here for fixed params @@ -22,6 +22,7 @@ params += [pytest.param([[5, 5, 64, 96]], pad_args) for pad_args in generation_funcs.gen_pad_args([[5, 5, 64, 96]])] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("input_shapes, pad_args", params) def test_run_pad_test(input_shapes, pad_args, device, function_level_defaults): datagen_func = [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_permute.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_permute.py index 68097654f3e..d9ab2571a58 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_permute.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_permute.py @@ -9,6 +9,7 @@ from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test +from models.utility_functions import skip_for_blackhole params = [ pytest.param([[1, 1, 32, 32]], permute_args) for permute_args in generation_funcs.gen_permute_args([[1, 1, 32, 32]]) @@ -19,6 +20,7 @@ ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("input_shapes, permute_args", params) def test_run_permute_test(input_shapes, permute_args, device, function_level_defaults): datagen_func = [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_prod.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_prod.py index bed946556c4..2012f9d944f 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_prod.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_prod.py @@ -16,6 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) +from models.utility_functions import skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -23,6 +24,7 @@ ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "dim", (3, 2, 1, 0, -1, -2, -3, -4), diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_remainder.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_remainder.py index 17f5fc71e90..71137290b77 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_remainder.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_remainder.py @@ -16,7 +16,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -24,6 +24,7 @@ ] +@skip_for_blackhole("Unsupported on BH, see #12349") @pytest.mark.parametrize( "input_shapes", [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_repeat_interleave.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_repeat_interleave.py index 3e945a1f893..1a297d86aa1 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_repeat_interleave.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_repeat_interleave.py @@ -15,7 +15,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import is_wormhole_b0 +from models.utility_functions import is_wormhole_b0, skip_for_blackhole shapes = ( [[1, 1, 32, 32]], # Single core @@ -25,6 +25,7 @@ ) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("input_shapes", shapes) @pytest.mark.parametrize("dim", [0, 2, -4, -2, 1, 3]) @pytest.mark.parametrize("repeat", [2, 3, 4]) diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_right_shift.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_right_shift.py index 6523b48f5ed..7aacc9735b1 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_right_shift.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_right_shift.py @@ -15,7 +15,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -40,6 +40,7 @@ mem_configs, ) @skip_for_grayskull("#TODO: GS implementation needs to be done") +@skip_for_blackhole("Mismatching on BH, see #12349") class TestRightShift: def test_run_right_shift_op( self, diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_round.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_round.py index 8bbc11894db..eb37b67da0b 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_round.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_round.py @@ -7,7 +7,7 @@ import random from functools import partial import ttnn -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole from tests.tt_eager.python_api_testing.sweep_tests import ( comparison_funcs, @@ -23,6 +23,7 @@ ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "decimals", [0], diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_stats.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_stats.py index 80f7a87a82e..433558c5d66 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_stats.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_stats.py @@ -9,7 +9,7 @@ from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test -from models.utility_functions import is_wormhole_b0 +from models.utility_functions import is_wormhole_b0, skip_for_blackhole import ttnn import numpy as np @@ -57,6 +57,7 @@ ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes_and_pcc", shapes, diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_sum.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_sum.py index aed595712eb..238143809c6 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_sum.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_sum.py @@ -9,8 +9,10 @@ from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test +from models.utility_functions import skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_transpose.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_transpose.py index 9ff799bebfe..bd0497aaee6 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_transpose.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_transpose.py @@ -9,6 +9,7 @@ from tests.tt_eager.python_api_testing.sweep_tests import comparison_funcs, generation_funcs from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import run_single_pytorch_test +from models.utility_functions import skip_for_grayskull, skip_for_blackhole shape_wh = [ [[1, 1, 32, 32]], # Single core @@ -16,6 +17,7 @@ ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("input_shapes", shape_wh) def test_run_transpose_wh_test(input_shapes, device, function_level_defaults): datagen_func = [ @@ -32,6 +34,7 @@ def test_run_transpose_wh_test(input_shapes, device, function_level_defaults): run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ( @@ -60,6 +63,7 @@ def test_run_transpose_hc_test(input_shapes, device, function_level_defaults): ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("input_shapes", shape_cn) def test_run_transpose_cn_test(input_shapes, device, function_level_defaults): datagen_func = [ @@ -76,6 +80,7 @@ def test_run_transpose_cn_test(input_shapes, device, function_level_defaults): run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ( @@ -98,6 +103,7 @@ def test_run_transpose_nh_test(input_shapes, device, function_level_defaults): run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ( @@ -120,6 +126,7 @@ def test_run_transpose_nw_test(input_shapes, device, function_level_defaults): run_single_pytorch_test("transpose", input_shapes, datagen_func, comparison_func, device, test_args) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ( diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_unary_floor_div.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_unary_floor_div.py index d440a2d06f7..415e9dddf8d 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_unary_floor_div.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_unary_floor_div.py @@ -12,7 +12,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.run_pytorch_ci_tests import ( run_single_pytorch_test, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole mem_configs = [ ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM), @@ -20,6 +20,7 @@ ] +@skip_for_blackhole("Unsupported on non WH arch, see #12349") @pytest.mark.parametrize( "input_shapes", [ diff --git a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_untilize_with_unpadding.py b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_untilize_with_unpadding.py index 2050c52e807..e19ff82541a 100644 --- a/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_untilize_with_unpadding.py +++ b/tests/tt_eager/python_api_testing/sweep_tests/pytests/tt_dnn/test_untilize_with_unpadding.py @@ -12,6 +12,7 @@ run_single_pytorch_test, ) import ttnn +from models.utility_functions import skip_for_blackhole def create_grid(x, y): @@ -67,6 +68,7 @@ def create_grid(x, y): ] +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("input_shapes, untilize_with_unpadding_args", params) def test_run_untilize_with_unpadding_test(input_shapes, untilize_with_unpadding_args, device): datagen_func = [ diff --git a/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mae.py b/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mae.py index ebdd3218c93..8d129f48712 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mae.py +++ b/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mae.py @@ -9,8 +9,10 @@ comparison_funcs, ) from loguru import logger +from models.utility_functions import skip_for_blackhole +@skip_for_blackhole("Mismatches on Blackhole, see #12349") @pytest.mark.parametrize( "input_shapes", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mse.py b/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mse.py index 3752e983187..41609676aa3 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mse.py +++ b/tests/tt_eager/python_api_testing/unit_testing/loss_ops/test_loss_mse.py @@ -9,8 +9,10 @@ comparison_funcs, ) from loguru import logger +from models.utility_functions import skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_attn_matmul.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_attn_matmul.py index 68bdc6501ad..ffb55fe0d72 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_attn_matmul.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_attn_matmul.py @@ -8,7 +8,7 @@ import ttnn from models.utility_functions import comp_pcc -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole import ttnn @@ -30,6 +30,7 @@ def generate_input_shapes(): yield [q_len, q_heads, batch_size, K], [batch_size, kv_heads, K, seq_len] +@skip_for_blackhole("Hanging on BH, see #12349") @pytest.mark.parametrize("in0_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @pytest.mark.parametrize("in1_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @pytest.mark.parametrize("out_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @@ -70,6 +71,7 @@ def test_attn_matmul(num_loops, enable_async, in0_dtype, in1_dtype, out_dtype, d device.enable_async(False) +@skip_for_blackhole("Hanging on BH, see #12349") @pytest.mark.skipif(is_grayskull(), reason="GS does not support fp32") @pytest.mark.parametrize("in_dtype", [ttnn.float32, ttnn.bfloat16, ttnn.bfloat8_b]) @pytest.mark.parametrize( @@ -115,6 +117,7 @@ def test_attn_matmul_fp32(num_loops, enable_async, in_dtype, device): device.enable_async(False) +@skip_for_blackhole("Hanging on BH, see #12349") @pytest.mark.parametrize("in0_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @pytest.mark.parametrize("in1_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @pytest.mark.parametrize("out_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @@ -153,6 +156,7 @@ def test_attn_matmul_with_program_cache( device.enable_async(False) +@skip_for_blackhole("Hanging on BH, see #12349") @pytest.mark.parametrize( "shard_orientation", (ttnn.ShardOrientation.ROW_MAJOR, ttnn.ShardOrientation.COL_MAJOR), @@ -274,6 +278,7 @@ def test_group_attn_matmul( device.enable_async(False) +@skip_for_blackhole("Hanging on BH, see #12349") @pytest.mark.parametrize("sharded", [False, True]) @pytest.mark.parametrize("output_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @pytest.mark.parametrize("in1_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @@ -368,6 +373,7 @@ def test_group_attn_matmul_with_program_cache( device.enable_async(False) +@skip_for_blackhole("Hanging on BH, see #12349") @pytest.mark.skipif(is_grayskull(), reason="GS does not support fp32") @pytest.mark.parametrize("in_dtype", [ttnn.float32, ttnn.bfloat16]) @pytest.mark.parametrize( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_bcast.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_bcast.py index 2b5992b346e..ddb53be7dba 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_bcast.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_bcast.py @@ -9,12 +9,14 @@ from loguru import logger from tests.ttnn.utils_for_testing import check_with_pcc_without_tensor_printout, update_process_id from tests.ttnn.ttnn_utility_fuction import get_shard_grid_from_num_cores +from models.utility_functions import skip_for_blackhole import ttnn from tt_lib.utils import ( _nearest_y, ) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_height, input_width, num_cores, shard_grid, shard_strategy", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_bert_ops.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_bert_ops.py index a6f50a68c88..9db289d9ce2 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_bert_ops.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_bert_ops.py @@ -15,7 +15,7 @@ comp_equal, comp_pcc, ) -from models.utility_functions import is_wormhole_b0, is_grayskull, is_wormhole_b0, is_blackhole, is_blackhole +from models.utility_functions import is_wormhole_b0, is_grayskull, is_wormhole_b0, is_blackhole from loguru import logger from models.utility_functions import torch2tt_tensor, tt2torch_tensor, pad_by_zero @@ -531,6 +531,7 @@ def not_fit_l1(M, K, N, fp32): return (M * K + K * N > 5000000) and (fp32 == True) +@pytest.mark.skipif(is_blackhole(), reason="Hanging on Blackhole, see #12349") @pytest.mark.skipif(is_grayskull(), reason="GS does not support fp32") @pytest.mark.parametrize("packer_l1_acc", [True, False], ids=["pack_l1", "no_pack_l1"]) @pytest.mark.parametrize("fp32_acc_mode", [True, False], ids=["fp32", "no_fp32"]) @@ -604,6 +605,7 @@ def test_bert_linear_batch4( logger.warning("L1 cannot fit large tensors in fp32 mode") +@pytest.mark.skipif(is_blackhole(), reason="Hangs on BH, see #12349") @pytest.mark.skipif(is_grayskull(), reason="not tested for GS") @pytest.mark.parametrize("packer_l1_acc", [True, False], ids=["pack_l1", "no_pack_l1"]) @pytest.mark.parametrize( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_binary_eq_int.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_binary_eq_int.py index 19e0c8ef5dd..0f51a51c170 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_binary_eq_int.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_binary_eq_int.py @@ -6,9 +6,10 @@ import pytest import ttnn from tests.tt_eager.python_api_testing.unit_testing.backward_ops.utility_funcs import data_gen_with_range, compare_pcc -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ((torch.Size([1, 1, 32, 32])),), @@ -40,6 +41,7 @@ def test_binary_eq(input_shapes, out_dtype, mem_configs, device): assert comp_pass +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ((torch.Size([1, 1, 32, 32])),), diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_concat.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_concat.py index d1b3b83aac5..75398e15a68 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_concat.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_concat.py @@ -9,7 +9,7 @@ import torch import ttnn -from models.utility_functions import print_diff_argmax +from models.utility_functions import print_diff_argmax, skip_for_blackhole import pytest from loguru import logger @@ -168,6 +168,7 @@ def test_concat_with_program_cache( run_concat(shapes, dim, device, layout, dtype, input_mem_config, output_mem_config) +@skip_for_blackhole("Alignment issue on BH, see #12349") @pytest.mark.parametrize( "input_shape, shard_shape, output_shard_shape, shard_grid", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_create_qkv_heads.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_create_qkv_heads.py index 6222d086c9c..24fe1aabf1d 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_create_qkv_heads.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_create_qkv_heads.py @@ -6,7 +6,7 @@ from loguru import logger from models.utility_functions import tt2torch_tensor, comp_pcc -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole import torch import ttnn @@ -122,6 +122,7 @@ def run_create_qkv_heads_test( assert passing_pcc_v +@skip_for_blackhole("L1 and Circular buffers are crashing on BH, see #12349") @pytest.mark.parametrize("device_params", [{"l1_small_size": 8192}], indirect=True) @pytest.mark.parametrize( "dtype", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_distributed_layernorm_post_allgather.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_distributed_layernorm_post_allgather.py index ce18b5d65f2..74d9f3d140c 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_distributed_layernorm_post_allgather.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_distributed_layernorm_post_allgather.py @@ -8,7 +8,7 @@ import ttnn -from models.utility_functions import tt2torch_tensor, torch2tt_tensor, skip_for_grayskull +from models.utility_functions import tt2torch_tensor, torch2tt_tensor, skip_for_grayskull, skip_for_blackhole from loguru import logger from tests.tt_eager.python_api_testing.sweep_tests.comparison_funcs import comp_allclose, comp_pcc @@ -107,6 +107,7 @@ def run_layernorm_part_2(inp_shape, n_devices, is_rmsnorm, dtype, device, fp32_e assert all_pass +@skip_for_blackhole("Mismatching on BH, see #12349") @skip_for_grayskull("Requires wormhole") @pytest.mark.parametrize( "dtype", @@ -141,6 +142,7 @@ def test_layernorm_part_2_with_program_cache( run_layernorm_part_2(inp_shape, n_devices, is_rmsnorm, dtype, device, fp32_enabled) +@skip_for_blackhole("Mismatching on BH, see #12349") @skip_for_grayskull("Requires wormhole") @pytest.mark.parametrize( "dtype", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_downsample.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_downsample.py index 2a17bd117ec..92f84e434fe 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_downsample.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_downsample.py @@ -22,9 +22,11 @@ create_conv_bias_tensor, create_conv_weight_tensor_special_padding, ) +from models.utility_functions import skip_for_blackhole import torch +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("device_params", [{"l1_small_size": 8192}], indirect=True) @pytest.mark.parametrize( "batch_size, output_channels, input_channels, input_height, input_width, stride_h, stride_w, num_cores, grid_size, height_sharded", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm.py index 0bee701ac77..1b7a4c0d34a 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm.py @@ -10,7 +10,7 @@ import ttnn -from models.utility_functions import pad_by_zero, torch2tt_tensor, comp_pcc, is_grayskull +from models.utility_functions import pad_by_zero, torch2tt_tensor, comp_pcc, is_grayskull, is_blackhole def ref_layernorm(x, gamma, beta, eps): @@ -171,6 +171,7 @@ def run_layernorm_mix_precision_tests(test_id, in_dtype, gamma_dtype, in0_mem_co assert passing, output +@pytest.mark.skipif(is_blackhole(), reason="Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "out_mem_config", (ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),), diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm_sharded.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm_sharded.py index 35254b83902..f6c888ed3c2 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm_sharded.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_layernorm_sharded.py @@ -13,7 +13,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.comparison_funcs import ( comp_pcc, ) -from models.utility_functions import torch2tt_tensor, is_wormhole_b0, is_grayskull +from models.utility_functions import torch2tt_tensor, is_wormhole_b0, is_grayskull, skip_for_blackhole def rms_norm(x, dim, gamma, beta, eps): @@ -21,6 +21,7 @@ def rms_norm(x, dim, gamma, beta, eps): # @pytest.mark.skipif(is_wormhole_b0() or is_blackhole(), reason="Unsupported on WH and BH") +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "out_mem_config", (ttnn.MemoryConfig(ttnn.TensorMemoryLayout.BLOCK_SHARDED, ttnn.BufferType.L1),), @@ -283,6 +284,7 @@ def test_layernorm_sharded_mix_precision_rm( assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shard_orientation", (ttnn.ShardOrientation.ROW_MAJOR, ttnn.ShardOrientation.COL_MAJOR), diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_matmul_dram_sharded.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_matmul_dram_sharded.py index 7bf2a871866..d258a76ba14 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_matmul_dram_sharded.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_matmul_dram_sharded.py @@ -18,6 +18,7 @@ untilize, is_close, ) +from models.utility_functions import skip_for_blackhole def find_max_subblock(out_block_h, out_block_w): @@ -190,6 +191,7 @@ def run_test_matmul_in1_dram_sharded( assert passing +@skip_for_blackhole("Segfault on BH, see #12349") @pytest.mark.parametrize( "fidelity", [ @@ -395,6 +397,7 @@ def run_test_matmul_in1_dram_sharded_mm_chain( assert True +@skip_for_blackhole("Segfaulting on BH, see #12349") @pytest.mark.parametrize( "fidelity", [ @@ -451,6 +454,7 @@ def test_matmul_in1_dram_sharded_with_mm_chain( ) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("packer_l1_acc", [True, False], ids=["pack_l1", "no_pack_l1"]) @pytest.mark.parametrize( "fp32_acc_mode", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_min_max.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_min_max.py index 94cce42f17a..1790198740b 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_min_max.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_min_max.py @@ -6,8 +6,10 @@ import pytest from functools import partial import ttnn +from models.utility_functions import skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "shape_dim", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_layernorm.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_layernorm.py index 5bde60039ee..217d367887e 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_layernorm.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_layernorm.py @@ -18,7 +18,7 @@ TILE_HEIGHT, TILE_WIDTH, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole def to_cpu(npu_tensor, shape, *, cpu_layout=ttnn.ROW_MAJOR_LAYOUT): @@ -398,6 +398,7 @@ def test_moreh_layernorm(input_shape_normalized_dims, elementwise_affine, eps, d run_moreh_layernorm(input_shape_normalized_dims, elementwise_affine, eps, device) +@skip_for_blackhole("Mismatching on BH, see #12349") @skip_for_grayskull("Using the transpose function in copy_tile causes a hang.") @pytest.mark.parametrize("eps", [1e-5], ids=["1e-5"]) @pytest.mark.parametrize( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss.py index a5355eba0da..fc59220021b 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss.py @@ -6,7 +6,7 @@ import ttnn import pytest -from models.utility_functions import comp_allclose_and_pcc +from models.utility_functions import comp_allclose_and_pcc, skip_for_blackhole from loguru import logger from tests.tt_eager.python_api_testing.unit_testing.misc.test_utils import ( @@ -81,6 +81,7 @@ def run_moreh_nll_loss(shape, ignore_index, reduction, none_weight, device, comp assert passing +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "shape", [[5, 10], [3000, 100], [200, 100, 90], [5, 50, 2, 7, 50, 70]], @@ -94,6 +95,7 @@ def test_moreh_nll_loss(shape, ignore_index, reduction, none_weight, device): run_moreh_nll_loss(shape, ignore_index, reduction, none_weight, device) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "shape", [ @@ -115,6 +117,7 @@ def test_moreh_nll_loss_callback(shape, reduction, none_weight, device, use_prog tt_dummy = to_npu(torch_dummy, device) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "shape", [ diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss_unreduced.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss_unreduced.py index 11d6947ff0b..f595f0c7e9a 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss_unreduced.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_nll_loss_unreduced.py @@ -6,7 +6,7 @@ import ttnn import pytest -from models.utility_functions import comp_allclose_and_pcc, is_wormhole_b0 +from models.utility_functions import comp_allclose_and_pcc, is_wormhole_b0, skip_for_blackhole from loguru import logger from tests.tt_eager.python_api_testing.unit_testing.misc.test_utils import ( @@ -108,6 +108,7 @@ def test_moreh_nll_loss_unreduced(shape, ignore_index, none_weight, compute_kern ) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "shape", [ diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_sum.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_sum.py index be7e7f861ed..656a946ac0f 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_sum.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_moreh_sum.py @@ -8,10 +8,7 @@ import ttnn -from models.utility_functions import ( - comp_allclose_and_pcc, - skip_for_grayskull, -) +from models.utility_functions import comp_allclose_and_pcc, skip_for_grayskull, skip_for_blackhole from tests.tt_eager.python_api_testing.unit_testing.misc.test_utils import ( get_compute_kernel_options, compute_kernel_options, @@ -487,6 +484,7 @@ def test_moreh_sum_backward_fp32_dest_acc(input_shape, dim, compute_kernel_optio assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @skip_for_grayskull() @pytest.mark.parametrize( "input_shape", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_nlp_create_qkv_heads_decode.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_nlp_create_qkv_heads_decode.py index 5a563f5c05c..8727b53290f 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_nlp_create_qkv_heads_decode.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_nlp_create_qkv_heads_decode.py @@ -17,6 +17,7 @@ tt2torch_tensor, skip_for_grayskull, nearest_32, + skip_for_blackhole, ) @@ -71,6 +72,7 @@ def run_test_create_head_interleaved(device, n_local_heads, n_local_kv_heads, he assert out_pass_q and out_pass_k and out_pass_v +@skip_for_blackhole("Requires eth connected devices to run, see #12349") @skip_for_grayskull("Requires eth connected devices to run") @pytest.mark.parametrize( "n_local_heads, n_local_kv_heads, head_dim, batch", @@ -172,6 +174,7 @@ def run_test_create_head_max_width_shard(device, n_local_heads, n_local_kv_heads assert out_pass_q and out_pass_k and out_pass_v +@skip_for_blackhole("Requires eth connected devices to run, see #12349") @skip_for_grayskull("Requires eth connected devices to run") @pytest.mark.parametrize( "n_local_heads, n_local_kv_heads, head_dim, batch", @@ -275,6 +278,7 @@ def run_test_create_min_width_shard( assert out_pass_q and out_pass_k and out_pass_v +@skip_for_blackhole("Requires eth connected devices to run, see #12349") @skip_for_grayskull("Requires eth connected devices to run") @pytest.mark.parametrize( "n_local_heads, n_local_kv_heads, head_dim", @@ -384,6 +388,7 @@ def run_test_create_width_shard_by_head( assert out_pass_q and out_pass_k and out_pass_v +@skip_for_blackhole("Requires eth connected devices to run, see #12349") @skip_for_grayskull("Requires eth connected devices to run") @pytest.mark.parametrize( "n_local_heads, n_local_kv_heads, head_dim", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_repeat.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_repeat.py index 14b799385ef..064ec07512b 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_repeat.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_repeat.py @@ -9,7 +9,7 @@ import torch import ttnn -from models.utility_functions import print_diff_argmax +from models.utility_functions import print_diff_argmax, skip_for_blackhole import pytest from loguru import logger @@ -49,6 +49,7 @@ def run_repeat(input_shape, repeats, device, layout, dtype, input_mem_config, ou assert passing +@skip_for_blackhole("Alignment failure on BH, see #12349") @pytest.mark.parametrize( "input_shape, repeats", ( @@ -99,6 +100,7 @@ def test_repeat( run_repeat(input_shape, repeats, device, layout, dtype, input_mem_config, output_mem_config) +@skip_for_blackhole("Alignment failure on BH, see #12349") @pytest.mark.parametrize( "input_shape, repeats", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshape.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshape.py index c58c06cb1f7..02e704aa50e 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshape.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshape.py @@ -5,10 +5,11 @@ import torch import ttnn -from models.utility_functions import print_diff_argmax +from models.utility_functions import print_diff_argmax, skip_for_blackhole import ttnn +@skip_for_blackhole("Mismatching on BH, see #12349") def test_tile_major_reshape(device): torch.manual_seed(0) diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshard.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshard.py index 722b7c532e5..53a00e13f1b 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshard.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_reshard.py @@ -10,7 +10,7 @@ comp_pcc, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole def run_reshard_test( @@ -82,6 +82,7 @@ def run_reshard_test( return torch_tensor, torch_tensor_after_round_trip +@skip_for_blackhole("Hitting assertion in reshard op on BH, see #12349") @pytest.mark.parametrize( "input_shape, input_layout, input_shard_grid, input_shard_shape, input_shard_orientation, input_sharding_scheme, output_shard_grid, output_shard_shape, output_shard_orientation, output_sharding_scheme", [ diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding.py index 973f35bb76b..b7ce9e9e902 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding.py @@ -7,7 +7,7 @@ from loguru import logger import ttnn -from models.utility_functions import comp_pcc, divup, is_grayskull +from models.utility_functions import comp_pcc, divup, is_grayskull, skip_for_blackhole def rotate_half(x): @@ -100,6 +100,7 @@ def test_rotary_embedding_prefill(W, Z, Y, X, cache_size, in_sharded, out_sharde assert p +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "W, Z, Y, X", ([1, 1, 32, 64], [1, 71, 32, 64], [1, 1, 64, 64], [1, 71, 64, 64], [1, 32, 32, 64], [1, 2, 32, 64]), @@ -245,6 +246,7 @@ def test_rotary_embedding_prefill_fp32( assert p +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.skipif(is_grayskull(), reason="GS does not support fp32") @pytest.mark.parametrize("W, Z, Y, X", [(1, 1, 32, 64)]) @pytest.mark.parametrize("cache_size", [2048]) diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding_llama.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding_llama.py index 476d95da5eb..05bc035479e 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding_llama.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_rotary_embedding_llama.py @@ -11,7 +11,7 @@ from tests.tt_eager.python_api_testing.sweep_tests.comparison_funcs import ( comp_pcc, ) -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole from models.demos.t3000.llama2_70b.tt.llama_common import precompute_freqs, freqs_to_rotation_matrix, gather_rotary_emb @@ -178,6 +178,7 @@ def run_test_rotary_embedding_llama( assert does_pass, f"PCC value is lower than {pcc}" +@skip_for_blackhole("Requires eth connected devices to run, only single chip BH available. See #12349") @skip_for_grayskull("Requires eth connected devices to run") @pytest.mark.parametrize( "batch, seq_len", @@ -253,6 +254,7 @@ def test_rotary_embedding_llama( ) +@skip_for_blackhole("Requires eth connected devices to run, only single chip BH available. See #12349") @skip_for_grayskull("Requires eth connected devices to run") @pytest.mark.parametrize( "batch, seq_len", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention.py index be5b730bf6f..14d6f99538e 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention.py @@ -12,7 +12,7 @@ import ttnn from loguru import logger import pytest -from models.utility_functions import skip_for_grayskull, skip_for_wormhole_b0 +from models.utility_functions import skip_for_grayskull, skip_for_wormhole_b0, skip_for_blackhole def is_watcher_enabled(): @@ -60,6 +60,7 @@ def run_test_sdpa_tt(device, b, nh, nkv, s, d, q_chunk_size, k_chunk_size, dtype # @pytest.mark.skip(reason="ND PCC issues") +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.skipif(is_watcher_enabled(), reason="Kernel OOM with watcher enabled") @skip_for_grayskull("Unsupported in GS since L1 runs OOM with most configs") @pytest.mark.parametrize("dtype", [ttnn.bfloat8_b, ttnn.bfloat16], ids=["bfp8", "bf16"]) @@ -85,6 +86,7 @@ def test_sdpa_tt(device, b, nh, nkv, s, d, q_chunk_size, k_chunk_size, dtype): # @pytest.mark.skip(reason="ND PCC issues") +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.skipif(is_watcher_enabled(), reason="Kernel OOM with watcher enabled") @skip_for_grayskull("Unsupported in GS since L1 runs OOM with most configs") @pytest.mark.parametrize("dtype", [ttnn.bfloat8_b, ttnn.bfloat16], ids=["bfp8", "bf16"]) diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention_decode.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention_decode.py index 84963dfff64..baebfd39db2 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention_decode.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_scaled_dot_product_attention_decode.py @@ -12,7 +12,7 @@ import ttnn from loguru import logger import pytest -from models.utility_functions import skip_for_grayskull, skip_for_wormhole_b0 +from models.utility_functions import skip_for_grayskull, skip_for_wormhole_b0, skip_for_blackhole import math import numpy as np @@ -637,6 +637,7 @@ def to_contiguous_cache(paged_cache, batch, num_kv, max_num_blocks_per_seq, bloc # return +@skip_for_blackhole("Unsupported on BH, see #12349") @skip_for_grayskull("Unsupported in GS since L1 runs OOM with most configs") # @pytest.mark.skip("Skipping due to potential nd pcc issue #9370") @pytest.mark.parametrize( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_sharded.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_sharded.py index a61e2192cfe..fcb3c2c7db0 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_sharded.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_sharded.py @@ -11,7 +11,7 @@ comp_equal, comp_pcc, ) -from models.utility_functions import is_wormhole_b0, is_wormhole_b0, is_blackhole +from models.utility_functions import is_wormhole_b0, is_wormhole_b0, is_blackhole, skip_for_blackhole from loguru import logger from models.utility_functions import torch2tt_tensor, tt2torch_tensor, pad_by_zero, roundup32 @@ -180,6 +180,7 @@ def test_sharded_rm( assert passing +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("H, num_cores", [[100352, 98], [25088, 98]]) @pytest.mark.parametrize("in_sharded", [True, False]) @pytest.mark.parametrize("out_sharded", [True, False]) @@ -255,6 +256,7 @@ def test_sharded_untilize(H, num_cores, in_sharded, out_sharded, dtype, device, assert passing +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("H, num_cores", [[25088, 98]]) @pytest.mark.parametrize("output_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) def test_sharded_tilize(H, num_cores, output_dtype, device, function_level_defaults): @@ -1066,6 +1068,7 @@ def test_sharded_program_cache(device, use_program_cache, function_level_default assert eq +@skip_for_blackhole("Hanging on BH, see #12349") @pytest.mark.parametrize("in0_sharded", [True, False], ids=["in0_sharded", "in0_unsharded"]) @pytest.mark.parametrize("out_sharded", [True, False], ids=["out_sharded", "out_unsharded"]) @pytest.mark.parametrize("M", [1600]) @@ -1335,6 +1338,7 @@ def test_sharded_matmul_2d_transposed( assert passing +@skip_for_blackhole("Hanging on BH, see #12349") def test_resharded_binary_to_matmul(device, function_level_defaults): grid_size_binary = device.compute_with_storage_grid_size() num_cores_binary = 98 @@ -1761,6 +1765,7 @@ def test_width_sharded_untilize_with_unpadding( assert passing +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("input_shape", [[8, 1, 49, 2048], [1, 1, 8, 2048], [16, 1, 49, 2048], [1, 1, 16, 2048]]) @pytest.mark.parametrize("sharding_config", [(True, True), (False, False)], ids=["both_sharded", "both_interleaved"]) @pytest.mark.parametrize("output_dtype", [ttnn.bfloat16, ttnn.bfloat8_b]) @@ -1832,6 +1837,7 @@ def test_sharded_tilize_with_val_padding(input_shape, sharding_config, output_dt assert passing +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("N", [8, 16]) @pytest.mark.parametrize("in_sharded", [True], ids=["in0_sharded"]) @pytest.mark.parametrize("out_sharded", [True], ids=["out_sharded"]) @@ -1990,6 +1996,7 @@ def test_sharded_matmul_1d_in0( # Have at least one example of 1d matmul with in1 mcasted that runs on WH +@skip_for_blackhole("Hangs on BH, see #12349") def test_sharded_matmul_1d_in1_wormhole(device, function_level_defaults): M = 4096 K = 64 diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_single_core_fused_ops.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_single_core_fused_ops.py index e9ceec51711..2a467ae3476 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_single_core_fused_ops.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_single_core_fused_ops.py @@ -12,12 +12,13 @@ is_close, ) -from models.utility_functions import comp_pcc, pad_by_zero +from models.utility_functions import comp_pcc, pad_by_zero, skip_for_blackhole shapes = [[1, 1, 32, 32], [1, 1, 32, 128], [1, 2, 128, 128]] # still have to skip b/c of very low PCC in WH B0 +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("shape", shapes) def test_softmax(shape, device): torch.manual_seed(1234) @@ -34,6 +35,7 @@ def test_softmax(shape, device): assert passing +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("shape", shapes) def test_layernorm(shape, device): torch.manual_seed(1234) diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax.py index bb5c14342bb..820416ce192 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax.py @@ -17,9 +17,10 @@ ) from models.utility_functions import print_diff_argmax, comp_pcc from models.utility_functions import torch2tt_tensor, tt2torch_tensor, pad_by_zero -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, is_blackhole, skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "dtype", (ttnn.bfloat16, ttnn.float32), @@ -27,8 +28,8 @@ ) @pytest.mark.parametrize("inplace", [True, False]) def test_softmax(device, inplace, dtype): - if is_grayskull() and dtype == ttnn.float32: - pytest.skip("Skipping float32 tests on Grayskull") + if (is_grayskull() or is_blackhole()) and dtype == ttnn.float32: + pytest.skip("Skipping float32 tests on Grayskull and Blackhole. For Blackhole see #12349") torch.manual_seed(0) sm_op = ttnn.softmax_in_place if inplace else ttnn.softmax @@ -69,6 +70,7 @@ def test_softmax(device, inplace, dtype): assert allclose, f"FAILED: {output}" +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("inplace", [True, False]) def test_softmax_with_program_cache(device, use_program_cache, inplace): torch.manual_seed(0) @@ -92,6 +94,7 @@ def test_softmax_with_program_cache(device, use_program_cache, inplace): assert allclose, f"FAILED: {output}" +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "in_dtype", (ttnn.bfloat16, ttnn.bfloat8_b), @@ -120,6 +123,7 @@ def test_softmax_mix_precision(device, inplace, in_dtype): assert allclose, f"FAILED: {output}" +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "seq_len", [64, 384], @@ -226,6 +230,7 @@ def test_scale_mask_softmax_inplace(device, in_dtype, in0_mem_config, causal_mas assert allclose, f"FAILED: {output}" +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "in0_mem_config", (ttnn.MemoryConfig(ttnn.TensorMemoryLayout.INTERLEAVED, ttnn.BufferType.DRAM),), diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax_sharded.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax_sharded.py index 0588d33f472..395af603163 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax_sharded.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_softmax_sharded.py @@ -17,9 +17,10 @@ ) from models.utility_functions import print_diff_argmax, comp_pcc from models.utility_functions import torch2tt_tensor, tt2torch_tensor, pad_by_zero -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("device_params", [{"l1_small_size": 8192}], indirect=True) @pytest.mark.parametrize( "in0_mem_config", @@ -93,6 +94,7 @@ def test_softmax_causal_mask(device, in_dtype, in0_mem_config): assert allclose, f"FAILED: {output}" +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("device_params", [{"l1_small_size": 8192}], indirect=True) @pytest.mark.parametrize( "causal_mask", @@ -189,6 +191,7 @@ def test_softmax(device, in_dtype, in0_mem_config, causal_mask): assert allclose, f"FAILED: {output}" +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("device_params", [{"l1_small_size": 8192}], indirect=True) @pytest.mark.parametrize( "causal_mask", @@ -286,6 +289,7 @@ def test_scale_mask_softmax_rm(device, in_dtype, in0_mem_config, causal_mask): assert allclose, f"FAILED: {output}" +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("device_params", [{"l1_small_size": 8192}], indirect=True) @pytest.mark.parametrize( "shard_orient", diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_sum.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_sum.py index 5175737af02..5bafca9f180 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_sum.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_sum.py @@ -6,7 +6,10 @@ import pytest import ttnn +from models.utility_functions import skip_for_blackhole + +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "shape_dim", ( @@ -37,6 +40,7 @@ def test_sum_for_dim_hw(device, use_program_cache, shape_dim): assert torch.equal(tt_dev[0, 0, 0, 0], torch.Tensor([value]).bfloat16()[0]) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "shape", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_hpadding_matmul.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_hpadding_matmul.py index 64f445787b3..b3148c62565 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_hpadding_matmul.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_hpadding_matmul.py @@ -16,6 +16,7 @@ print_diff_argmax, is_close, comp_pcc, + skip_for_blackhole, ) import torch @@ -61,5 +62,6 @@ def run_tilize_matmul_test(M, K, N, device): assert passing_pcc +@skip_for_blackhole("Hanging on BH, see #12349") def test_tilize_hpadding_matmul(device): run_tilize_matmul_test(4, 32 * 9, 32, device) diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_matmul.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_matmul.py index cc864731fa3..b7207685127 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_matmul.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_tilize_matmul.py @@ -6,11 +6,7 @@ import ttnn from loguru import logger -from models.utility_functions import ( - untilize, - tilize_to_list, - comp_pcc, -) +from models.utility_functions import untilize, tilize_to_list, comp_pcc, skip_for_blackhole import torch @@ -50,5 +46,6 @@ def run_tilize_matmul_test(M, K, N, device): assert passing_pcc +@skip_for_blackhole("Hanging on BH, see #12349") def test_run_tilize_matmul_test(device): run_tilize_matmul_test(32, 32, 32, device) diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_transpose.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_transpose.py index 153f3a9aa39..eade5649f90 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_transpose.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_transpose.py @@ -11,7 +11,7 @@ from loguru import logger from models.utility_functions import is_grayskull from tests.tt_eager.python_api_testing.sweep_tests.comparison_funcs import comp_pcc, comp_equal -from models.utility_functions import skip_for_grayskull +from models.utility_functions import skip_for_grayskull, skip_for_blackhole from tests.ttnn.utils_for_testing import assert_with_pcc @@ -57,6 +57,7 @@ def transpose( assert device.num_program_cache_entries() == expected_program_cache_size +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "dtype", (ttnn.bfloat16, ttnn.float32), @@ -96,6 +97,7 @@ def test_transpose_wh_bfp4(device): transpose(input_shape, device, dim0=-2, dim1=-1, input_dtype=ttnn.bfloat4_b) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "dtype", (ttnn.bfloat16, ttnn.float32), @@ -301,6 +303,7 @@ def test_transpose_wh_sharded_program_cache(dtype, device, use_program_cache): ) +@skip_for_blackhole("Mismatching on BH, see #12349") @skip_for_grayskull("Grayskull has pcc issue when transpose used untilize") @pytest.mark.parametrize("n", [1]) @pytest.mark.parametrize("c", [1]) @@ -333,6 +336,7 @@ def test_tranpose_hw_rm_with_padding(device, n, c, h, w): assert_with_pcc(torch_output_tensor, activation_pyt_padded_out, 0.9999) +@skip_for_blackhole("Mismatching on BH, see #12349") @skip_for_grayskull("Grayskull has pcc issue when transpose used untilize") @pytest.mark.parametrize("n", [16]) @pytest.mark.parametrize("c", [128]) @@ -372,6 +376,7 @@ def run_tranpose_hw_rm_program_cache(device, n, c, h, w, use_program_cache): assert_with_pcc(torch_output_tensor, activation_pyt_padded_out, 0.9999) +@skip_for_blackhole("Mismatching on BH, see #12349") @skip_for_grayskull("Grayskull has pcc issue when transpose used untilize") @pytest.mark.parametrize("n", [16]) @pytest.mark.parametrize("c", [128]) @@ -393,6 +398,7 @@ def test_tranpose_hw_rm_with_program_cache(device, n, c, h, w, use_program_cache assert device.num_program_cache_entries() == 1 +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("n", [16]) @pytest.mark.parametrize("c", [224]) @pytest.mark.parametrize("h", [16]) @@ -462,6 +468,7 @@ def run_tranpose_hw_sharded_rm_with_program_cache(device, n, c, h, w): assert_with_pcc(torch_output_tensor, tt_output_tensor, 0.9999) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("n", [16]) @pytest.mark.parametrize("c", [128]) @pytest.mark.parametrize("h", [128]) @@ -482,6 +489,7 @@ def test_tranpose_hw_sharded_rm_with_program_cache(device, n, c, h, w, use_progr assert device.num_program_cache_entries() == 3 +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("n", [16]) @pytest.mark.parametrize("c", [128]) @pytest.mark.parametrize("h", [128]) @@ -521,6 +529,7 @@ def run_tranpose_hc_rm_with_program_cache(device, n, c, h, w, use_program_cache) assert_with_pcc(torch_output_tensor, activation_pyt_padded_out, 0.9999) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("n", [20]) @pytest.mark.parametrize("c", [128]) @pytest.mark.parametrize("h", [256]) @@ -570,6 +579,7 @@ def run_tranpose_hc_sharded(device, n, c, h, w, grid_size): assert_with_pcc(torch_output_tensor, tt_output_tensor, 0.9999) +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "n, c, h, w, grid_size", [ diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_unary_ops_ttnn.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_unary_ops_ttnn.py index 10c5177a6d6..bda1f32c355 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_unary_ops_ttnn.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_unary_ops_ttnn.py @@ -6,6 +6,7 @@ import pytest import ttnn from tests.tt_eager.python_api_testing.unit_testing.backward_ops.utility_funcs import data_gen_with_range, compare_pcc +from models.utility_functions import skip_for_blackhole @pytest.mark.parametrize( @@ -28,6 +29,7 @@ def test_unary_square_ttnn(input_shapes, device): assert comp_pass +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shapes", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_untilize_test.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_untilize_test.py index 2561d8edd62..742f2f568fb 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_untilize_test.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_untilize_test.py @@ -8,9 +8,10 @@ import ttnn from models.utility_functions import untilize, comp_pcc -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "dtype", (ttnn.bfloat16, ttnn.float32), diff --git a/tests/tt_eager/python_api_testing/unit_testing/misc/test_update_cache.py b/tests/tt_eager/python_api_testing/unit_testing/misc/test_update_cache.py index 960daf38309..550e465b475 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/misc/test_update_cache.py +++ b/tests/tt_eager/python_api_testing/unit_testing/misc/test_update_cache.py @@ -8,9 +8,10 @@ from loguru import logger from models.utility_functions import nearest_32, pad_by_zero from tests.tt_eager.python_api_testing.sweep_tests.comparison_funcs import comp_pcc, comp_equal -from models.utility_functions import is_grayskull +from models.utility_functions import is_grayskull, skip_for_blackhole +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("head_dim", [64]) @pytest.mark.parametrize("max_seq_len", [2048]) @pytest.mark.parametrize("num_users", [8, 16, 32, 64]) @@ -139,6 +140,7 @@ def test_update_cache_decode( assert eq_cache and eq_update +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize("head_dim", [64]) @pytest.mark.parametrize("max_seq_len", [2048]) @pytest.mark.parametrize("num_users", [8, 16, 32, 64]) diff --git a/tests/tt_eager/python_api_testing/unit_testing/test_prod_all.py b/tests/tt_eager/python_api_testing/unit_testing/test_prod_all.py index e52b13e373b..1c177686b37 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/test_prod_all.py +++ b/tests/tt_eager/python_api_testing/unit_testing/test_prod_all.py @@ -8,7 +8,7 @@ from functools import partial import ttnn -from models.utility_functions import comp_allclose_and_pcc +from models.utility_functions import comp_allclose_and_pcc, skip_for_blackhole from tests.tt_eager.python_api_testing.sweep_tests import ( comparison_funcs, @@ -33,6 +33,7 @@ def get_tensors(input_shape, output_shape, device): return tt_input, tt_output, torch_input +@skip_for_blackhole("Hangs on Blackhole, see #12349") @pytest.mark.parametrize( "shapes", ( diff --git a/tests/tt_eager/python_api_testing/unit_testing/test_prod_nc.py b/tests/tt_eager/python_api_testing/unit_testing/test_prod_nc.py index d8a92c0193e..71641bc8071 100644 --- a/tests/tt_eager/python_api_testing/unit_testing/test_prod_nc.py +++ b/tests/tt_eager/python_api_testing/unit_testing/test_prod_nc.py @@ -7,7 +7,7 @@ from loguru import logger import ttnn -from models.utility_functions import comp_allclose_and_pcc +from models.utility_functions import comp_allclose_and_pcc, skip_for_blackhole TILE_HEIGHT = 32 TILE_WIDTH = 32 @@ -28,6 +28,7 @@ def get_tensors(input_shape, output_shape, device): return tt_input, tt_output, torch_input +@skip_for_blackhole("Mismatching on BH, see #12349") @pytest.mark.parametrize( "input_shape", ( diff --git a/tests/ttnn/unit_tests/operations/test_moreh_getitem.py b/tests/ttnn/unit_tests/operations/test_moreh_getitem.py index f46f03813ca..ddc86b94d5c 100644 --- a/tests/ttnn/unit_tests/operations/test_moreh_getitem.py +++ b/tests/ttnn/unit_tests/operations/test_moreh_getitem.py @@ -6,7 +6,7 @@ import ttnn import pytest -from models.utility_functions import comp_allclose_and_pcc +from models.utility_functions import comp_allclose_and_pcc, is_blackhole, skip_for_blackhole from loguru import logger @@ -20,6 +20,7 @@ def to_output_5d_shape(shape, index_dims, index_size): return output_5d_shape +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dim", ( @@ -82,6 +83,7 @@ def test_getitem_RAW_MJOR_one_index(shape_index_dim, dtype, index_size, device): assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dims", ( @@ -143,6 +145,7 @@ def test_getitem_RAW_MAJOR_two_indices(shape_index_dims, dtype, index_size, devi assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dims", ( @@ -202,6 +205,7 @@ def test_getitem_RAW_MAJOR_three_indices(shape_index_dims, dtype, index_size, de assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dim", ( @@ -314,6 +318,7 @@ def test_getitem_tilized_one_index(shape_index_dim, dtype, index_size, row_major assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dims", ( @@ -406,6 +411,7 @@ def test_getitem_tilized_two_indices(shape_index_dims, dtype, index_size, row_ma assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dims", ( @@ -492,6 +498,7 @@ def test_getitem_tilized_three_indices(shape_index_dims, dtype, index_size, row_ assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dims", ( @@ -574,6 +581,7 @@ def test_getitem_tilized_four_indices(shape_index_dims, dtype, index_size, row_m assert passing +@skip_for_blackhole("Mismatching on Blackhole, see #12349") @pytest.mark.parametrize( "shape_index_dims", (((10, 3, 5, 7, 80), (0, 1, 2, 3, 4)),),