record_forge_property = <function record_property.<locals>.append_property at 0x7f8225c84af0>
variant = 'ghostnet_100'

    @pytest.mark.push
    @pytest.mark.nightly
    @pytest.mark.parametrize("variant", variants, ids=variants)
    def test_ghostnet_timm(record_forge_property, variant):
        # Build Module Name
        module_name = build_module_name(
            framework=Framework.PYTORCH,
            model="ghostnet",
            variant=variant,
            source=Source.TIMM,
            task=Task.IMAGE_CLASSIFICATION,
        )
    
        # Record Forge Property
        record_forge_property("model_name", module_name)
    
        # Load the model and prepare input data
        framework_model, inputs = load_ghostnet_model(variant)
    
        # Forge compile framework model
        compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name)
    
        # Model Verification
>       verify(inputs, framework_model, compiled_model)

forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py:42: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f82dd3f7f70>
inputs = (tensor([[[[-1.92953, -1.92953, -1.91241,  ..., -2.03228, -1.94666, -1.92953],
          [-1.99803, -1.89528, -1.91241... ..., -0.95041, -1.49072, -1.38614],
          [-1.28157, -1.42100, -1.22928,  ..., -0.74126, -1.12471, -1.28157]]]]),)
inputs_and_parameters = [tensor([[[[-1.92953, -1.92953, -1.91241,  ..., -2.03228, -1.94666, -1.92953],
          [-1.99803, -1.89528, -1.91241...True), tensor([[[[1.59695, 0.57756, 2.44167, 0.77118, 1.27917, 0.69326, 1.43299, 1.02739]]]], requires_grad=True), ...]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*to_pt_tensors(inputs)]
    
        inputs_and_parameters = [
            *self.inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        assert all(
            [isinstance(t, torch.Tensor) for t in inputs_and_parameters]
        ), "All inputs should be torch tensors by now."
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch
                    # module. This is because we want to be able to optimize the parameters both on the device
                    # (through our runtime) and via the torch optimizers. So this ensures that whichever side updates
                    # the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_THROW @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/allocator/bank_manager.cpp:131: tt::exception
E       info:
E       Out of Memory: Not enough space to allocate 5033164800 B DRAM buffer across 12 banks, where each bank needs to store 419430400 B
E       backtrace:
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x14bfe5) [0x7f82912e4fe5]
E        --- tt::tt_metal::BankManager::allocate_buffer(unsigned long, unsigned long, bool, CoreRangeSet const&, std::optional<unsigned int>)
E        --- tt::tt_metal::Allocator::allocate_buffer(tt::tt_metal::v0::Buffer*)
E        --- tt::tt_metal::v0::Buffer::allocate_impl()
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x118b0f) [0x7f82912b1b0f]
E        --- tt::tt_metal::v0::Buffer::create(tt::tt_metal::v0::IDevice*, unsigned long, unsigned long, tt::tt_metal::BufferType, tt::tt_metal::TensorMemoryLayout, std::optional<tt::tt_metal::ShardSpecBuffer> const&, std::optional<bool>, std::optional<tt::tt_metal::SubDeviceId>)
E        --- tt::tt_metal::tensor_impl::allocate_buffer_on_device(tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&)
E        --- std::shared_ptr<tt::tt_metal::v0::Buffer> tt::tt_metal::tensor_impl::initialize_data_on_device<float, tt::tt_metal::borrowed_buffer::Buffer>(tt::tt_metal::borrowed_buffer::Buffer<float>&, tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1ea91df) [0x7f82933ff1df]
E        --- tt::tt_metal::Tensor tt::tt_metal::tensor_impl::to_device<float>(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(_ZN2tt8tt_metal11tensor_impl8dispatchIZNS1_17to_device_wrapperIJRNS0_6TensorERPNS0_2v07IDeviceERKNS0_12MemoryConfigERhEEEDaDpOT_EUlTyDpOT0_E_JS5_S9_SC_SD_EEEDaNS0_8DataTypeEOT_SJ_+0x6d) [0x7f8293405bfd]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1eaf8f1) [0x7f82934058f1]
E        --- tt::tt_metal::tensor_ops::tensor_to(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- tt::tt_metal::Tensor::to(tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char) const
E        --- ttnn::operations::core::to_device(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, std::optional<tt::tt_metal::MemoryConfig> const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x1bf89e) [0x7f8294f6489e]
E        --- tt::runtime::ttnn::LayoutConverter::handleHostInputLayoutNoTypecast(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertHostTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::toLayout(tt::runtime::Tensor, tt::runtime::Device, tt::runtime::Layout)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x3177f0) [0x7f82a29467f0]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x31773e) [0x7f82a294673e]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0xe0d05) [0x7f82a270fd05]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18ab32) [0x5587f7c49b32]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x59c7) [0x5587f7c39a97]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x5587f7c3f574]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x5587f7c58172]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x8af) [0x5587f7c3497f]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]

/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/compiled_graph_state.py:253: RuntimeError

record_forge_property = <function record_property.<locals>.append_property at 0x7f82d78097e0>

    @pytest.mark.nightly
    @pytest.mark.push
    def test_mobilenetv1_basic(record_forge_property):
        # Build Module Name
        module_name = build_module_name(
            framework=Framework.PYTORCH,
            model="mobilenet_v1",
            variant="basic",
            source=Source.TORCHVISION,
            task=Task.IMAGE_CLASSIFICATION,
        )
    
        # Record Forge Property
        record_forge_property("model_name", module_name)
    
        # Load the model and prepare input data
        framework_model, inputs = load_mobilenet_model("mobilenet_v1")
    
        # Forge compile framework model
        compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name)
    
        #  Model Verification
>       verify(inputs, framework_model, compiled_model)

forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py:42: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f8286ec9a20>
inputs = (tensor([[[[-1.92953, -1.92953, -1.91241,  ..., -2.03228, -1.94666, -1.92953],
          [-1.99803, -1.89528, -1.91241... ..., -0.95041, -1.49072, -1.38614],
          [-1.28157, -1.42100, -1.22928,  ..., -0.74126, -1.12471, -1.28157]]]]),)
inputs_and_parameters = [tensor([[[[-1.92953, -1.92953, -1.91241,  ..., -2.03228, -1.94666, -1.92953],
          [-1.99803, -1.89528, -1.91241...0.99999, 0.99999, 0.99999, 0.99999, 0.99999, 0.99999, 0.99999, 0.99999, 0.99999, 0.99999]]]], requires_grad=True), ...]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*to_pt_tensors(inputs)]
    
        inputs_and_parameters = [
            *self.inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        assert all(
            [isinstance(t, torch.Tensor) for t in inputs_and_parameters]
        ), "All inputs should be torch tensors by now."
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch
                    # module. This is because we want to be able to optimize the parameters both on the device
                    # (through our runtime) and via the torch optimizers. So this ensures that whichever side updates
                    # the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_THROW @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/allocator/bank_manager.cpp:131: tt::exception
E       info:
E       Out of Memory: Not enough space to allocate 4294967296 B DRAM buffer across 12 banks, where each bank needs to store 357916672 B
E       backtrace:
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x14bfe5) [0x7f82912e4fe5]
E        --- tt::tt_metal::BankManager::allocate_buffer(unsigned long, unsigned long, bool, CoreRangeSet const&, std::optional<unsigned int>)
E        --- tt::tt_metal::Allocator::allocate_buffer(tt::tt_metal::v0::Buffer*)
E        --- tt::tt_metal::v0::Buffer::allocate_impl()
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x118b0f) [0x7f82912b1b0f]
E        --- tt::tt_metal::v0::Buffer::create(tt::tt_metal::v0::IDevice*, unsigned long, unsigned long, tt::tt_metal::BufferType, tt::tt_metal::TensorMemoryLayout, std::optional<tt::tt_metal::ShardSpecBuffer> const&, std::optional<bool>, std::optional<tt::tt_metal::SubDeviceId>)
E        --- tt::tt_metal::tensor_impl::allocate_buffer_on_device(tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&)
E        --- std::shared_ptr<tt::tt_metal::v0::Buffer> tt::tt_metal::tensor_impl::initialize_data_on_device<float, tt::tt_metal::borrowed_buffer::Buffer>(tt::tt_metal::borrowed_buffer::Buffer<float>&, tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1ea91df) [0x7f82933ff1df]
E        --- tt::tt_metal::Tensor tt::tt_metal::tensor_impl::to_device<float>(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(_ZN2tt8tt_metal11tensor_impl8dispatchIZNS1_17to_device_wrapperIJRNS0_6TensorERPNS0_2v07IDeviceERKNS0_12MemoryConfigERhEEEDaDpOT_EUlTyDpOT0_E_JS5_S9_SC_SD_EEEDaNS0_8DataTypeEOT_SJ_+0x6d) [0x7f8293405bfd]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1eaf8f1) [0x7f82934058f1]
E        --- tt::tt_metal::tensor_ops::tensor_to(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- tt::tt_metal::Tensor::to(tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char) const
E        --- ttnn::operations::core::to_device(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, std::optional<tt::tt_metal::MemoryConfig> const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x1bf89e) [0x7f8294f6489e]
E        --- tt::runtime::ttnn::LayoutConverter::handleHostInputLayoutNoTypecast(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertHostTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::toLayout(tt::runtime::Tensor, tt::runtime::Device, tt::runtime::Layout)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x3177f0) [0x7f82a29467f0]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x31773e) [0x7f82a294673e]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0xe0d05) [0x7f82a270fd05]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18ab32) [0x5587f7c49b32]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x59c7) [0x5587f7c39a97]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x5587f7c3f574]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x5587f7c58172]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x8af) [0x5587f7c3497f]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]

/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/compiled_graph_state.py:253: RuntimeError

variant = 'microsoft/resnet-50'
record_forge_property = <function record_property.<locals>.append_property at 0x7f8224d6f250>

    @pytest.mark.push
    @pytest.mark.nightly
    @pytest.mark.parametrize("variant", variants, ids=variants)
    def test_resnet_hf(variant, record_forge_property):
        random.seed(0)
    
        # Record model details
        module_name = build_module_name(
            framework=Framework.PYTORCH,
            model="resnet",
            variant="50",
            source=Source.HUGGINGFACE,
            task=Task.IMAGE_CLASSIFICATION,
        )
        record_forge_property("model_name", module_name)
    
        # Load tiny dataset
        dataset = load_dataset("zh-plus/tiny-imagenet")
        images = random.sample(dataset["valid"]["image"], 10)
    
        # Load framework model
        framework_model = download_model(ResNetForImageClassification.from_pretrained, variant, return_dict=False)
    
        # Compile model
        input_sample = [torch.rand(1, 3, 224, 224)]
        compiled_model = forge.compile(framework_model, input_sample)
    
        # Verify data on sample input
>       verify(input_sample, framework_model, compiled_model, VerifyConfig(value_checker=AutomaticValueChecker(pcc=0.95)))

forge/test/models/pytorch/vision/resnet/test_resnet.py:55: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f8225f163b0>
inputs = (tensor([[[[0.49626, 0.76822, 0.08848,  ..., 0.99462, 0.68062, 0.51416],
          [0.06670, 0.74769, 0.14386,  ..., 0...0.15323,  ..., 0.90259, 0.88306, 0.96599],
          [0.31521, 0.63833, 0.24750,  ..., 0.43326, 0.19156, 0.85300]]]]),)
inputs_and_parameters = [tensor([[[[0.49626, 0.76822, 0.08848,  ..., 0.99462, 0.68062, 0.51416],
          [0.06670, 0.74769, 0.14386,  ..., 0...e+01,  1.42619e+02,  1.51779e+02,  2.47439e-01,  9.74630e-02,  1.03912e+02,  2.69300e-01]]]], requires_grad=True), ...]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*to_pt_tensors(inputs)]
    
        inputs_and_parameters = [
            *self.inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        assert all(
            [isinstance(t, torch.Tensor) for t in inputs_and_parameters]
        ), "All inputs should be torch tensors by now."
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch
                    # module. This is because we want to be able to optimize the parameters both on the device
                    # (through our runtime) and via the torch optimizers. So this ensures that whichever side updates
                    # the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_THROW @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/allocator/bank_manager.cpp:131: tt::exception
E       info:
E       Out of Memory: Not enough space to allocate 1073741824 B DRAM buffer across 12 banks, where each bank needs to store 89481216 B
E       backtrace:
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x14bfe5) [0x7f82912e4fe5]
E        --- tt::tt_metal::BankManager::allocate_buffer(unsigned long, unsigned long, bool, CoreRangeSet const&, std::optional<unsigned int>)
E        --- tt::tt_metal::Allocator::allocate_buffer(tt::tt_metal::v0::Buffer*)
E        --- tt::tt_metal::v0::Buffer::allocate_impl()
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x118b0f) [0x7f82912b1b0f]
E        --- tt::tt_metal::v0::Buffer::create(tt::tt_metal::v0::IDevice*, unsigned long, unsigned long, tt::tt_metal::BufferType, tt::tt_metal::TensorMemoryLayout, std::optional<tt::tt_metal::ShardSpecBuffer> const&, std::optional<bool>, std::optional<tt::tt_metal::SubDeviceId>)
E        --- tt::tt_metal::tensor_impl::allocate_buffer_on_device(tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&)
E        --- std::shared_ptr<tt::tt_metal::v0::Buffer> tt::tt_metal::tensor_impl::initialize_data_on_device<float, tt::tt_metal::borrowed_buffer::Buffer>(tt::tt_metal::borrowed_buffer::Buffer<float>&, tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1ea91df) [0x7f82933ff1df]
E        --- tt::tt_metal::Tensor tt::tt_metal::tensor_impl::to_device<float>(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(_ZN2tt8tt_metal11tensor_impl8dispatchIZNS1_17to_device_wrapperIJRNS0_6TensorERPNS0_2v07IDeviceERKNS0_12MemoryConfigERhEEEDaDpOT_EUlTyDpOT0_E_JS5_S9_SC_SD_EEEDaNS0_8DataTypeEOT_SJ_+0x6d) [0x7f8293405bfd]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1eaf8f1) [0x7f82934058f1]
E        --- tt::tt_metal::tensor_ops::tensor_to(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- tt::tt_metal::Tensor::to(tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char) const
E        --- ttnn::operations::core::to_device(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, std::optional<tt::tt_metal::MemoryConfig> const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x1bf89e) [0x7f8294f6489e]
E        --- tt::runtime::ttnn::LayoutConverter::handleHostInputLayoutNoTypecast(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertHostTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::toLayout(tt::runtime::Tensor, tt::runtime::Device, tt::runtime::Layout)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x3177f0) [0x7f82a29467f0]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x31773e) [0x7f82a294673e]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0xe0d05) [0x7f82a270fd05]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18ab32) [0x5587f7c49b32]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x59c7) [0x5587f7c39a97]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x5587f7c3f574]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x5587f7c58172]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x8af) [0x5587f7c3497f]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]

/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/compiled_graph_state.py:253: RuntimeError

record_forge_property = <function record_property.<locals>.append_property at 0x7f8225c6b5b0>
variant = 'resnext50_32x4d'

    @pytest.mark.push
    @pytest.mark.nightly
    @pytest.mark.parametrize("variant", ["resnext50_32x4d"])
    def test_resnext_50_torchhub_pytorch(record_forge_property, variant):
        # Build Module Name
        module_name = build_module_name(
            framework=Framework.PYTORCH,
            model="resnext",
            source=Source.TORCH_HUB,
            variant=variant,
            task=Task.IMAGE_CLASSIFICATION,
        )
    
        # Record Forge Property
        record_forge_property("model_name", module_name)
    
        # Load the model and prepare input data
        framework_model, inputs = load_resnext_model(variant)
    
        # Forge compile framework model
        compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name)
    
        # Model Verification
>       verify(inputs, framework_model, compiled_model)

forge/test/models/pytorch/vision/resnext/test_resnext.py:43: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f83375b26b0>
inputs = (tensor([[[[-1.92953, -1.92953, -1.91241,  ..., -2.03228, -1.94666, -1.92953],
          [-1.99803, -1.89528, -1.91241... ..., -0.95041, -1.49072, -1.38614],
          [-1.28157, -1.42100, -1.22928,  ..., -0.74126, -1.12471, -1.28157]]]]),)
inputs_and_parameters = [tensor([[[[-1.92953, -1.92953, -1.91241,  ..., -2.03228, -1.94666, -1.92953],
          [-1.99803, -1.89528, -1.91241....25258e+00, 3.00196e+00, 5.26707e+00, 6.10776e+00, 4.91007e+00, 3.34621e+00, 4.80464e+00]]]], requires_grad=True), ...]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*to_pt_tensors(inputs)]
    
        inputs_and_parameters = [
            *self.inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        assert all(
            [isinstance(t, torch.Tensor) for t in inputs_and_parameters]
        ), "All inputs should be torch tensors by now."
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch
                    # module. This is because we want to be able to optimize the parameters both on the device
                    # (through our runtime) and via the torch optimizers. So this ensures that whichever side updates
                    # the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_THROW @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/allocator/bank_manager.cpp:131: tt::exception
E       info:
E       Out of Memory: Not enough space to allocate 2147483648 B DRAM buffer across 12 banks, where each bank needs to store 178958336 B
E       backtrace:
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x14bfe5) [0x7f82912e4fe5]
E        --- tt::tt_metal::BankManager::allocate_buffer(unsigned long, unsigned long, bool, CoreRangeSet const&, std::optional<unsigned int>)
E        --- tt::tt_metal::Allocator::allocate_buffer(tt::tt_metal::v0::Buffer*)
E        --- tt::tt_metal::v0::Buffer::allocate_impl()
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0x118b0f) [0x7f82912b1b0f]
E        --- tt::tt_metal::v0::Buffer::create(tt::tt_metal::v0::IDevice*, unsigned long, unsigned long, tt::tt_metal::BufferType, tt::tt_metal::TensorMemoryLayout, std::optional<tt::tt_metal::ShardSpecBuffer> const&, std::optional<bool>, std::optional<tt::tt_metal::SubDeviceId>)
E        --- tt::tt_metal::tensor_impl::allocate_buffer_on_device(tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&)
E        --- std::shared_ptr<tt::tt_metal::v0::Buffer> tt::tt_metal::tensor_impl::initialize_data_on_device<float, tt::tt_metal::borrowed_buffer::Buffer>(tt::tt_metal::borrowed_buffer::Buffer<float>&, tt::tt_metal::v0::IDevice*, tt::tt_metal::TensorSpec const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1ea91df) [0x7f82933ff1df]
E        --- tt::tt_metal::Tensor tt::tt_metal::tensor_impl::to_device<float>(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(_ZN2tt8tt_metal11tensor_impl8dispatchIZNS1_17to_device_wrapperIJRNS0_6TensorERPNS0_2v07IDeviceERKNS0_12MemoryConfigERhEEEDaDpOT_EUlTyDpOT0_E_JS5_S9_SC_SD_EEEDaNS0_8DataTypeEOT_SJ_+0x6d) [0x7f8293405bfd]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x1eaf8f1) [0x7f82934058f1]
E        --- tt::tt_metal::tensor_ops::tensor_to(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char)
E        --- tt::tt_metal::Tensor::to(tt::tt_metal::v0::IDevice*, tt::tt_metal::MemoryConfig const&, unsigned char) const
E        --- ttnn::operations::core::to_device(tt::tt_metal::Tensor const&, tt::tt_metal::v0::IDevice*, std::optional<tt::tt_metal::MemoryConfig> const&, unsigned char)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x1bf89e) [0x7f8294f6489e]
E        --- tt::runtime::ttnn::LayoutConverter::handleHostInputLayoutNoTypecast(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertHostTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::LayoutConverter::convertTensorLayout(tt::tt_metal::Tensor const&, std::optional<std::variant<std::reference_wrapper<tt::tt_metal::v0::IDevice>, std::reference_wrapper<tt::tt_metal::distributed::MeshDevice> > >)
E        --- tt::runtime::ttnn::toLayout(tt::runtime::Tensor, tt::runtime::Device, tt::runtime::Layout)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x3177f0) [0x7f82a29467f0]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x31773e) [0x7f82a294673e]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0xe0d05) [0x7f82a270fd05]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18ab32) [0x5587f7c49b32]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x59c7) [0x5587f7c39a97]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x5587f7c3f574]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x5587f7c58172]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x8af) [0x5587f7c3497f]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5587f7c5810b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6c0) [0x5587f7c34790]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x1984d1) [0x5587f7c574d1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5632) [0x5587f7c39702]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5587f7c3f61d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5587f7c5462c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29d464) [0x5587f7d5c464]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5587f7c4039b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x68ce) [0x5587f7c3a99e]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2b60) [0x5587f7c36c30]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5587f7c4a38c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18db) [0x5587f7c359ab]

/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/compiled_graph_state.py:253: RuntimeError