diff --git a/forge/test/models/pytorch/multimodal/vilt/test_vilt.py b/forge/test/models/pytorch/multimodal/vilt/test_vilt.py index 798eee5a1..3996b9d5a 100644 --- a/forge/test/models/pytorch/multimodal/vilt/test_vilt.py +++ b/forge/test/models/pytorch/multimodal/vilt/test_vilt.py @@ -56,6 +56,7 @@ def generate_model_vilt_question_answering_hf_pytorch(variant): @pytest.mark.nightly +@pytest.mark.push @pytest.mark.parametrize("variant", variants, ids=variants) def test_vilt_question_answering_hf_pytorch(record_forge_property, variant): # Build Module Name diff --git a/forge/test/models/pytorch/vision/dla/test_dla.py b/forge/test/models/pytorch/vision/dla/test_dla.py index 26a901b9a..dd8095ca8 100644 --- a/forge/test/models/pytorch/vision/dla/test_dla.py +++ b/forge/test/models/pytorch/vision/dla/test_dla.py @@ -2,44 +2,29 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -import requests -import torchvision.transforms as transforms -from PIL import Image import forge from forge.verify.verify import verify -from test.models.pytorch.vision.dla.utils.dla_model import ( - dla34, - dla46_c, - dla46x_c, - dla60, - dla60x, - dla60x_c, - dla102, - dla102x, - dla102x2, - dla169, -) +from test.models.pytorch.vision.dla.utils.utils import load_dla_model, post_processing from test.models.utils import Framework, Source, Task, build_module_name -variants_func = { - "dla34": dla34, - "dla46_c": dla46_c, - "dla46x_c": dla46x_c, - "dla60x_c": dla60x_c, - "dla60": dla60, - "dla60x": dla60x, - "dla102": dla102, - "dla102x": dla102x, - "dla102x2": dla102x2, - "dla169": dla169, -} -variants = list(variants_func.keys()) +variants = [ + pytest.param("dla34", marks=[pytest.mark.push]), + "dla46_c", + "dla46x_c", + "dla60", + "dla60x", + "dla60x_c", + "dla102", + "dla102x", + "dla102x2", + "dla169", +] @pytest.mark.nightly -@pytest.mark.parametrize("variant", variants, ids=variants) +@pytest.mark.parametrize("variant", variants) def test_dla_pytorch(record_forge_property, variant): if variant != "dla34": pytest.skip("Skipping due to the current CI/CD pipeline limitations") @@ -52,30 +37,17 @@ def test_dla_pytorch(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) - func = variants_func[variant] - - # Load data sample - url = "https://images.rawpixel.com/image_1300/cHJpdmF0ZS9sci9pbWFnZXMvd2Vic2l0ZS8yMDIyLTA1L3BkMTA2LTA0Ny1jaGltXzEuanBn.jpg" - image = Image.open(requests.get(url, stream=True).raw) - - # Preprocessing - transform = transforms.Compose( - [ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - ] - ) - img_tensor = transform(image).unsqueeze(0) - - framework_model = func(pretrained="imagenet") - framework_model.eval() - - inputs = [img_tensor] + # Load the model and prepare input data + framework_model, inputs = load_dla_model(variant) # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) # Model Verification verify(inputs, framework_model, compiled_model) + + # Inference + output = compiled_model(*inputs) + + # post processing + post_processing(output) diff --git a/forge/test/models/pytorch/vision/dla/utils/utils.py b/forge/test/models/pytorch/vision/dla/utils/utils.py new file mode 100644 index 000000000..ee2769a33 --- /dev/null +++ b/forge/test/models/pytorch/vision/dla/utils/utils.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: (c) 2025 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 +import os +import urllib + +import requests +import torch +import torchvision.transforms as transforms +from PIL import Image + +from test.models.pytorch.vision.dla.utils import dla_model + + +def load_dla_model(variant): + + func = getattr(dla_model, variant) + + # Load data sample + url = "https://images.rawpixel.com/image_1300/cHJpdmF0ZS9sci9pbWFnZXMvd2Vic2l0ZS8yMDIyLTA1L3BkMTA2LTA0Ny1jaGltXzEuanBn.jpg" + image = Image.open(requests.get(url, stream=True).raw) + + # Preprocessing + transform = transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + ) + img_tensor = transform(image).unsqueeze(0) + + framework_model = func(pretrained="imagenet") + framework_model.eval() + + inputs = [img_tensor] + + return framework_model, inputs + + +url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt" + + +def post_processing(output, top_k=5): + + probabilities = torch.nn.functional.softmax(output[0][0], dim=0) + urllib.request.urlretrieve(url, "imagenet_classes.txt") + + with open("imagenet_classes.txt", "r") as f: + categories = [s.strip() for s in f.readlines()] + topk_prob, topk_catid = torch.topk(probabilities, top_k) + for i in range(topk_prob.size(0)): + print(categories[topk_catid[i]], topk_prob[i].item()) + + # Cleanup + os.remove("imagenet_classes.txt") diff --git a/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py b/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py index 4581bb3cb..87888a2d7 100644 --- a/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py +++ b/forge/test/models/pytorch/vision/ghostnet/test_ghostnet.py @@ -1,23 +1,22 @@ # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC # SPDX-License-Identifier: Apache-2.0 -import urllib import pytest -import timm -from PIL import Image -from timm.data import resolve_data_config -from timm.data.transforms_factory import create_transform import forge from forge.verify.verify import verify +from test.models.pytorch.vision.ghostnet.utils.utils import ( + load_ghostnet_model, + post_processing, +) from test.models.utils import Framework, Source, Task, build_module_name -from test.utils import download_model variants = ["ghostnet_100"] +@pytest.mark.push @pytest.mark.nightly @pytest.mark.parametrize("variant", variants, ids=variants) def test_ghostnet_timm(record_forge_property, variant): @@ -33,25 +32,17 @@ def test_ghostnet_timm(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) - # STEP 2: Create Forge module from PyTorch model - framework_model = download_model(timm.create_model, variant, pretrained=True) - framework_model.eval() - - # STEP 3: Prepare input - url, filename = ( - "https://github.com/pytorch/hub/raw/master/images/dog.jpg", - "dog.jpg", - ) - urllib.request.urlretrieve(url, filename) - img = Image.open(filename) - data_config = resolve_data_config({}, model=framework_model) - transforms = create_transform(**data_config, is_training=False) - img_tensor = transforms(img).unsqueeze(0) - - inputs = [img_tensor] + # Load the model and prepare input data + framework_model, inputs = load_ghostnet_model(variant) # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) # Model Verification verify(inputs, framework_model, compiled_model) + + # Inference + output = compiled_model(*inputs) + + # Post processing + post_processing(output) diff --git a/forge/test/models/pytorch/vision/ghostnet/utils/__init__.py b/forge/test/models/pytorch/vision/ghostnet/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/forge/test/models/pytorch/vision/ghostnet/utils/utils.py b/forge/test/models/pytorch/vision/ghostnet/utils/utils.py new file mode 100644 index 000000000..13a8560e4 --- /dev/null +++ b/forge/test/models/pytorch/vision/ghostnet/utils/utils.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: (c) 2025 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 +import os +import urllib + +import timm +import torch +from PIL import Image +from timm.data import resolve_data_config +from timm.data.transforms_factory import create_transform + +from test.utils import download_model + + +def load_ghostnet_model(variant): + + # Create Forge module from PyTorch model + framework_model = download_model(timm.create_model, variant, pretrained=True) + framework_model.eval() + + # Prepare input + url, filename = ( + "https://github.com/pytorch/hub/raw/master/images/dog.jpg", + "dog.jpg", + ) + urllib.request.urlretrieve(url, filename) + img = Image.open(filename) + data_config = resolve_data_config({}, model=framework_model) + transforms = create_transform(**data_config, is_training=False) + img_tensor = transforms(img).unsqueeze(0) + + return framework_model, [img_tensor] + + +url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt" + + +def post_processing(output, top_k=5): + + probabilities = torch.nn.functional.softmax(output[0][0], dim=0) + urllib.request.urlretrieve(url, "imagenet_classes.txt") + + with open("imagenet_classes.txt", "r") as f: + categories = [s.strip() for s in f.readlines()] + topk_prob, topk_catid = torch.topk(probabilities, top_k) + for i in range(topk_prob.size(0)): + print(categories[topk_catid[i]], topk_prob[i].item()) + + # Cleanup + os.remove("imagenet_classes.txt") + os.remove("dog.jpg") diff --git a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py index adf72c01e..933347c14 100644 --- a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py +++ b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v1.py @@ -3,29 +3,22 @@ # SPDX-License-Identifier: Apache-2.0 import pytest import requests -import torch from PIL import Image from transformers import AutoImageProcessor, AutoModelForImageClassification import forge from forge.verify.verify import verify -from test.models.pytorch.vision.mobilenet.utils.mobilenet_v1 import MobileNetV1 +from test.models.pytorch.vision.mobilenet.utils.utils import ( + load_mobilenet_model, + post_processing, +) from test.models.utils import Framework, Source, Task, build_module_name from test.utils import download_model -def generate_model_mobilenetV1_base_custom_pytorch(): - # Create Forge module from PyTorch model - model = MobileNetV1(9) - - input_shape = (1, 3, 64, 64) - image_tensor = torch.rand(*input_shape) - - return model, [image_tensor], {} - - @pytest.mark.nightly +@pytest.mark.push def test_mobilenetv1_basic(record_forge_property): # Build Module Name module_name = build_module_name( @@ -39,14 +32,21 @@ def test_mobilenetv1_basic(record_forge_property): # Record Forge Property record_forge_property("model_name", module_name) - framework_model, inputs, _ = generate_model_mobilenetV1_base_custom_pytorch() + # Load the model and prepare input data + framework_model, inputs = load_mobilenet_model("mobilenet_v1") # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) - # Model Verification + # Model Verification verify(inputs, framework_model, compiled_model) + # Inference + output = compiled_model(*inputs) + + # Post processing + post_processing(output) + def generate_model_mobilenetv1_imgcls_hf_pytorch(variant): # Create Forge module from PyTorch model diff --git a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py index a05b8db50..c1a616dc6 100644 --- a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py +++ b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v2.py @@ -20,25 +20,16 @@ import forge from forge.verify.verify import verify +from test.models.pytorch.vision.mobilenet.utils.utils import ( + load_mobilenet_model, + post_processing, +) from test.models.utils import Framework, Source, Task, build_module_name from test.utils import download_model -def generate_model_mobilenetV2_imgcls_torchhub_pytorch(): - model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", "mobilenet_v2", pretrained=True) - - # Image preprocessing - url = "http://images.cocodataset.org/val2017/000000039769.jpg" - image = Image.open(requests.get(url, stream=True).raw) - # TODO : Choose image preprocessor from torchvision, - # to make a compatible postprocessing of the predicted class - preprocessor = download_model(AutoImageProcessor.from_pretrained, "google/mobilenet_v2_1.0_224") - image_tensor = preprocessor(images=image, return_tensors="pt").pixel_values - - return model, [image_tensor], {} - - @pytest.mark.nightly +@pytest.mark.push def test_mobilenetv2_basic(record_forge_property): # Build Module Name module_name = build_module_name( @@ -52,7 +43,8 @@ def test_mobilenetv2_basic(record_forge_property): # Record Forge Property record_forge_property("model_name", module_name) - framework_model, inputs, _ = generate_model_mobilenetV2_imgcls_torchhub_pytorch() + # Load the model and prepare input data + framework_model, inputs = load_mobilenet_model("mobilenet_v2") # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) @@ -60,6 +52,12 @@ def test_mobilenetv2_basic(record_forge_property): # Model Verification verify(inputs, framework_model, compiled_model) + # Inference + output = compiled_model(*inputs) + + # Post processing + post_processing(output) + def generate_model_mobilenetV2I96_imgcls_hf_pytorch(variant): preprocessor = download_model(AutoImageProcessor.from_pretrained, variant) diff --git a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py index e1ae4db2e..ff478a960 100644 --- a/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py +++ b/forge/test/models/pytorch/vision/mobilenet/test_mobilenet_v3.py @@ -4,40 +4,31 @@ import urllib import pytest -import requests import timm import torch from loguru import logger from PIL import Image from timm.data import resolve_data_config from timm.data.transforms_factory import create_transform -from transformers import AutoImageProcessor import forge from forge.verify.verify import verify +from test.models.pytorch.vision.mobilenet.utils.utils import ( + load_mobilenet_model, + post_processing, +) from test.models.utils import Framework, Source, Task, build_module_name from test.utils import download_model - -def generate_model_mobilenetV3_imgcls_torchhub_pytorch(variant): - model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", variant, pretrained=True) - - # Run inference on Tenstorrent device - url = "http://images.cocodataset.org/val2017/000000039769.jpg" - image = Image.open(requests.get(url, stream=True).raw) - # TODO : Choose image preprocessor from torchvision, to make a compatible postprocessing of the predicted class - preprocessor = AutoImageProcessor.from_pretrained("google/mobilenet_v2_1.0_224") - image_tensor = preprocessor(images=image, return_tensors="pt").pixel_values - - return model, [image_tensor], {} - - -variants = ["mobilenet_v3_large", "mobilenet_v3_small"] +variants = [ + pytest.param("mobilenet_v3_large", marks=[pytest.mark.push]), + pytest.param("mobilenet_v3_small"), +] @pytest.mark.nightly -@pytest.mark.parametrize("variant", variants, ids=variants) +@pytest.mark.parametrize("variant", variants) def test_mobilenetv3_basic(record_forge_property, variant): if variant != "mobilenet_v3_large": pytest.skip("Skipping due to the current CI/CD pipeline limitations") @@ -54,9 +45,8 @@ def test_mobilenetv3_basic(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) - framework_model, inputs, _ = generate_model_mobilenetV3_imgcls_torchhub_pytorch( - variant, - ) + # Load the model and prepare input data + framework_model, inputs = load_mobilenet_model(variant) # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) @@ -64,6 +54,12 @@ def test_mobilenetv3_basic(record_forge_property, variant): # Model Verification verify(inputs, framework_model, compiled_model) + # Inference + output = compiled_model(*inputs) + + # Post processing + post_processing(output) + def generate_model_mobilenetV3_imgcls_timm_pytorch(variant): # Both options are good diff --git a/forge/test/models/pytorch/vision/mobilenet/utils/utils.py b/forge/test/models/pytorch/vision/mobilenet/utils/utils.py new file mode 100644 index 000000000..02ece294a --- /dev/null +++ b/forge/test/models/pytorch/vision/mobilenet/utils/utils.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: (c) 2025 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 +import os +import urllib + +import torch +from PIL import Image +from torchvision import transforms + +from test.models.pytorch.vision.mobilenet.utils.mobilenet_v1 import MobileNetV1 +from test.utils import download_model + + +def load_mobilenet_model(model_name): + + # Create model + if model_name == "mobilenet_v1": + model = MobileNetV1(9) + else: + model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", model_name, pretrained=True) + + model.eval() + + # Load data sample + url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg") + urllib.request.urlretrieve(url, filename) + + # Preprocessing + input_image = Image.open(filename) + preprocess = transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + ) + input_tensor = preprocess(input_image) + input_batch = input_tensor.unsqueeze(0) + + return model, [input_batch] + + +url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt" + + +def post_processing(output, top_k=5): + + probabilities = torch.nn.functional.softmax(output[0][0], dim=0) + urllib.request.urlretrieve(url, "imagenet_classes.txt") + + with open("imagenet_classes.txt", "r") as f: + categories = [s.strip() for s in f.readlines()] + topk_prob, topk_catid = torch.topk(probabilities, top_k) + for i in range(topk_prob.size(0)): + print(categories[topk_catid[i]], topk_prob[i].item()) + + # Cleanup + os.remove("imagenet_classes.txt") + os.remove("dog.jpg") diff --git a/forge/test/models/pytorch/vision/resnext/test_resnext.py b/forge/test/models/pytorch/vision/resnext/test_resnext.py index 638857f22..092471104 100644 --- a/forge/test/models/pytorch/vision/resnext/test_resnext.py +++ b/forge/test/models/pytorch/vision/resnext/test_resnext.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC # SPDX-License-Identifier: Apache-2.0 + import pytest import torch from pytorchcv.model_provider import get_model as ptcv_get_model @@ -8,11 +9,15 @@ import forge from forge.verify.verify import verify -from test.models.pytorch.vision.resnext.utils.image_utils import get_image_tensor +from test.models.pytorch.vision.resnext.utils.utils import ( + load_resnext_model, + post_processing, +) from test.models.utils import Framework, Source, Task, build_module_name from test.utils import download_model +@pytest.mark.push @pytest.mark.nightly @pytest.mark.parametrize("variant", ["resnext50_32x4d"]) def test_resnext_50_torchhub_pytorch(record_forge_property, variant): @@ -28,12 +33,8 @@ def test_resnext_50_torchhub_pytorch(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) - # STEP 2: Create Forge module from PyTorch model - framework_model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", variant, pretrained=True) - framework_model.eval() - - input_batch = get_image_tensor() - inputs = [input_batch] + # Load the model and prepare input data + framework_model, inputs = load_resnext_model(variant) # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) @@ -41,7 +42,14 @@ def test_resnext_50_torchhub_pytorch(record_forge_property, variant): # Model Verification verify(inputs, framework_model, compiled_model) + # Inference + output = compiled_model(*inputs) + # Post processing + post_processing(output) + + +@pytest.mark.push @pytest.mark.nightly @pytest.mark.parametrize("variant", ["resnext101_32x8d"]) def test_resnext_101_torchhub_pytorch(record_forge_property, variant): @@ -61,8 +69,8 @@ def test_resnext_101_torchhub_pytorch(record_forge_property, variant): framework_model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", variant, pretrained=True) framework_model.eval() - input_batch = get_image_tensor() - inputs = [input_batch] + # Load the model and prepare input data + framework_model, inputs = load_resnext_model(variant) # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) @@ -70,6 +78,12 @@ def test_resnext_101_torchhub_pytorch(record_forge_property, variant): # Model Verification verify(inputs, framework_model, compiled_model) + # Inference + output = compiled_model(*inputs) + + # Post processing + post_processing(output) + @pytest.mark.nightly @pytest.mark.parametrize("variant", ["resnext101_32x8d_wsl"]) diff --git a/forge/test/models/pytorch/vision/resnext/utils/image_utils.py b/forge/test/models/pytorch/vision/resnext/utils/image_utils.py deleted file mode 100644 index 8a0f9dd5e..000000000 --- a/forge/test/models/pytorch/vision/resnext/utils/image_utils.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC - -# SPDX-License-Identifier: Apache-2.0 -import torch -from loguru import logger -from PIL import Image -from torchvision import transforms - - -def get_image_tensor(): - # Image processing - try: - torch.hub.download_url_to_file("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg") - input_image = Image.open("dog.jpg") - preprocess = transforms.Compose( - [ - transforms.Resize(224), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - ] - ) - input_tensor = preprocess(input_image) - input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model - except: - logger.warning( - "Failed to download the image file, replacing input with random tensor. Please check if the URL is up to date" - ) - input_batch = torch.rand(1, 3, 224, 224) - return input_batch diff --git a/forge/test/models/pytorch/vision/resnext/utils/utils.py b/forge/test/models/pytorch/vision/resnext/utils/utils.py new file mode 100644 index 000000000..76b954c87 --- /dev/null +++ b/forge/test/models/pytorch/vision/resnext/utils/utils.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC + +# SPDX-License-Identifier: Apache-2.0 +import os +import urllib + +import torch +from PIL import Image +from torchvision import transforms + +from test.utils import download_model + + +def load_resnext_model(model_name): + + # Create model + model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", model_name, pretrained=True) + model.eval() + + # Load data sample + url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg") + urllib.request.urlretrieve(url, filename) + + # Preprocessing + input_image = Image.open(filename) + preprocess = transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + ) + input_tensor = preprocess(input_image) + input_batch = input_tensor.unsqueeze(0) + + return model, [input_batch] + + +url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt" + + +def post_processing(output, top_k=5): + + probabilities = torch.nn.functional.softmax(output[0][0], dim=0) + urllib.request.urlretrieve(url, "imagenet_classes.txt") + + with open("imagenet_classes.txt", "r") as f: + categories = [s.strip() for s in f.readlines()] + topk_prob, topk_catid = torch.topk(probabilities, top_k) + for i in range(topk_prob.size(0)): + print(categories[topk_catid[i]], topk_prob[i].item()) + + # Cleanup + os.remove("imagenet_classes.txt") + os.remove("dog.jpg") diff --git a/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py b/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py index 22bd68168..5a3499e40 100644 --- a/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py +++ b/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py @@ -5,47 +5,27 @@ import pytest import timm -import torch -from PIL import Image from timm.data import resolve_data_config from timm.data.transforms_factory import create_transform -from torchvision import transforms import forge from forge.verify.verify import verify +from test.models.pytorch.vision.wideresnet.utils.utils import ( + generate_model_wideresnet_imgcls_pytorch, + post_processing, +) from test.models.utils import Framework, Source, Task, build_module_name from test.utils import download_model - -def generate_model_wideresnet_imgcls_pytorch(variant): - # STEP 2: Create Forge module from PyTorch model - framework_model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", variant, pretrained=True) - framework_model.eval() - - # STEP 3: Prepare input - url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg") - urllib.request.urlretrieve(url, filename) - input_image = Image.open(filename) - preprocess = transforms.Compose( - [ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), - ] - ) - input_tensor = preprocess(input_image) - img_tensor = input_tensor.unsqueeze(0) - - return framework_model, [img_tensor] - - -variants = ["wide_resnet50_2", "wide_resnet101_2"] +variants = [ + pytest.param("wide_resnet50_2", marks=[pytest.mark.push]), + pytest.param("wide_resnet101_2"), +] @pytest.mark.nightly -@pytest.mark.parametrize("variant", variants, ids=variants) +@pytest.mark.parametrize("variant", variants) def test_wideresnet_pytorch(record_forge_property, variant): if variant != "wide_resnet50_2": pytest.skip("Skipping due to the current CI/CD pipeline limitations") @@ -62,6 +42,7 @@ def test_wideresnet_pytorch(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) + # Load the model and prepare input data (framework_model, inputs) = generate_model_wideresnet_imgcls_pytorch(variant) # Forge compile framework model @@ -70,6 +51,12 @@ def test_wideresnet_pytorch(record_forge_property, variant): # Model Verification verify(inputs, framework_model, compiled_model) + # Inference + output = compiled_model(*inputs) + + # Post processing + post_processing(output) + def generate_model_wideresnet_imgcls_timm(variant): # STEP 2: Create Forge module from PyTorch model diff --git a/forge/test/models/pytorch/vision/wideresnet/utils/__init__.py b/forge/test/models/pytorch/vision/wideresnet/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/forge/test/models/pytorch/vision/wideresnet/utils/utils.py b/forge/test/models/pytorch/vision/wideresnet/utils/utils.py new file mode 100644 index 000000000..7e991c9bd --- /dev/null +++ b/forge/test/models/pytorch/vision/wideresnet/utils/utils.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: (c) 2025 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 +import os +import urllib + +import torch +from PIL import Image +from torchvision import transforms + +from test.utils import download_model + + +def generate_model_wideresnet_imgcls_pytorch(variant): + # STEP 2: Create Forge module from PyTorch model + framework_model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", variant, pretrained=True) + framework_model.eval() + + # STEP 3: Prepare input + url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg") + urllib.request.urlretrieve(url, filename) + input_image = Image.open(filename) + preprocess = transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + ) + input_tensor = preprocess(input_image) + img_tensor = input_tensor.unsqueeze(0) + + return framework_model, [img_tensor] + + +url = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt" + + +def post_processing(output, top_k=5): + + probabilities = torch.nn.functional.softmax(output[0][0], dim=0) + urllib.request.urlretrieve(url, "imagenet_classes.txt") + + with open("imagenet_classes.txt", "r") as f: + categories = [s.strip() for s in f.readlines()] + topk_prob, topk_catid = torch.topk(probabilities, top_k) + for i in range(topk_prob.size(0)): + print(categories[topk_catid[i]], topk_prob[i].item()) + + # Cleanup + os.remove("imagenet_classes.txt") + os.remove("dog.jpg")