#0: Merge branch 'main' into llama32-vision

.github/actions/docker-run/action.yml

@@ -38,7 +38,7 @@ runs:
       uses: ./.github/actions/generate-docker-tag
       with:
         image: ${{ inputs.docker_os_arch }}
-    - name: Set 
+    - name: Set
       shell: bash
       run: |
         echo "RUNNER_UID=$(id -u)" >> $GITHUB_ENV
@@ -63,8 +63,10 @@ runs:
         # The most important option below is `--rm`. Otherwise, the machines will fill up with undeleted containers.
         # The mounting of /etc/passwd, /etc/shadow, and /etc/bashrc is required in order for the correct file permissions
         # for newly created files.
-        # Passing HOME variable is necessary to avoid Python lib installation into /home/ubuntu/.local folder which 
+        # Passing HOME variable is necessary to avoid Python lib installation into /home/ubuntu/.local folder which
         # may not be writable by the RUNNER_UID user.
+        # --log-driver none: Do not save logs to disk as we're printing them to GitHub
+        # and it takes up space
         options: |
           -u ${{ env.RUNNER_UID }}:${{ env.RUNNER_GID }}
           --rm
@@ -73,6 +75,7 @@ runs:
           -v /etc/bashrc:/etc/bashrc:ro
           -v ${{ github.workspace }}:${{ github.workspace }}
           --net=host
+          --log-driver none
           ${{ inputs.docker_opts }}
           -e LOGURU_LEVEL=${{ env.LOGURU_LEVEL }}
           -e PYTHONPATH=${{ github.workspace }}

.github/workflows/all-static-checks.yaml

@@ -8,6 +8,38 @@ on:
       - "main"
 
 jobs:
+  pre-commit:
+    name: Run Pre-commit Hooks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Fetch all history so 'origin/main' is available
+          fetch-refs: true  # Ensure all refs are fetched
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Run Pre-commit
+        uses: pre-commit/action@v3.0.1
+        with:
+          extra_args: |
+            --from-ref ${{ github.event_name == 'pull_request' && format('refs/remotes/origin/{0}', github.event.pull_request.base.ref) || 'HEAD^' }} \
+            --to-ref HEAD
+        continue-on-error: false
+  check-black:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Do Nothing
+        run: echo "Black is covered by pre-commit. This is a placeholder to be removed after updating branch restrictions."
+
+
   check-spdx-licenses:
     runs-on: ubuntu-latest
     steps:
@@ -27,11 +59,6 @@ jobs:
       - uses: actions/checkout@v4
       - name: Check kernel count in base metal is less than maximum
         run: if (( $(find tt_metal/kernels/ -type f | wc -l) > 7 )); then exit 1; fi
-  check-black:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: psf/black@23.10.1
   check-doc:
     runs-on: ubuntu-latest
     steps:

.github/workflows/build-artifact.yaml

@@ -127,6 +127,8 @@ jobs:
             -e ARCH_NAME=${{ matrix.arch }}
             -w ${{ github.workspace }}
           run: |
+            set -eu # basic shell hygiene
+
             # /tmp is a tmpfs; more efficient than persisted storage
             mkdir -p /tmp/ccache
             export CCACHE_TEMPDIR=/tmp/ccache

.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml

@@ -54,27 +54,6 @@ jobs:
               cmd: tests/scripts/single_card/nightly/run_gs_only.sh,
               timeout: 40
             },
-            {
-              name: "API tests GS",
-              arch: grayskull,
-              runs-on: ["cloud-virtual-machine", "E150", "in-service"],
-              cmd: ./tests/scripts/run_tests.sh --tt-arch grayskull --pipeline-type frequent_api --dispatch-mode fast,
-              timeout: 10
-            },
-            {
-              name: "API tests N300 WH B0",
-              arch: wormhole_b0,
-              runs-on: ["cloud-virtual-machine", "N300", "in-service"],
-              cmd: ./tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type frequent_api --dispatch-mode fast,
-              timeout: 10
-            },
-            {
-              name: "API tests N150 WH B0",
-              arch: wormhole_b0,
-              runs-on: ["cloud-virtual-machine", "N150", "in-service"],
-              cmd: ./tests/scripts/run_tests.sh --tt-arch wormhole_b0 --pipeline-type frequent_api --dispatch-mode fast,
-              timeout: 10
-            },
             {
               name: "[Unstable] N150 models",
               arch: wormhole_b0,

.github/workflows/ttnn-run-sweeps.yaml

@@ -145,6 +145,15 @@ on:
           - eltwise.unary_backward.hardsigmoid_bw.hardsigmoid_bw
           - eltwise.unary_backward.lgamma_bw.lgamma_bw
           - eltwise.unary_backward.multigammaln_bw.multigammaln_bw
+          - eltwise.unary_backward.leaky_relu_bw.leaky_relu_bw
+          - eltwise.unary_backward.elu_bw.elu_bw
+          - eltwise.unary_backward.celu_bw.celu_bw
+          - eltwise.unary_backward.selu_bw.selu_bw
+          - eltwise.unary_backward.silu_bw.silu_bw
+          - eltwise.unary_backward.floor_bw.floor_bw
+          - eltwise.unary_backward.tanhshrink_bw.tanhshrink_bw
+          - eltwise.unary_backward.hardswish_bw.hardswish_bw
+          - eltwise.unary_backward.rpow_bw.rpow_bw
           - eltwise.unary.lgamma
           - eltwise.unary.logit
           - eltwise.unary.mish
@@ -211,6 +220,8 @@ on:
           - eltwise.binary_backward.subalpha_bw.subalpha_bw
           - eltwise.binary_backward.xlogy_bw.xlogy_bw
           - eltwise.binary_backward.hypot_bw.hypot_bw
+          - eltwise.binary_backward.rsub_bw.rsub_bw
+          - eltwise.binary_backward.squared_difference_bw.squared_difference_bw
           - eltwise.composite.binary.addalpha.addalpha
           - eltwise.composite.binary.subalpha.subalpha
           - eltwise.composite.binary.minimum.minimum
@@ -228,6 +239,7 @@ on:
           - eltwise.ternary.where.where_pytorch2
           - reduction.topk.topk
           - reduction.argmax.argmax
+          - embedding.embedding
           - matmul.full.matmul_default_block_sharded
           - matmul.full.matmul_default_height_sharded
           - matmul.full.matmul_default_interleaved
@@ -257,6 +269,7 @@ on:
           - data_movement.index_select.index_select_pytorch2
           - data_movement.split.split_with_sizes_pytorch2
           - data_movement.repeat.repeat
+          - data_movement.nonzero.nonzero
           - conv2d.full.conv2d_misc
           - conv2d.full.conv2d_sharding
           - conv2d.full.conv2d_sliding_window

README.md

@@ -53,7 +53,6 @@
 | [ViT](./models/demos/grayskull/vit)                                         | 9     | [e150](https://tenstorrent.com/hardware/grayskull)       | 1,360   | 2,000      |             |
 | [ViT](./models/demos/wormhole/vit)                                          | 8     | [n150](https://tenstorrent.com/hardware/wormhole)        | 912     | 1,600      |             |
 | [Stable Diffusion 1.4 (512x512)](./models/demos/wormhole/stable_diffusion)  | 1     | [n150](https://tenstorrent.com/hardware/wormhole)        | 0.167   | 0.3        |             |
-| [U-Net](./models/experimental/functional_unet)                              | 2     | [n150](https://tenstorrent.com/hardware/wormhole)        | 530     | 1000       | [v0.53.0-rc22](https://github.com/tenstorrent/tt-metal/tree/v0.53.0-rc22) |
 
 
 ## NLPs

models/experimental/yolov4/demo/demo.py

@@ -13,7 +13,8 @@
 
 from models.experimental.yolov4.reference.yolov4 import Yolov4
 from models.experimental.yolov4.ttnn.yolov4 import TtYOLOv4
-
+from models.experimental.yolov4.ttnn.weight_parameter_update import update_weight_parameters
+from collections import OrderedDict
 import ttnn
 from models.utility_functions import skip_for_grayskull
 
@@ -418,11 +419,7 @@ def do_detect(model, img, conf_thresh, nms_thresh, n_classes, device=None, class
         if not is_torch_model:
             input_shape = img.shape
             input_tensor = torch.permute(img, (0, 2, 3, 1))
-
-            input_tensor = input_tensor.reshape(
-                input_tensor.shape[0], 1, input_tensor.shape[1] * input_tensor.shape[2], input_tensor.shape[3]
-            )
-            input_tensor = ttnn.from_torch(input_tensor, device=device)
+            input_tensor = ttnn.from_torch(input_tensor, ttnn.bfloat16)
             img = input_tensor
             t1 = time.time()
 
@@ -534,33 +531,44 @@ def do_detect(model, img, conf_thresh, nms_thresh, n_classes, device=None, class
 
 @skip_for_grayskull()
 @pytest.mark.parametrize("device_params", [{"l1_small_size": 16384}], indirect=True)
-def test_yolov4_model(device, model_location_generator, reset_seeds, input_path):
+@pytest.mark.parametrize(
+    "use_pretrained_weight",
+    [True, False],
+    ids=[
+        "pretrained_weight_true",
+        "pretrained_weight_false",
+    ],
+)
+def test_yolov4_model(device, model_location_generator, reset_seeds, input_path, use_pretrained_weight):
     model_path = model_location_generator("models", model_subdir="Yolo")
-    if model_path == "models":
-        if not os.path.exists("tests/ttnn/integration_tests/yolov4/yolov4.pth"):  # check if yolov4.th is availble
-            os.system(
-                "tests/ttnn/integration_tests/yolov4/yolov4_weights_download.sh"
-            )  # execute the yolov4_weights_download.sh file
-
-        weights_pth = "tests/ttnn/integration_tests/yolov4/yolov4.pth"
-    else:
-        weights_pth = str(model_path / "yolov4.pth")
-
-    ttnn_model = TtYOLOv4(weights_pth)
-
-    torch_model = Yolov4()
+    if use_pretrained_weight:
+        if model_path == "models":
+            if not os.path.exists("tests/ttnn/integration_tests/yolov4/yolov4.pth"):  # check if yolov4.th is availble
+                os.system(
+                    "tests/ttnn/integration_tests/yolov4/yolov4_weights_download.sh"
+                )  # execute the yolov4_weights_download.sh file
+
+            weights_pth = "tests/ttnn/integration_tests/yolov4/yolov4.pth"
+        else:
+            weights_pth = str(model_path / "yolov4.pth")
 
-    new_state_dict = {}
-    ds_state_dict = {k: v for k, v in ttnn_model.torch_model.items()}
+        ttnn_model = TtYOLOv4(weights_pth)
+        torch_model = Yolov4()
+        new_state_dict = {}
+        ds_state_dict = {k: v for k, v in ttnn_model.torch_model.items()}
 
-    keys = [name for name, parameter in torch_model.state_dict().items()]
-    values = [parameter for name, parameter in ds_state_dict.items()]
+        keys = [name for name, parameter in torch_model.state_dict().items()]
+        values = [parameter for name, parameter in ds_state_dict.items()]
 
-    for i in range(len(keys)):
-        new_state_dict[keys[i]] = values[i]
+        for i in range(len(keys)):
+            new_state_dict[keys[i]] = values[i]
 
-    torch_model.load_state_dict(new_state_dict)
-    torch_model.eval()
+        torch_model.load_state_dict(new_state_dict)
+        torch_model.eval()
+    else:
+        torch_model = Yolov4.from_random_weights()
+        ttnn_weights = update_weight_parameters(OrderedDict(torch_model.state_dict()))
+        ttnn_model = TtYOLOv4(ttnn_weights)
 
     n_classes = 80
     namesfile = "models/experimental/yolov4/demo/coco.names"

models/experimental/yolov4/reference/yolov4.py

@@ -36,3 +36,16 @@ def forward(self, input: torch.Tensor):
         x4, x5, x6 = self.head(x20, x13, x6)
 
         return x4, x5, x6
+
+    @staticmethod
+    def from_random_weights():
+        model = Yolov4()
+        model.eval()
+
+        new_state_dict = {}
+        for name, parameter in model.state_dict().items():
+            if isinstance(parameter, torch.FloatTensor):
+                new_state_dict[name] = parameter
+
+        model.load_state_dict(new_state_dict)
+        return model

models/experimental/yolov4/ttnn/weight_parameter_update.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: © 2024 Tenstorrent Inc.
+
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from collections import OrderedDict
+
+
+def update_weigth_keys(key):
+    key = key.replace("downsample", "down")
+    key = key.replace("neck", "neek")
+    if ".res" in key:
+
+        def res_name_update(match):
+            chr = match.group(1)
+            num = int(match.group(2))
+            if num == 0 or num == 1:
+                return f".{chr}.0.conv.{num}."
+            if num == 3 or num == 4:
+                return f".{chr}.1.conv.{num-3}."
+
+        key = re.sub(r"\.res\.", r".resblock.", key)
+        key = re.sub(r"\.(\d+)\.(\d+)\.", res_name_update, key)
+        return key
+    if "neek" in key:
+
+        def neek_underscore_update_rule(match):
+            chr = match.group(1)
+            num1 = int(match.group(2))
+            num2 = int(match.group(3))
+            dict = {
+                (7, 2): 8,
+                (7, 3): 9,
+                (7, 4): 11,
+                (8, 2): 12,
+                (7, 5): 13,
+                (9, 2): 15,
+                (9, 3): 16,
+                (9, 4): 18,
+                (10, 2): 19,
+                (9, 5): 20,
+            }
+            if chr == "b":
+                return f".conv{dict[(num1, num2)]}.conv.1."
+            return f".conv{dict[(num1, num2)]}.conv.0."
+
+        def neck_rename_update(match):
+            chr = match.group(1)
+            num = int(match.group(2))
+            if num <= 7:
+                return f".conv{num}.conv.1." if chr == "b" else f".conv{num}.conv.0."
+            dict = {8: 10, 9: 14, 10: 17}
+            return f".conv{dict[num]}.conv.1." if chr == "b" else f".conv{dict[num]}.conv.0."
+
+        updated_name = re.sub(r"\.([a-z])(\d+)_(\d+)\.", neek_underscore_update_rule, key)
+        if key != updated_name:  # chk if name got updated
+            return updated_name
+        updated_name = re.sub(r"\.([a-z])(\d+)\.", neck_rename_update, key)
+        if key != updated_name:
+            return updated_name
+    key = re.sub(r"\.c(\d+)\.", r".conv\1.conv.0.", key)
+    key = re.sub(r"\.b(\d+)\.", r".conv\1.conv.1.", key)
+    return key
+
+
+def update_weight_parameters(model_weight):
+    ttnn_model_random_weight = OrderedDict()
+    for key, weight in model_weight.items():
+        updated_key = update_weigth_keys(key)
+        ttnn_model_random_weight[updated_key] = weight
+    return ttnn_model_random_weight

models/experimental/yolov4/ttnn/yolov4.py

@@ -25,7 +25,10 @@
 
 class TtYOLOv4:
     def __init__(self, path) -> None:
-        self.torch_model = torch.load(path)
+        if type(path) is str:
+            self.torch_model = torch.load(path)
+        else:
+            self.torch_model = path
         self.torch_keys = self.torch_model.keys()
         self.down1 = Down1(self)
         self.down2 = Down2(self)