ttnn-run-sweeps.yaml

name: "ttnn - Run sweeps"

on:
  workflow_dispatch:
    inputs:
      sweep_name:
        type: choice
        description: "Which sweep module to run?"
        required: true
        default: "ALL SWEEPS (Nightly)"
        options:
          - ALL SWEEPS (Nightly)
          - add
          - tilize
          - untilize
          - ccl.line_all_gather
          - ccl.all_gather_n300
          - ccl.all_gather_n300_focused
          - creation.zeros.zeros
          - creation.empty.empty
          - creation.zeros_like.zeros_like
          - eltwise.unary.abs.abs_pytorch2
          - eltwise.unary.relu.relu
          - eltwise.unary.relu.relu_pytorch2
          - eltwise.unary.gelu.gelu
          - eltwise.unary.gelu.gelu_pytorch2
          - eltwise.unary.hardsigmoid.hardsigmoid_pytorch2
          - eltwise.unary.leaky_relu.leaky_relu_pytorch2
          - eltwise.unary.abs.abs
          - eltwise.unary.cos.cos
          - eltwise.unary.cos.cos_pytorch2
          - eltwise.unary.sin.sin
          - eltwise.unary.sin.sin_pytorch2
          - eltwise.unary.tril.tril_pytorch2
          - eltwise.unary.clamp.clamp
          - eltwise.unary.clamp.clamp_pytorch2
          - eltwise.unary.clamp.clamp_min_pytorch2
          - eltwise.unary.clip.clip
          - eltwise.unary.cbrt.cbrt
          - eltwise.unary.rsub.rsub
          - eltwise.unary.rsub.rsub_pytorch2
          - eltwise.unary.rsqrt.rsqrt_pytorch2
          - eltwise.unary.rdiv.rdiv
          - eltwise.unary.frac.frac
          - eltwise.unary.ceil.ceil
          - eltwise.unary.ceil.ceil_pytorch2
          - eltwise.unary.trunc.trunc
          - eltwise.unary.floor.floor
          - eltwise.unary.floor.floor_pytorch2
          - eltwise.unary.clone.clone
          - eltwise.unary.elu.elu
          - eltwise.unary.elu.elu_pytorch2
          - eltwise.unary.erfc.erfc
          - eltwise.unary.exp.exp
          - eltwise.unary.exp.exp_pytorch2
          - eltwise.unary.exp2.exp2
          - eltwise.unary.expm1.expm1
          - eltwise.unary.tanh.tanh
          - eltwise.unary.tanh.tanh_pytorch2
          - eltwise.unary.atanh.atanh
          - eltwise.unary.atan.atan
          - eltwise.unary.sign.sign
          - eltwise.unary.rad2deg.rad2deg
          - eltwise.unary.deg2rad.deg2rad
          - eltwise.unary.relu6.relu6
          - eltwise.unary.log.log
          - eltwise.unary.log.log_pytorch2
          - eltwise.unary.log1p.log1p
          - eltwise.unary.log2.log2
          - eltwise.unary.log10.log10
          - eltwise.unary.bitwise.bitwise_and
          - eltwise.unary.bitwise.bitwise_left_shift
          - eltwise.unary.bitwise.bitwise_not
          - eltwise.unary.bitwise.bitwise_not_pytorch2
          - eltwise.unary.bitwise.bitwise_or
          - eltwise.unary.bitwise.bitwise_right_shift
          - eltwise.unary.bitwise.bitwise_xor
          - eltwise.unary.log_sigmoid.log_sigmoid
          - eltwise.unary.logical_not.logical_not_
          - eltwise.unary.logical_not.logical_not
          - eltwise.unary.logical_not.logical_not_output
          - eltwise.unary.logical_not.logical_not_pytorch2
          - eltwise.unary.neg.neg_pytorch2
          - eltwise.unary.erf.erf
          - eltwise.unary.erfinv.erfinv
          - eltwise.unary.i0.i0
          - eltwise.unary.silu.silu
          - eltwise.unary.silu.silu_pytorch2
          - eltwise.unary.glu.glu
          - eltwise.unary.geglu.geglu
          - eltwise.unary.swiglu.swiglu
          - eltwise.unary.sigmoid.sigmoid
          - eltwise.unary.sigmoid.sigmoid_pytorch2
          - eltwise.unary.sigmoid_accurate.sigmoid_accurate
          - eltwise.unary.tril.tril
          - eltwise.unary.tril.tril_sharded
          - eltwise.unary.triu.triu
          - eltwise.unary.triu.triu_sharded
          - eltwise.unary.normalize_hw.normalize_hw
          - eltwise.unary.normalize_global.normalize_global
          - eltwise.unary.heaviside.heaviside
          - eltwise.unary.heaviside.heaviside_sharded
          - eltwise.unary.hardtanh.hardtanh
          - eltwise.unary.hardtanh.hardtanh_sharded
          - eltwise.unary.hardswish.hardswish
          - eltwise.unary.hardswish.hardswish_sharded
          - eltwise.unary.hardsigmoid.hardsigmoid
          - eltwise.unary.hardsigmoid.hardsigmoid_sharded
          - eltwise.unary.hardshrink.hardshrink
          - eltwise.unary.hardshrink.hardshrink_sharded
          - eltwise.unary.softmax.softmax
          - eltwise.unary.identity.identity
          - eltwise.unary.neg.neg
          - eltwise.unary.sinh.sinh
          - eltwise.unary.asinh.asinh
          - eltwise.unary.cosh.cosh
          - eltwise.unary.relu_min.relu_min
          - eltwise.unary.relu_max.relu_max
          - eltwise.unary.softplus.softplus
          - eltwise.unary_backward.fill_zero_bw
          - eltwise.unary_backward.log_sigmoid_bw
          - eltwise.unary_backward.logit_bw
          - eltwise.unary_backward.neg_bw
          - eltwise.unary_backward.hardshrink_bw
          - eltwise.unary_backward.softshrink_bw
          - eltwise.unary_backward.acos_bw.acos_bw
          - eltwise.unary_backward.acosh_bw.acosh_bw
          - eltwise.unary_backward.atan_bw.atan_bw
          - eltwise.unary_backward.cos_bw.cos_bw
          - eltwise.unary_backward.frac_bw.frac_bw
          - eltwise.unary_backward.i0_bw.i0_bw
          - eltwise.unary_backward.rad2deg_bw.rad2deg_bw
          - eltwise.unary_backward.relu_bw.relu_bw
          - eltwise.unary_backward.rsqrt_bw.rsqrt_bw
          - eltwise.unary_backward.sigmoid_bw.sigmoid_bw
          - eltwise.unary_backward.tan_bw.tan_bw
          - eltwise.unary_backward.trunc_bw.trunc_bw
          - eltwise.unary_backward.clamp_bw.clamp_bw
          - eltwise.unary_backward.hardtanh_bw.hardtanh_bw
          - eltwise.unary_backward.mul_bw.mul_bw
          - eltwise.unary_backward.softplus_bw.softplus_bw
          - eltwise.unary_backward.threshold_bw.threshold_bw
          - eltwise.unary_backward.div_bw.div_bw
          - eltwise.unary_backward.log_bw.log_bw
          - eltwise.unary_backward.relu6_bw.relu6_bw
          - eltwise.unary_backward.log10_bw.log10_bw
          - eltwise.unary_backward.abs_bw.abs_bw
          - eltwise.unary_backward.sinh_bw.sinh_bw
          - eltwise.unary_backward.sin_bw.sin_bw
          - eltwise.unary_backward.square_bw.square_bw
          - eltwise.unary_backward.rdiv_bw.rdiv_bw
          - eltwise.unary_backward.bias_gelu_bw.bias_gelu_bw
          - eltwise.unary_backward.pow_bw.pow_bw
          - eltwise.unary_backward.exp_bw.exp_bw
          - eltwise.unary_backward.tanh_bw.tanh_bw
          - eltwise.unary_backward.sqrt_bw.sqrt_bw
          - eltwise.unary_backward.add_bw.add_bw
          - eltwise.unary_backward.assign_bw.assign_bw
          - eltwise.unary_backward.fill_bw.fill_bw
          - eltwise.unary_backward.hardsigmoid_bw.hardsigmoid_bw
          - eltwise.unary_backward.lgamma_bw.lgamma_bw
          - eltwise.unary_backward.multigammaln_bw.multigammaln_bw
          - eltwise.unary_backward.leaky_relu_bw.leaky_relu_bw
          - eltwise.unary_backward.elu_bw.elu_bw
          - eltwise.unary_backward.celu_bw.celu_bw
          - eltwise.unary_backward.selu_bw.selu_bw
          - eltwise.unary_backward.silu_bw.silu_bw
          - eltwise.unary_backward.floor_bw.floor_bw
          - eltwise.unary_backward.round_bw.round_bw
          - eltwise.unary_backward.tanhshrink_bw.tanhshrink_bw
          - eltwise.unary_backward.hardswish_bw.hardswish_bw
          - eltwise.unary_backward.rpow_bw.rpow_bw
          - eltwise.unary_complex.conj
          - eltwise.unary_complex.is_real
          - eltwise.unary_complex.is_imag
          - eltwise.unary_complex.reciprocal
          - eltwise.unary_complex.reciprocal_bw
          - eltwise.binary_complex.div_bw.div_bw
          - eltwise.binary_complex.add_bw.add_bw
          - eltwise.binary_complex.sub_bw.sub_bw
          - eltwise.binary_complex.mul_bw.mul_bw
          - eltwise.unary.lgamma
          - eltwise.unary.logit
          - eltwise.unary.mish
          - eltwise.unary.multigammaln
          - eltwise.unary.isfinite
          - eltwise.unary.isinf
          - eltwise.unary.isnan
          - eltwise.unary.isneginf
          - eltwise.unary.isposinf
          - eltwise.binary.add.add_all_pytorch2
          - eltwise.binary.add.add_set2_pytorch2
          - eltwise.binary.add.add_different_memory_configs
          - eltwise.unary.gtz.gtz
          - eltwise.unary.ltz.ltz
          - eltwise.unary.gez.gez
          - eltwise.unary.lez.lez
          - eltwise.unary.nez.nez
          - eltwise.unary.prelu.prelu
          - eltwise.unary.hardswish.hardswish_pytorch2
          - eltwise.unary.hardtanh.hardtanh_pytorch2
          - eltwise.unary.leaky_relu.leaky_relu
          - eltwise.unary.reglu.reglu
          - eltwise.unary_complex.polar.polar
          - eltwise.unary_complex.angle.angle
          - eltwise.unary_complex.polar_bw.polar_bw
          - eltwise.unary_complex.angle_bw.angle_bw
          - eltwise.unary_complex.conj_bw
          - eltwise.binary.subtract.subtract
          - eltwise.binary.subtract.subtract_tensor_pytorch2
          - eltwise.binary.multiply.multiply
          - eltwise.binary.multiply.mul_tensor_pytorch2
          - eltwise.binary.multiply.multiply_scalar_pytorch2
          - eltwise.binary.div.div
          - eltwise.binary.div.div_tensor_pytorch2
          - eltwise.binary.div_no_nan.div_no_nan
          - eltwise.binary.logical_or.logical_or_
          - eltwise.binary.logical_or.logical_or
          - eltwise.binary.logical_or.logical_or_output
          - eltwise.binary.logical_xor.logical_xor_
          - eltwise.binary.logical_xor.logical_xor
          - eltwise.binary.logical_and.logical_and_
          - eltwise.binary.logical_and.logical_and
          - eltwise.binary.logical_and.logical_and_output
          - eltwise.binary.polyval.polyval
          - eltwise.binary.remainder.remainder
          - eltwise.binary.squared_difference.squared_difference
          - eltwise.binary.squared_difference_output.squared_difference_output
          - eltwise.binary.remainder.remainder_scalar_pytorch2
          - eltwise.binary.bcast.bcast_h_sharded
          - eltwise.binary.bcast.bcast
          - eltwise.binary.eq.eq_scalar_pytorch2
          - eltwise.binary.gt.gt_scalar_pytorch2
          - eltwise.binary.le.le_tensor_pytorch2
          - eltwise.binary.fmod.fmod
          - eltwise.binary.floor_divide.floor_divide_pytorch2
          - eltwise.binary.logaddexp.logaddexp
          - eltwise.binary.logaddexp2.logaddexp2
          - eltwise.binary.ldexp.ldexp
          - eltwise.binary.lt.lt_tensor_pytorch2
          - eltwise.binary.lt.lt_scalar_pytorch2
          - eltwise.binary.ne.ne_scalar_pytorch2
          - eltwise.binary.hypot.hypot
          - eltwise.binary.xlogy.xlogy
          - eltwise.binary_backward.ldexp_bw.ldexp_bw
          - eltwise.binary_backward.logaddexp_bw
          - eltwise.binary_backward.logaddexp2_bw
          - eltwise.binary_backward.addalpha_bw.addalpha_bw
          - eltwise.binary_backward.subalpha_bw.subalpha_bw
          - eltwise.binary_backward.xlogy_bw.xlogy_bw
          - eltwise.binary_backward.hypot_bw.hypot_bw
          - eltwise.binary_backward.add_bw.add_bw
          - eltwise.binary_backward.sub_bw.sub_bw
          - eltwise.binary_backward.mul_bw.mul_bw
          - eltwise.binary_backward.div_bw.div_bw
          - eltwise.binary_backward.fmod_bw.fmod_bw
          - eltwise.binary_backward.remainder_bw.remainder_bw
          - eltwise.binary_backward.rsub_bw.rsub_bw
          - eltwise.binary_backward.squared_difference_bw.squared_difference_bw
          - eltwise.composite.binary.addalpha.addalpha
          - eltwise.composite.binary.subalpha.subalpha
          - eltwise.composite.binary.minimum.minimum
          - eltwise.composite.binary.minimum.minimum_pytorch2
          - eltwise.composite.binary.maximum.maximum
          - eltwise.composite.binary.maximum.maximum_pytorch2
          - eltwise.composite.binary.pow.pow_pytorch2
          - eltwise.composite.binary.pow.pow_scalar_pytorch2
          - eltwise.composite.binary.pow.pow_tensor_pytorch2
          - eltwise.ternary.addcmul.addcmul
          - eltwise.ternary.addcdiv.addcdiv
          - eltwise.ternary.mac.mac
          - eltwise.ternary.lerp.lerp
          - eltwise.ternary.where.where
          - eltwise.ternary.where.where_pytorch2
          - eltwise.ternary_backward.addcmul_bw
          - eltwise.ternary_backward.addcdiv_bw
          - embedding.embedding
          - embedding_bw.embedding_bw
          - fused.softmax_traces
          - fused.layer_norm_traces
          - reduction.backward.prod_bw.prod_bw
          - reduction.topk.topk
          - reduction.argmax.argmax
          - reduction.prod
          - reduction.sum
          - reduction.var.var
          - reduction.std.std
          - reduction.mean.mean
          - reduction.traces.argmax_traces
          - reduction.traces.max_traces
          - reduction.traces.mean_traces
          - reduction.traces.sum_traces
          - reduction.traces.topk_traces
          - matmul.full.matmul_default_block_sharded
          - matmul.full.matmul_default_height_sharded
          - matmul.full.matmul_default_interleaved
          - matmul.full.matmul_default_width_sharded
          - matmul.short.matmul_create_program_config
          - matmul.short.matmul_default_sharded
          - matmul.short.matmul_default
          - matmul.short.matmul_user_program_config_mcast_1d
          - matmul.short.matmul_user_program_config_mcast_2d
          - matmul.short.matmul_user_program_config
          - matmul.short.matmul_traces
          - matmul.short.matmul
          - losses.l1_loss
          - losses.mse_loss
          - data_movement.concat.concat_interleaved_n_tensors
          - data_movement.concat.concat_interleaved
          - data_movement.concat.concat_sharded
          - data_movement.concat.concat_pytorch2
          - data_movement.slice.slice_pytorch2_rm
          - data_movement.slice.slice_pytorch2_tiled
          - data_movement.permute.permute
          - data_movement.permute.permute_pytorch2_tiled
          - data_movement.permute.permute_pytorch2_rm
          - data_movement.transpose.transpose_pytorch2
          - data_movement.transpose.transpose_interleaved
          - data_movement.transpose.t_pytorch2
          - data_movement.copy.copy
          - data_movement.expand.expand_pytorch2
          - data_movement.fill.fill_pytorch2
          - data_movement.index_select.index_select_pytorch2
          - data_movement.split.split_with_sizes_pytorch2
          - data_movement.repeat.repeat
          - data_movement.reshape.reshape
          - data_movement.repeat_interleave.repeat_interleave
          - data_movement.nonzero.nonzero
          - data_movement.backward.concat_bw.concat_bw
          - conv2d.full.conv2d_misc
          - conv2d.full.conv2d_sharding
          - conv2d.full.conv2d_sliding_window
          - conv2d.short.conv2d_short_sweep
          - pooling.global_avg_pool2d
          - pooling.max_pool2d
          - max_pool2d.short.max_pool2d_short_sweep
          - max_pool2d.full.max_pool2d_params
          - max_pool2d.full.max_pool2d_large_dims
          - transformer.concatenate_heads.concatenate_heads
          - transformer.split_query_key_value_and_split_heads.split_query_key_value_and_split_heads
          - transformer.split_query_key_value_and_split_heads.split_query_key_value_and_split_heads_kv_input
          - transformer.attention_softmax.attention_softmax
          - transformer.attention_softmax.attention_softmax_
          - data_movement.stack.stack_pytorch2
          - data_movement.repeat.repeat_pytorch2
          - data_movement.split.split_pytorch2
          - data_movement.unsqueeze.unsqueeze_pytorch2
          - data_movement.squeeze.squeeze_pytorch2
          - data_movement.embedding.embedding_pytorch2
          - data_movement.view.view_pytorch2
  schedule:
    - cron: "0 21 * * *" # This cron schedule runs the workflow at 9:00pm UTC nightly

jobs:
  build-artifact:
    uses: ./.github/workflows/build-artifact.yaml
    secrets: inherit

  ttnn-generate-sweeps:
    needs: build-artifact
    env:
      TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
      ARCH_NAME: wormhole_b0
      ELASTIC_USERNAME: ${{ secrets.SWEEPS_ELASTIC_USERNAME }}
      ELASTIC_PASSWORD: ${{ secrets.SWEEPS_ELASTIC_PASSWORD }}
      LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
    timeout-minutes: 30
    runs-on: [build, in-service]
    steps:
      - uses: tenstorrent/tt-metal/.github/actions/checkout-with-submodule-lfs@main
      - name: Set up dynamic env vars for build
        run: |
          echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
      - uses: ./.github/actions/prepare-metal-run
        with:
          arch: wormhole_b0
      - name: Run ttnn sweeps generation (single sweep)
        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.sweep_name != 'ALL SWEEPS (Nightly)' }}
        run: |
          source ${{ github.workspace }}/python_env/bin/activate
          cd $TT_METAL_HOME
          export PYTHONPATH=$TT_METAL_HOME
          python tests/sweep_framework/sweeps_parameter_generator.py --module-name ${{ github.event.inputs.sweep_name }} --elastic cloud --tag ci-main --explicit
      - name: Run ttnn sweeps generation (all sweeps)
        if: ${{ github.event_name == 'schedule' || github.event.inputs.sweep_name == 'ALL SWEEPS (Nightly)' }}
        run: |
          source ${{ github.workspace }}/python_env/bin/activate
          cd $TT_METAL_HOME
          export PYTHONPATH=$TT_METAL_HOME
          python tests/sweep_framework/sweeps_parameter_generator.py --elastic cloud --tag ci-main --explicit

  ttnn-run-sweeps:
    needs: ttnn-generate-sweeps
    strategy:
      # Do not fail-fast because we need to ensure all tests go to completion
      # so we try not to get hanging machines
      fail-fast: false
      matrix:
        test-group:
          [
            {
              name: "Grayskull E150 Sweeps",
              arch: grayskull,
              runs-on: ["cloud-virtual-machine", "E150", "in-service"],
              tt-smi-cmd: "tt-smi-metal -r 0"
            },
            {
              name: "Wormhole N150 Sweeps",
              arch: wormhole_b0,
              runs-on: ["cloud-virtual-machine", "N150", "in-service"],
              tt-smi-cmd: "tt-smi-metal -r 0"
            },
            {
              name: "Wormhole N300 Sweeps",
              arch: wormhole_b0,
              runs-on: ["cloud-virtual-machine", "N300", "in-service"],
              tt-smi-cmd: "tt-smi-metal -r 0"
            }
          ]
    env:
      TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
      ARCH_NAME: ${{ matrix.test-group.arch }}
      ELASTIC_USERNAME: ${{ secrets.SWEEPS_ELASTIC_USERNAME }}
      ELASTIC_PASSWORD: ${{ secrets.SWEEPS_ELASTIC_PASSWORD }}
      TT_SMI_RESET_COMMAND: ${{ matrix.test-group.tt-smi-cmd }}
      LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
    timeout-minutes: 720
    runs-on: ${{ matrix.test-group.runs-on }}
    steps:
      - uses: tenstorrent/tt-metal/.github/actions/checkout-with-submodule-lfs@main
      - name: Set up dynamic env vars for build
        run: |
          echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
      - uses: ./.github/actions/prepare-metal-run
        with:
          arch: ${{ matrix.test-group.arch }}
      - name: Run ttnn sweeps (single sweep)
        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.sweep_name != 'ALL SWEEPS (Nightly)' }}
        run: |
          source ${{ github.workspace }}/python_env/bin/activate
          cd $TT_METAL_HOME
          export PYTHONPATH=$TT_METAL_HOME
          python tests/sweep_framework/sweeps_runner.py --module-name ${{ github.event.inputs.sweep_name }} --elastic cloud --tag ci-main
      - name: Run ttnn sweeps (all sweeps, nightly)
        if: ${{ github.event_name == 'schedule' || github.event.inputs.sweep_name == 'ALL SWEEPS (Nightly)' }}
        run: |
          source ${{ github.workspace }}/python_env/bin/activate
          cd $TT_METAL_HOME
          export PYTHONPATH=$TT_METAL_HOME
          python tests/sweep_framework/sweeps_runner.py --elastic cloud --tag ci-main --suite-name nightly