diff --git a/.github/workflows/ttnn-run-sweeps.yaml b/.github/workflows/ttnn-run-sweeps.yaml index f9e835ed3fe2..781bdaaa9fd0 100644 --- a/.github/workflows/ttnn-run-sweeps.yaml +++ b/.github/workflows/ttnn-run-sweeps.yaml @@ -174,11 +174,16 @@ on: - eltwise.unary.logit - eltwise.unary.mish - eltwise.unary.multigammaln - - eltwise.unary.isfinite - - eltwise.unary.isinf - - eltwise.unary.isnan - - eltwise.unary.isneginf - - eltwise.unary.isposinf + - eltwise.unary.isfinite.isfinite + - eltwise.unary.isfinite.isfinite_sharded + - eltwise.unary.isinf.isinf + - eltwise.unary.isinf.isinf_sharded + - eltwise.unary.isnan.isnan + - eltwise.unary.isnan.isnan_sharded + - eltwise.unary.isneginf.isneginf + - eltwise.unary.isneginf.isneginf_sharded + - eltwise.unary.isposinf.isposinf + - eltwise.unary.isposinf.isposinf_sharded - eltwise.binary.add.add_all_pytorch2 - eltwise.binary.add.add_set2_pytorch2 - eltwise.binary.add.add_different_memory_configs diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isfinite/isfinite_sharded.py b/tests/sweep_framework/sweeps/eltwise/unary/isfinite/isfinite_sharded.py index fec93f8f95dc..121673c0b638 100644 --- a/tests/sweep_framework/sweeps/eltwise/unary/isfinite/isfinite_sharded.py +++ b/tests/sweep_framework/sweeps/eltwise/unary/isfinite/isfinite_sharded.py @@ -44,7 +44,7 @@ def invalidate_vector(test_vector) -> Tuple[bool, Optional[str]]: pre_sharded_height = math.prod(input_shape[:-1]) pre_sharded_width = input_shape[-1] - if input_layout == ttnn.ROW_MAJOR_LAYOUT and test_vector["input_a_dtype"] == ttnn.bfloat8_b: + if input_layout == "ROW_MAJOR_LAYOUT" and test_vector["input_a_dtype"] == ttnn.bfloat8_b: return True, "bfloat8_b is only supported on tiled layout" return False, None @@ -101,32 +101,3 @@ def run( pcc = check_with_pcc(torch_output_tensor, output_tensor, 0.999) # print(pcc) return [check_with_pcc(torch_output_tensor, output_tensor, 0.999), e2e_perf] - - -# # Run sweeps locally -# from tests.sweep_framework.framework.permutations import * - -# start_time = start_measuring_time() -# for suite in parameters.keys(): -# device_id = 0 -# device = ttnn.open_device(device_id=device_id) -# suite_vectors = list(permutations(parameters[suite])) -# print(len(suite_vectors)) -# for vector in suite_vectors: -# invalidate_res = invalidate_vector(vector) -# if invalidate_res[0]: -# print(f"Invalidated: {invalidate_res[1]}") -# continue -# try: -# passed, _ = run(**vector, device=device) -# # if passed[0] != True: -# # print(passed) -# except Exception as e: -# print(e) - -# # break - -# ttnn.close_device(device) - -# e2e_perf = stop_measuring_time(start_time) -# print(f"time {e2e_perf / 1000000000}s") diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isinf.py b/tests/sweep_framework/sweeps/eltwise/unary/isinf/isinf.py similarity index 100% rename from tests/sweep_framework/sweeps/eltwise/unary/isinf.py rename to tests/sweep_framework/sweeps/eltwise/unary/isinf/isinf.py diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isinf/isinf_sharded.py b/tests/sweep_framework/sweeps/eltwise/unary/isinf/isinf_sharded.py new file mode 100644 index 000000000000..94b80869dc4c --- /dev/null +++ b/tests/sweep_framework/sweeps/eltwise/unary/isinf/isinf_sharded.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: © 2024 Tenstorrent Inc. + +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional, Tuple +from functools import partial + +import json +import torch +import random +import ttnn +import math +from tests.sweep_framework.sweep_utils.utils import gen_shapes, sanitize_shape_rm, get_device_grid_size +from tests.sweep_framework.sweep_utils.sharding_utils import gen_sharded_spec_unary, parse_sharding_spec +from tests.tt_eager.python_api_testing.sweep_tests.generation_funcs import gen_rand_inf + +from tests.ttnn.utils_for_testing import check_with_pcc, start_measuring_time, stop_measuring_time +from models.utility_functions import torch_random + +# Override the default timeout in seconds for hang detection. +TIMEOUT = 120 +Y, X = get_device_grid_size() + +random.seed(0) + + +# Parameters provided to the test vector generator are defined here. +# They are defined as dict-type suites that contain the arguments to the run function as keys, and lists of possible inputs as values. +# Each suite has a key name (in this case "suite_1" and "suite_2") which will associate the test vectors to this specific suite of inputs. +# Developers can create their own generator functions and pass them to the parameters as inputs. +parameters = { + "nightly": { + "input_spec": gen_sharded_spec_unary(16, Y, X), + "input_a_dtype": [ttnn.bfloat16, ttnn.bfloat8_b], + }, +} + + +# Invalidate vector is called during the generation phase where each vector will be passed in. +# If invalidated, the vector will still be stored but will be skipped. +# Returns False, None if the vector is valid, and True, str with a reason for invalidation if it is invalid. +def invalidate_vector(test_vector) -> Tuple[bool, Optional[str]]: + input_shape, sharding_strategy, _, _, input_layout = test_vector["input_spec"].values() + pre_sharded_height = math.prod(input_shape[:-1]) + pre_sharded_width = input_shape[-1] + + if input_layout == "ROW_MAJOR_LAYOUT" and test_vector["input_a_dtype"] == ttnn.bfloat8_b: + return True, "bfloat8_b is only supported on tiled layout" + + return False, None + + +# This is the run instructions for the test, defined by the developer. +# The run function must take the above-defined parameters as inputs. +# The runner will call this run function with each test vector, and the returned results from this function will be stored. +# If you defined a mesh_device_fixture above, the object you yielded will be passed into this function as 'device'. Otherwise, it will be the default ttnn device opened by the infra. +def run( + input_spec, + input_a_dtype, + *, + device, +) -> list: + data_seed = random.randint(0, 20000000) + torch.manual_seed(data_seed) + + input_shape, sharding_strategy, shard_orientation, tensor_hw_as_shard_shape, input_layout = parse_sharding_spec( + input_spec + ) + + # print( + # f"X {X} Y {Y} input_shape {input_shape} {input_a_dtype} {input_layout} {sharding_strategy} {shard_orientation} tensor_hw_as_shard_shape {tensor_hw_as_shard_shape}" + # ) + + if input_layout == ttnn.ROW_MAJOR_LAYOUT: + input_shape = sanitize_shape_rm(input_shape) + + torch_input_tensor_a = gen_rand_inf(input_shape, low=-100, high=100) + torch_output_tensor = torch.isinf(torch_input_tensor_a) + + sharded_config = ttnn.create_sharded_memory_config( + shape=input_shape, + core_grid=ttnn.CoreGrid(y=Y, x=X), + strategy=sharding_strategy, + orientation=shard_orientation, + use_height_and_width_as_shard_shape=tensor_hw_as_shard_shape, + ) + + input_tensor_a = ttnn.from_torch( + torch_input_tensor_a, + dtype=input_a_dtype, + layout=input_layout, + device=device, + memory_config=sharded_config, + ) + + start_time = start_measuring_time() + output_tensor = ttnn.isinf(input_tensor_a, memory_config=sharded_config) + e2e_perf = stop_measuring_time(start_time) + output_tensor = ttnn.to_torch(output_tensor) + + pcc = check_with_pcc(torch_output_tensor, output_tensor, 0.999) + # print(pcc) + return [check_with_pcc(torch_output_tensor, output_tensor, 0.999), e2e_perf] diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isnan.py b/tests/sweep_framework/sweeps/eltwise/unary/isnan/isnan.py similarity index 100% rename from tests/sweep_framework/sweeps/eltwise/unary/isnan.py rename to tests/sweep_framework/sweeps/eltwise/unary/isnan/isnan.py diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isnan/isnan_sharded.py b/tests/sweep_framework/sweeps/eltwise/unary/isnan/isnan_sharded.py new file mode 100644 index 000000000000..d86bd84c2c3f --- /dev/null +++ b/tests/sweep_framework/sweeps/eltwise/unary/isnan/isnan_sharded.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: © 2024 Tenstorrent Inc. + +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional, Tuple +from functools import partial + +import json +import torch +import random +import ttnn +import math +from tests.sweep_framework.sweep_utils.utils import gen_shapes, sanitize_shape_rm, get_device_grid_size +from tests.sweep_framework.sweep_utils.sharding_utils import gen_sharded_spec_unary, parse_sharding_spec +from tests.tt_eager.python_api_testing.sweep_tests.generation_funcs import gen_rand_inf + +from tests.ttnn.utils_for_testing import check_with_pcc, start_measuring_time, stop_measuring_time +from models.utility_functions import torch_random + +# Override the default timeout in seconds for hang detection. +TIMEOUT = 120 +Y, X = get_device_grid_size() + +random.seed(0) + + +# Parameters provided to the test vector generator are defined here. +# They are defined as dict-type suites that contain the arguments to the run function as keys, and lists of possible inputs as values. +# Each suite has a key name (in this case "suite_1" and "suite_2") which will associate the test vectors to this specific suite of inputs. +# Developers can create their own generator functions and pass them to the parameters as inputs. +parameters = { + "nightly": { + "input_spec": gen_sharded_spec_unary(16, Y, X), + "input_a_dtype": [ttnn.bfloat16, ttnn.bfloat8_b], + }, +} + + +# Invalidate vector is called during the generation phase where each vector will be passed in. +# If invalidated, the vector will still be stored but will be skipped. +# Returns False, None if the vector is valid, and True, str with a reason for invalidation if it is invalid. +def invalidate_vector(test_vector) -> Tuple[bool, Optional[str]]: + input_shape, sharding_strategy, _, _, input_layout = test_vector["input_spec"].values() + pre_sharded_height = math.prod(input_shape[:-1]) + pre_sharded_width = input_shape[-1] + + if input_layout == "ROW_MAJOR_LAYOUT" and test_vector["input_a_dtype"] == ttnn.bfloat8_b: + return True, "bfloat8_b is only supported on tiled layout" + + return False, None + + +# This is the run instructions for the test, defined by the developer. +# The run function must take the above-defined parameters as inputs. +# The runner will call this run function with each test vector, and the returned results from this function will be stored. +# If you defined a mesh_device_fixture above, the object you yielded will be passed into this function as 'device'. Otherwise, it will be the default ttnn device opened by the infra. +def run( + input_spec, + input_a_dtype, + *, + device, +) -> list: + data_seed = random.randint(0, 20000000) + torch.manual_seed(data_seed) + + input_shape, sharding_strategy, shard_orientation, tensor_hw_as_shard_shape, input_layout = parse_sharding_spec( + input_spec + ) + + # print( + # f"X {X} Y {Y} input_shape {input_shape} {input_a_dtype} {input_layout} {sharding_strategy} {shard_orientation} tensor_hw_as_shard_shape {tensor_hw_as_shard_shape}" + # ) + + if input_layout == ttnn.ROW_MAJOR_LAYOUT: + input_shape = sanitize_shape_rm(input_shape) + + torch_input_tensor_a = gen_rand_inf(input_shape, low=-100, high=100) + torch_output_tensor = torch.isnan(torch_input_tensor_a) + + sharded_config = ttnn.create_sharded_memory_config( + shape=input_shape, + core_grid=ttnn.CoreGrid(y=Y, x=X), + strategy=sharding_strategy, + orientation=shard_orientation, + use_height_and_width_as_shard_shape=tensor_hw_as_shard_shape, + ) + + input_tensor_a = ttnn.from_torch( + torch_input_tensor_a, + dtype=input_a_dtype, + layout=input_layout, + device=device, + memory_config=sharded_config, + ) + + start_time = start_measuring_time() + output_tensor = ttnn.isnan(input_tensor_a, memory_config=sharded_config) + e2e_perf = stop_measuring_time(start_time) + output_tensor = ttnn.to_torch(output_tensor) + + pcc = check_with_pcc(torch_output_tensor, output_tensor, 0.999) + # print(pcc) + return [check_with_pcc(torch_output_tensor, output_tensor, 0.999), e2e_perf] + + +# Run sweeps locally +# from tests.sweep_framework.framework.permutations import * + +# start_time = start_measuring_time() +# for suite in parameters.keys(): +# device_id = 0 +# device = ttnn.open_device(device_id=device_id) +# suite_vectors = list(permutations(parameters[suite])) +# print(len(suite_vectors)) +# for vector in suite_vectors: +# invalidate_res = invalidate_vector(vector) +# if invalidate_res[0]: +# print(f"Invalidated: {invalidate_res[1]}") +# continue +# try: +# passed, _ = run(**vector, device=device) +# # if passed[0] != True: +# # print(passed) +# except Exception as e: +# print(e) + +# # break + +# ttnn.close_device(device) + +# e2e_perf = stop_measuring_time(start_time) +# print(f"time {e2e_perf / 1000000000}s") diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isneginf.py b/tests/sweep_framework/sweeps/eltwise/unary/isneginf/isneginf.py similarity index 100% rename from tests/sweep_framework/sweeps/eltwise/unary/isneginf.py rename to tests/sweep_framework/sweeps/eltwise/unary/isneginf/isneginf.py diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isneginf/isneginf_sharded.py b/tests/sweep_framework/sweeps/eltwise/unary/isneginf/isneginf_sharded.py new file mode 100644 index 000000000000..b8bdb32ef662 --- /dev/null +++ b/tests/sweep_framework/sweeps/eltwise/unary/isneginf/isneginf_sharded.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: © 2024 Tenstorrent Inc. + +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional, Tuple +from functools import partial + +import json +import torch +import random +import ttnn +import math +from tests.sweep_framework.sweep_utils.utils import gen_shapes, sanitize_shape_rm, get_device_grid_size +from tests.sweep_framework.sweep_utils.sharding_utils import gen_sharded_spec_unary, parse_sharding_spec +from tests.tt_eager.python_api_testing.sweep_tests.generation_funcs import gen_rand_inf + +from tests.ttnn.utils_for_testing import check_with_pcc, start_measuring_time, stop_measuring_time +from models.utility_functions import torch_random + +# Override the default timeout in seconds for hang detection. +TIMEOUT = 120 +Y, X = get_device_grid_size() + +random.seed(0) + + +# Parameters provided to the test vector generator are defined here. +# They are defined as dict-type suites that contain the arguments to the run function as keys, and lists of possible inputs as values. +# Each suite has a key name (in this case "suite_1" and "suite_2") which will associate the test vectors to this specific suite of inputs. +# Developers can create their own generator functions and pass them to the parameters as inputs. +parameters = { + "nightly": { + "input_spec": gen_sharded_spec_unary(16, Y, X), + "input_a_dtype": [ttnn.bfloat16, ttnn.bfloat8_b], + }, +} + + +# Invalidate vector is called during the generation phase where each vector will be passed in. +# If invalidated, the vector will still be stored but will be skipped. +# Returns False, None if the vector is valid, and True, str with a reason for invalidation if it is invalid. +def invalidate_vector(test_vector) -> Tuple[bool, Optional[str]]: + input_shape, sharding_strategy, _, _, input_layout = test_vector["input_spec"].values() + pre_sharded_height = math.prod(input_shape[:-1]) + pre_sharded_width = input_shape[-1] + + if input_layout == "ROW_MAJOR_LAYOUT" and test_vector["input_a_dtype"] == ttnn.bfloat8_b: + return True, "bfloat8_b is only supported on tiled layout" + + return False, None + + +# This is the run instructions for the test, defined by the developer. +# The run function must take the above-defined parameters as inputs. +# The runner will call this run function with each test vector, and the returned results from this function will be stored. +# If you defined a mesh_device_fixture above, the object you yielded will be passed into this function as 'device'. Otherwise, it will be the default ttnn device opened by the infra. +def run( + input_spec, + input_a_dtype, + *, + device, +) -> list: + data_seed = random.randint(0, 20000000) + torch.manual_seed(data_seed) + + input_shape, sharding_strategy, shard_orientation, tensor_hw_as_shard_shape, input_layout = parse_sharding_spec( + input_spec + ) + + # print( + # f"X {X} Y {Y} input_shape {input_shape} {input_a_dtype} {input_layout} {sharding_strategy} {shard_orientation} tensor_hw_as_shard_shape {tensor_hw_as_shard_shape}" + # ) + + if input_layout == ttnn.ROW_MAJOR_LAYOUT: + input_shape = sanitize_shape_rm(input_shape) + + torch_input_tensor_a = gen_rand_inf(input_shape, low=-100, high=100) + torch_output_tensor = torch.isneginf(torch_input_tensor_a) + + sharded_config = ttnn.create_sharded_memory_config( + shape=input_shape, + core_grid=ttnn.CoreGrid(y=Y, x=X), + strategy=sharding_strategy, + orientation=shard_orientation, + use_height_and_width_as_shard_shape=tensor_hw_as_shard_shape, + ) + + input_tensor_a = ttnn.from_torch( + torch_input_tensor_a, + dtype=input_a_dtype, + layout=input_layout, + device=device, + memory_config=sharded_config, + ) + + start_time = start_measuring_time() + output_tensor = ttnn.isneginf(input_tensor_a, memory_config=sharded_config) + e2e_perf = stop_measuring_time(start_time) + output_tensor = ttnn.to_torch(output_tensor) + + pcc = check_with_pcc(torch_output_tensor, output_tensor, 0.999) + # print(pcc) + return [check_with_pcc(torch_output_tensor, output_tensor, 0.999), e2e_perf] diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isposinf.py b/tests/sweep_framework/sweeps/eltwise/unary/isposinf/isposinf.py similarity index 100% rename from tests/sweep_framework/sweeps/eltwise/unary/isposinf.py rename to tests/sweep_framework/sweeps/eltwise/unary/isposinf/isposinf.py diff --git a/tests/sweep_framework/sweeps/eltwise/unary/isposinf/isposinf_sharded.py b/tests/sweep_framework/sweeps/eltwise/unary/isposinf/isposinf_sharded.py new file mode 100644 index 000000000000..63bf5a9610d6 --- /dev/null +++ b/tests/sweep_framework/sweeps/eltwise/unary/isposinf/isposinf_sharded.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: © 2024 Tenstorrent Inc. + +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional, Tuple +from functools import partial + +import json +import torch +import random +import ttnn +import math +from tests.sweep_framework.sweep_utils.utils import gen_shapes, sanitize_shape_rm, get_device_grid_size +from tests.sweep_framework.sweep_utils.sharding_utils import gen_sharded_spec_unary, parse_sharding_spec +from tests.tt_eager.python_api_testing.sweep_tests.generation_funcs import gen_rand_inf + +from tests.ttnn.utils_for_testing import check_with_pcc, start_measuring_time, stop_measuring_time +from models.utility_functions import torch_random + +# Override the default timeout in seconds for hang detection. +TIMEOUT = 120 +Y, X = get_device_grid_size() + +random.seed(0) + + +# Parameters provided to the test vector generator are defined here. +# They are defined as dict-type suites that contain the arguments to the run function as keys, and lists of possible inputs as values. +# Each suite has a key name (in this case "suite_1" and "suite_2") which will associate the test vectors to this specific suite of inputs. +# Developers can create their own generator functions and pass them to the parameters as inputs. +parameters = { + "nightly": { + "input_spec": gen_sharded_spec_unary(16, Y, X), + "input_a_dtype": [ttnn.bfloat16, ttnn.bfloat8_b], + }, +} + + +# Invalidate vector is called during the generation phase where each vector will be passed in. +# If invalidated, the vector will still be stored but will be skipped. +# Returns False, None if the vector is valid, and True, str with a reason for invalidation if it is invalid. +def invalidate_vector(test_vector) -> Tuple[bool, Optional[str]]: + input_shape, sharding_strategy, _, _, input_layout = test_vector["input_spec"].values() + pre_sharded_height = math.prod(input_shape[:-1]) + pre_sharded_width = input_shape[-1] + + if input_layout == "ROW_MAJOR_LAYOUT" and test_vector["input_a_dtype"] == ttnn.bfloat8_b: + return True, "bfloat8_b is only supported on tiled layout" + + return False, None + + +# This is the run instructions for the test, defined by the developer. +# The run function must take the above-defined parameters as inputs. +# The runner will call this run function with each test vector, and the returned results from this function will be stored. +# If you defined a mesh_device_fixture above, the object you yielded will be passed into this function as 'device'. Otherwise, it will be the default ttnn device opened by the infra. +def run( + input_spec, + input_a_dtype, + *, + device, +) -> list: + data_seed = random.randint(0, 20000000) + torch.manual_seed(data_seed) + + input_shape, sharding_strategy, shard_orientation, tensor_hw_as_shard_shape, input_layout = parse_sharding_spec( + input_spec + ) + + # print( + # f"X {X} Y {Y} input_shape {input_shape} {input_a_dtype} {input_layout} {sharding_strategy} {shard_orientation} tensor_hw_as_shard_shape {tensor_hw_as_shard_shape}" + # ) + + if input_layout == ttnn.ROW_MAJOR_LAYOUT: + input_shape = sanitize_shape_rm(input_shape) + + torch_input_tensor_a = gen_rand_inf(input_shape, low=-100, high=100) + torch_output_tensor = torch.isfinite(torch_input_tensor_a) + + sharded_config = ttnn.create_sharded_memory_config( + shape=input_shape, + core_grid=ttnn.CoreGrid(y=Y, x=X), + strategy=sharding_strategy, + orientation=shard_orientation, + use_height_and_width_as_shard_shape=tensor_hw_as_shard_shape, + ) + + input_tensor_a = ttnn.from_torch( + torch_input_tensor_a, + dtype=input_a_dtype, + layout=input_layout, + device=device, + memory_config=sharded_config, + ) + + start_time = start_measuring_time() + output_tensor = ttnn.isfinite(input_tensor_a, memory_config=sharded_config) + e2e_perf = stop_measuring_time(start_time) + output_tensor = ttnn.to_torch(output_tensor) + + pcc = check_with_pcc(torch_output_tensor, output_tensor, 0.999) + # print(pcc) + return [check_with_pcc(torch_output_tensor, output_tensor, 0.999), e2e_perf] + + +# Run sweeps locally +# from tests.sweep_framework.framework.permutations import * + +# start_time = start_measuring_time() +# for suite in parameters.keys(): +# device_id = 0 +# device = ttnn.open_device(device_id=device_id) +# suite_vectors = list(permutations(parameters[suite])) +# print(len(suite_vectors)) +# for vector in suite_vectors: +# invalidate_res = invalidate_vector(vector) +# if invalidate_res[0]: +# print(f"Invalidated: {invalidate_res[1]}") +# continue +# try: +# passed, _ = run(**vector, device=device) +# # if passed[0] != True: +# # print(passed) +# except Exception as e: +# print(e) + +# # break + +# ttnn.close_device(device) + +# e2e_perf = stop_measuring_time(start_time) +# print(f"time {e2e_perf / 1000000000}s")