From bd6c51234a66519793fa78013df4e837584bc352 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 01:53:40 -0700 Subject: [PATCH 01/14] make changes to existing scheduled unet testing --- .github/workflows/test_iree.yml | 4 ++++ .../config_sdxl_scheduled_unet_cpu_llvm_task.json | 9 +-------- .../configs/config_sdxl_scheduled_unet_gpu_rocm.json | 9 +-------- .../real_weights_data_flags.txt | 8 +++++++- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml index 078693c1e..518dda5e1 100644 --- a/.github/workflows/test_iree.yml +++ b/.github/workflows/test_iree.yml @@ -104,12 +104,14 @@ jobs: sdxl-prompt-encoder-config-file: config_sdxl_prompt_encoder_cpu_llvm_task.json sdxl-unet-config-file: config_sdxl_scheduled_unet_cpu_llvm_task.json sdxl-vae-decode-config-file: config_sdxl_vae_decode_cpu_llvm_task.json + backend: cpu - name: gpu_mi250_rocm runs-on: nodai-amdgpu-mi250-x86-64 models-config-file: config_gpu_rocm_models.json sdxl-prompt-encoder-config-file: config_sdxl_prompt_encoder_gpu_rocm.json sdxl-unet-config-file: config_sdxl_scheduled_unet_gpu_rocm.json sdxl-vae-decode-config-file: config_sdxl_vae_decode_gpu_rocm.json + backend: rocm env: VENV_DIR: ${{ github.workspace }}/.venv IREE_TEST_FILES: ~/iree_tests_cache @@ -192,6 +194,8 @@ jobs: --timeout=1200 \ --durations=0 \ --config-files=${SDXL_UNET_CONFIG_FILE_PATH} + env: + IREE_TEST_BACKEND: ${{ matrix.backend }} - name: "Running real weights SDXL vae decode tests" id: vae diff --git a/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json index ec69c48ce..c331e8b34 100644 --- a/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json @@ -6,14 +6,7 @@ ], "iree_run_module_flags": [ "--device=local-task", - "--parameters=model=real_weights.irpa", - "--module=sdxl_scheduled_unet_pipeline_fp16_cpu.vmfb", - "--input=1x4x128x128xf16=@inference_input.0.bin", - "--input=2x64x2048xf16=@inference_input.1.bin", - "--input=2x1280xf16=@inference_input.2.bin", - "--input=1xf16=@inference_input.3.bin", - "--expected_output=1x4x128x128xf16=@inference_output.0.bin", - "--expected_f16_threshold=0.8f" + "--parameters=model=real_weights.irpa" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json b/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json index 5731caef9..a1c63e6c6 100644 --- a/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json +++ b/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json @@ -21,14 +21,7 @@ ], "iree_run_module_flags": [ "--device=hip", - "--parameters=model=real_weights.irpa", - "--module=sdxl_scheduled_unet_pipeline_fp16_rocm.vmfb", - "--input=1x4x128x128xf16=@inference_input.0.bin", - "--input=2x64x2048xf16=@inference_input.1.bin", - "--input=2x1280xf16=@inference_input.2.bin", - "--input=1xf16=@inference_input.3.bin", - "--expected_output=1x4x128x128xf16=@inference_output.0.bin", - "--expected_f16_threshold=0.7f" + "--parameters=model=real_weights.irpa" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt index d3f5a12fa..fca184fcb 100644 --- a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt @@ -1 +1,7 @@ - +--module=sdxl_scheduled_unet_pipeline_fp16_${IREE_TEST_BACKEND}.vmfb +--input=1x4x128x128xf16=@inference_input.0.bin +--input=2x64x2048xf16=@inference_input.1.bin +--input=2x1280xf16=@inference_input.2.bin +--input=1xf16=@inference_input.3.bin +--expected_output=1x4x128x128xf16=@inference_output.0.bin +--expected_f16_threshold=0.7f From 1f8fa6cc564655f5bcc0fd420bc25307fe517569 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 01:55:16 -0700 Subject: [PATCH 02/14] update tolerance --- .../sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt index fca184fcb..9ed04ee0b 100644 --- a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt @@ -4,4 +4,4 @@ --input=2x1280xf16=@inference_input.2.bin --input=1xf16=@inference_input.3.bin --expected_output=1x4x128x128xf16=@inference_output.0.bin ---expected_f16_threshold=0.7f +--expected_f16_threshold=0.8f From afbf4b78d3b2ff9e109723808899f437406ebb34 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 02:31:32 -0700 Subject: [PATCH 03/14] add functionality for run time env vars --- iree_tests/conftest.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/iree_tests/conftest.py b/iree_tests/conftest.py index 7c0e78e06..ed656c652 100644 --- a/iree_tests/conftest.py +++ b/iree_tests/conftest.py @@ -319,7 +319,13 @@ def __init__(self, spec, **kwargs): self.run_args = ["iree-run-module", f"--module={vmfb_name}"] self.run_args.extend(self.spec.iree_run_module_flags) - self.run_args.append(f"--flagfile={self.spec.data_flagfile_name}") + + # expand data flag file, so beter for logging and can use environmnet variables + flag_file_path = f"{self.test_cwd}/{self.spec.data_flagfile_name}" + file = open(flag_file_path) + for line in file: + self.run_args.append(line.rstrip()) + logging.getLogger().info(self.run_args) def runtest(self): # TODO(scotttodd): log files needed by the test (remote files / git LFS) @@ -385,6 +391,8 @@ def test_compile(self): compile_env["IREE_TEST_PATH_EXTENSION"] = os.getenv( "IREE_TEST_PATH_EXTENSION", default=str(self.test_cwd) ) + + # expand environment variable for logging path_extension = compile_env["IREE_TEST_PATH_EXTENSION"] cmd = subprocess.list2cmdline(self.compile_args) cmd = cmd.replace("${IREE_TEST_PATH_EXTENSION}", f"{path_extension}") @@ -401,8 +409,15 @@ def test_compile(self): def test_run(self): run_env = os.environ.copy() - cmd = subprocess.list2cmdline(self.run_args) + run_env["IREE_TEST_BACKEND"] = os.getenv( + "IREE_TEST_BACKEND", default="none" + ) + # expand environment variable for logging + backend = run_env["IREE_TEST_BACKEND"] + cmd = subprocess.list2cmdline(self.run_args) + cmd = cmd.replace("${IREE_TEST_BACKEND}", f"{backend}") + # TODO(scotttodd): expand flagfile(s) logging.getLogger().info( f"Launching run command:\n" # From ef3087ac5bd0a5b11891181f0f3fc29756118fa4 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 02:33:45 -0700 Subject: [PATCH 04/14] update prompt encoder testing for more generality as well --- .../config_sdxl_prompt_encoder_cpu_llvm_task.json | 9 +-------- .../configs/config_sdxl_prompt_encoder_gpu_rocm.json | 9 +-------- .../sdxl-prompt-encoder-tank/real_weights_data_flags.txt | 7 +++++++ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json index 0ac14f612..bb8e02f44 100644 --- a/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json @@ -6,14 +6,7 @@ ], "iree_run_module_flags": [ "--device=local-task", - "--parameters=model=real_weights.irpa", - "--input=1x64xi64=@inference_input.0.bin", - "--input=1x64xi64=@inference_input.1.bin", - "--input=1x64xi64=@inference_input.2.bin", - "--input=1x64xi64=@inference_input.3.bin", - "--expected_output=2x64x2048xf16=@inference_output.0.bin", - "--expected_output=2x1280xf16=@inference_output.1.bin", - "--expected_f16_threshold=1.0f" + "--parameters=model=real_weights.irpa" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json b/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json index 186a05488..a0ff72dc6 100644 --- a/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json +++ b/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json @@ -18,14 +18,7 @@ ], "iree_run_module_flags": [ "--device=hip", - "--parameters=model=real_weights.irpa", - "--input=1x64xi64=@inference_input.0.bin", - "--input=1x64xi64=@inference_input.1.bin", - "--input=1x64xi64=@inference_input.2.bin", - "--input=1x64xi64=@inference_input.3.bin", - "--expected_output=2x64x2048xf16=@inference_output.0.bin", - "--expected_output=2x1280xf16=@inference_output.1.bin", - "--expected_f16_threshold=1.0f" + "--parameters=model=real_weights.irpa" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt index e69de29bb..f43de4f3d 100644 --- a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt @@ -0,0 +1,7 @@ +--input=1x64xi64=@inference_input.0.bin +--input=1x64xi64=@inference_input.1.bin +--input=1x64xi64=@inference_input.2.bin +--input=1x64xi64=@inference_input.3.bin +--expected_output=2x64x2048xf16=@inference_output.0.bin +--expected_output=2x1280xf16=@inference_output.1.bin +--expected_f16_threshold=1.0f From b235dcb8dd4b7b60edb0df86acf93fd03344a4d9 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 02:36:05 -0700 Subject: [PATCH 05/14] generalize vae testing as well --- iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json | 5 +---- iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json | 5 +---- .../models/sdxl-vae-decode-tank/real_weights_data_flags.txt | 3 +++ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json index a6e8429ec..bb50c28d9 100644 --- a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json @@ -6,10 +6,7 @@ ], "iree_run_module_flags": [ "--device=local-task", - "--parameters=model=real_weights.irpa", - "--input=1x4x128x128xf16=@inference_input.0.bin", - "--expected_output=1x3x1024x1024xf16=@inference_output.0.bin", - "--expected_f16_threshold=0.02f" + "--parameters=model=real_weights.irpa" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json b/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json index 57a82e98a..e1b46f673 100644 --- a/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json +++ b/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json @@ -15,10 +15,7 @@ ], "iree_run_module_flags": [ "--device=hip", - "--parameters=model=real_weights.irpa", - "--input=1x4x128x128xf16=@inference_input.0.bin", - "--expected_output=1x3x1024x1024xf16=@inference_output.0.bin", - "--expected_f16_threshold=0.4f" + "--parameters=model=real_weights.irpa" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt index e69de29bb..a914234e3 100644 --- a/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt @@ -0,0 +1,3 @@ +--input=1x4x128x128xf16=@inference_input.0.bin +--expected_output=1x3x1024x1024xf16=@inference_output.0.bin +--expected_f16_threshold=0.02f From 10f6db3b242e62f583104c2a87ecdd7786c4f505 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 02:48:19 -0700 Subject: [PATCH 06/14] update tolerances for vae --- iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json | 3 ++- iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json | 3 ++- .../models/sdxl-vae-decode-tank/real_weights_data_flags.txt | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json index bb50c28d9..e7ad43742 100644 --- a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json @@ -6,7 +6,8 @@ ], "iree_run_module_flags": [ "--device=local-task", - "--parameters=model=real_weights.irpa" + "--parameters=model=real_weights.irpa", + "--expected_f16_threshold=0.05f" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json b/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json index e1b46f673..bb1347990 100644 --- a/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json +++ b/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json @@ -15,7 +15,8 @@ ], "iree_run_module_flags": [ "--device=hip", - "--parameters=model=real_weights.irpa" + "--parameters=model=real_weights.irpa", + "--expected_f16_threshold=0.4f" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt index a914234e3..5c8637a67 100644 --- a/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt @@ -1,3 +1,2 @@ --input=1x4x128x128xf16=@inference_input.0.bin --expected_output=1x3x1024x1024xf16=@inference_output.0.bin ---expected_f16_threshold=0.02f From 71947a78a4d1fdd55a0f439b1e9561629966ec7b Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 03:47:56 -0700 Subject: [PATCH 07/14] make changes to vae configs, real weight flags, and add in sd3-vae --- .../config_sdxl_vae_decode_cpu_llvm_task.json | 9 +++++++-- .../config_sdxl_vae_decode_gpu_rocm.json | 1 + .../models/sd-vae/sd3-vae-decode/model.mlirbc | 3 +++ .../real_weights_data_flags.txt | 3 +++ .../sd3-vae-decode/splat_data_flags.txt | 2 ++ .../models/sd-vae/sd3-vae-decode/splats.irpa | 3 +++ .../sd-vae/sd3-vae-decode/test_cases.json | 19 +++++++++++++++++++ .../sdxl-vae-decode-tank/model.mlirbc | 0 .../real_weights_data_flags.txt | 0 .../sdxl-vae-decode-tank/splat_data_flags.txt | 0 .../sdxl-vae-decode-tank/splats.irpa | 0 .../sdxl-vae-decode-tank/test_cases.json | 0 12 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 iree_tests/pytorch/models/sd-vae/sd3-vae-decode/model.mlirbc create mode 100644 iree_tests/pytorch/models/sd-vae/sd3-vae-decode/real_weights_data_flags.txt create mode 100644 iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splat_data_flags.txt create mode 100644 iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splats.irpa create mode 100644 iree_tests/pytorch/models/sd-vae/sd3-vae-decode/test_cases.json rename iree_tests/pytorch/models/{ => sd-vae}/sdxl-vae-decode-tank/model.mlirbc (100%) rename iree_tests/pytorch/models/{ => sd-vae}/sdxl-vae-decode-tank/real_weights_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => sd-vae}/sdxl-vae-decode-tank/splat_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => sd-vae}/sdxl-vae-decode-tank/splats.irpa (100%) rename iree_tests/pytorch/models/{ => sd-vae}/sdxl-vae-decode-tank/test_cases.json (100%) diff --git a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json index e7ad43742..a4873edbd 100644 --- a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json @@ -2,12 +2,17 @@ "config_name": "cpu_llvm_task", "iree_compile_flags" : [ "--iree-hal-target-backends=llvm-cpu", - "--iree-llvmcpu-target-cpu-features=host" + "--iree-llvmcpu-target-cpu-features=host", + "--iree-llvmcpu-fail-on-out-of-bounds-stack-allocation=false", + "--iree-llvmcpu-distribution-size=32", + "--iree-opt-const-eval=false", + "--iree-llvmcpu-enable-ukernels=all", + "--iree-global-opt-enable-quantized-matmul-reassociation" ], "iree_run_module_flags": [ "--device=local-task", "--parameters=model=real_weights.irpa", - "--expected_f16_threshold=0.05f" + "--expected_f32_threshold=0.01f" ], "skip_compile_tests": [], "skip_run_tests": [], diff --git a/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json b/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json index bb1347990..142e21ef8 100644 --- a/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json +++ b/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json @@ -16,6 +16,7 @@ "iree_run_module_flags": [ "--device=hip", "--parameters=model=real_weights.irpa", + "--expected_f32_threshold=0.6f", "--expected_f16_threshold=0.4f" ], "skip_compile_tests": [], diff --git a/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/model.mlirbc b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/model.mlirbc new file mode 100644 index 000000000..11acbcdc7 --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/model.mlirbc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab1d958d33b3cc291623fe9e0244690d8c07d9b883dfe50401028274c0c0cc8 +size 69621 diff --git a/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/real_weights_data_flags.txt new file mode 100644 index 000000000..9a331db21 --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/real_weights_data_flags.txt @@ -0,0 +1,3 @@ +--parameters=model=real_weights.irpa +--input=1x16x128x128xf16=@inference_input.0.bin +--expected_output=3x1024x1024xf32=@inference_output.0.bin diff --git a/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splat_data_flags.txt b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splat_data_flags.txt new file mode 100644 index 000000000..27011a98b --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splat_data_flags.txt @@ -0,0 +1,2 @@ +--input="1x16x128x128xf16" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splats.irpa b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splats.irpa new file mode 100644 index 000000000..d10eec9f1 --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/splats.irpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ea36e8272f34ac0626b4e81d56fda3747e65884cf0fe12b9cd683cee88f09dd +size 36864 diff --git a/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/test_cases.json b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/test_cases.json new file mode 100644 index 000000000..6a49268f9 --- /dev/null +++ b/iree_tests/pytorch/models/sd-vae/sd3-vae-decode/test_cases.json @@ -0,0 +1,19 @@ +{ + "file_format": "test_cases_v0", + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_files": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_files": [ + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-vae/inference_input.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-vae/inference_output.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-vae/real_weights.irpa" + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/model.mlirbc b/iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/model.mlirbc similarity index 100% rename from iree_tests/pytorch/models/sdxl-vae-decode-tank/model.mlirbc rename to iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/model.mlirbc diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sdxl-vae-decode-tank/real_weights_data_flags.txt rename to iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sdxl-vae-decode-tank/splat_data_flags.txt rename to iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/splats.irpa b/iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/sdxl-vae-decode-tank/splats.irpa rename to iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/splats.irpa diff --git a/iree_tests/pytorch/models/sdxl-vae-decode-tank/test_cases.json b/iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/sdxl-vae-decode-tank/test_cases.json rename to iree_tests/pytorch/models/sd-vae/sdxl-vae-decode-tank/test_cases.json From 133ec67db18df2cea420858ddcdaa17d8c406801 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 03:50:13 -0700 Subject: [PATCH 08/14] add in f16 tolerance for vae --- iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json index a4873edbd..25dc13997 100644 --- a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json @@ -12,7 +12,8 @@ "iree_run_module_flags": [ "--device=local-task", "--parameters=model=real_weights.irpa", - "--expected_f32_threshold=0.01f" + "--expected_f32_threshold=0.01f", + "--expected_f16_threshold=0.02f" ], "skip_compile_tests": [], "skip_run_tests": [], From db550fdf8df2a8553e90a7a2c3f410a1705d3e34 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 04:12:45 -0700 Subject: [PATCH 09/14] all changes needed for sd3-unet and sdxl-unet merge --- ...fig_sdxl_scheduled_unet_cpu_llvm_task.json | 14 +++++++---- .../config_sdxl_scheduled_unet_gpu_rocm.json | 4 +++- .../models/sd-unet/sd3-mmdit/model.mlirbc | 3 +++ .../sd3-mmdit/real_weights_data_flags.txt | 7 ++++++ .../sd-unet/sd3-mmdit/splat_data_flags.txt | 5 ++++ .../models/sd-unet/sd3-mmdit/splats.irpa | 3 +++ .../models/sd-unet/sd3-mmdit/test_cases.json | 22 ++++++++++++++++++ .../sdxl-scheduled-unet-3-tank/model.mlirbc | 0 .../real_weights_data_flags.txt | 2 +- ...sdxl_scheduled_unet_pipeline_fp16_cpu.vmfb | Bin ...dxl_scheduled_unet_pipeline_fp16_rocm.vmfb | Bin .../splat_data_flags.txt | 0 .../sdxl-scheduled-unet-3-tank/splats.irpa | 0 .../test_cases.json | 0 14 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 iree_tests/pytorch/models/sd-unet/sd3-mmdit/model.mlirbc create mode 100644 iree_tests/pytorch/models/sd-unet/sd3-mmdit/real_weights_data_flags.txt create mode 100644 iree_tests/pytorch/models/sd-unet/sd3-mmdit/splat_data_flags.txt create mode 100644 iree_tests/pytorch/models/sd-unet/sd3-mmdit/splats.irpa create mode 100644 iree_tests/pytorch/models/sd-unet/sd3-mmdit/test_cases.json rename iree_tests/pytorch/models/{ => sd-unet}/sdxl-scheduled-unet-3-tank/model.mlirbc (100%) rename iree_tests/pytorch/models/{ => sd-unet}/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt (88%) rename iree_tests/pytorch/models/{ => sd-unet}/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_cpu.vmfb (100%) rename iree_tests/pytorch/models/{ => sd-unet}/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_rocm.vmfb (100%) rename iree_tests/pytorch/models/{ => sd-unet}/sdxl-scheduled-unet-3-tank/splat_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => sd-unet}/sdxl-scheduled-unet-3-tank/splats.irpa (100%) rename iree_tests/pytorch/models/{ => sd-unet}/sdxl-scheduled-unet-3-tank/test_cases.json (100%) diff --git a/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json index c331e8b34..1f900bc2d 100644 --- a/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json @@ -2,7 +2,12 @@ "config_name": "cpu_llvm_task", "iree_compile_flags" : [ "--iree-hal-target-backends=llvm-cpu", - "--iree-llvmcpu-target-cpu-features=host" + "--iree-llvmcpu-target-cpu-features=host", + "--iree-llvmcpu-fail-on-out-of-bounds-stack-allocation=false", + "--iree-llvmcpu-distribution-size=32", + "--iree-opt-const-eval=false", + "--iree-llvmcpu-enable-ukernels=all", + "--iree-global-opt-enable-quantized-matmul-reassociation" ], "iree_run_module_flags": [ "--device=local-task", @@ -10,8 +15,9 @@ ], "skip_compile_tests": [], "skip_run_tests": [], - "expected_compile_failures": [], - "expected_run_failures": [ + "expected_compile_failures": [ "sdxl-scheduled-unet-3-tank", - ] + "sd3-mmdit" + ], + "expected_run_failures": [] } diff --git a/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json b/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json index a1c63e6c6..5d01fb711 100644 --- a/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json +++ b/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json @@ -25,6 +25,8 @@ ], "skip_compile_tests": [], "skip_run_tests": [], - "expected_compile_failures": [], + "expected_compile_failures": [ + "sd3-mmdit", + ], "expected_run_failures": [] } diff --git a/iree_tests/pytorch/models/sd-unet/sd3-mmdit/model.mlirbc b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/model.mlirbc new file mode 100644 index 000000000..980aebec2 --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/model.mlirbc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ba72246a5d83ee2a9849a70bf62295c8a79f88f16dee71eeb08bd0ba8cfe15 +size 430357 diff --git a/iree_tests/pytorch/models/sd-unet/sd3-mmdit/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/real_weights_data_flags.txt new file mode 100644 index 000000000..b4da9146c --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/real_weights_data_flags.txt @@ -0,0 +1,7 @@ +--parameters=model=real_weights.irpa +--input=2x16x128x128xf16=@inference_input.0.bin +--input=2x154x4096xf16=@inference_input.1.bin +--input=2x2048xf16=@inference_input.2.bin +--input=1xf16=@inference_input.3.bin +--expected_output=2x16x128x128xf32=@inference_output.0.bin +--expected_f16_threshold=1.0f diff --git a/iree_tests/pytorch/models/sd-unet/sd3-mmdit/splat_data_flags.txt b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/splat_data_flags.txt new file mode 100644 index 000000000..473cba22f --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/splat_data_flags.txt @@ -0,0 +1,5 @@ +--input="2x16x128x128xf16" +--input="2x154x4096xf16" +--input="2x2048xf16" +--input="1xf16" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sd-unet/sd3-mmdit/splats.irpa b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/splats.irpa new file mode 100644 index 000000000..42a66d94a --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/splats.irpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c7e4c5a1303b4576eefefd0e1ad720bc065b319a66f907a4448b1d448ddbbe3 +size 98304 diff --git a/iree_tests/pytorch/models/sd-unet/sd3-mmdit/test_cases.json b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/test_cases.json new file mode 100644 index 000000000..4c2cdcef5 --- /dev/null +++ b/iree_tests/pytorch/models/sd-unet/sd3-mmdit/test_cases.json @@ -0,0 +1,22 @@ +{ + "file_format": "test_cases_v0", + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_files": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_files": [ + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-mmdit/inference_input.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-mmdit/inference_input.1.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-mmdit/inference_input.2.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-mmdit/inference_input.3.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-mmdit/inference_output.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-mmdit/real_weights.irpa" + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/model.mlirbc b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/model.mlirbc similarity index 100% rename from iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/model.mlirbc rename to iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/model.mlirbc diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt similarity index 88% rename from iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt rename to iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt index 9ed04ee0b..fca184fcb 100644 --- a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt +++ b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/real_weights_data_flags.txt @@ -4,4 +4,4 @@ --input=2x1280xf16=@inference_input.2.bin --input=1xf16=@inference_input.3.bin --expected_output=1x4x128x128xf16=@inference_output.0.bin ---expected_f16_threshold=0.8f +--expected_f16_threshold=0.7f diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_cpu.vmfb b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_cpu.vmfb similarity index 100% rename from iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_cpu.vmfb rename to iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_cpu.vmfb diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_rocm.vmfb b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_rocm.vmfb similarity index 100% rename from iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_rocm.vmfb rename to iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/sdxl_scheduled_unet_pipeline_fp16_rocm.vmfb diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splat_data_flags.txt rename to iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splats.irpa b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/splats.irpa rename to iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/splats.irpa diff --git a/iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/test_cases.json b/iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank/test_cases.json rename to iree_tests/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank/test_cases.json From e2559153789df4359ca71ee3eb1e42ed0f36dff4 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 04:35:17 -0700 Subject: [PATCH 10/14] all config, tolerance updates for prompt encoder --- ...fig_sdxl_prompt_encoder_cpu_llvm_task.json | 7 +++++- .../config_sdxl_prompt_encoder_gpu_rocm.json | 4 ++- .../sd-clip/sd3-prompt-encoder/model.mlirbc | 3 +++ .../real_weights_data_flags.txt | 9 +++++++ .../sd3-prompt-encoder/splat_data_flags.txt | 7 ++++++ .../sd-clip/sd3-prompt-encoder/splats.irpa | 3 +++ .../sd3-prompt-encoder/test_cases.json | 25 +++++++++++++++++++ .../sdxl-prompt-encoder-tank/model.mlirbc | 0 .../prompt_encoder_info.json | 24 ++++++++++++++++++ .../real_weights_data_flags.txt | 0 .../splat_data_flags.txt | 0 .../sdxl-prompt-encoder-tank/splats.irpa | 0 .../sdxl-prompt-encoder-tank/test_cases.json | 0 13 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/model.mlirbc create mode 100644 iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/real_weights_data_flags.txt create mode 100644 iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splat_data_flags.txt create mode 100644 iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splats.irpa create mode 100644 iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/test_cases.json rename iree_tests/pytorch/models/{ => sd-clip}/sdxl-prompt-encoder-tank/model.mlirbc (100%) create mode 100644 iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json rename iree_tests/pytorch/models/{ => sd-clip}/sdxl-prompt-encoder-tank/real_weights_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => sd-clip}/sdxl-prompt-encoder-tank/splat_data_flags.txt (100%) rename iree_tests/pytorch/models/{ => sd-clip}/sdxl-prompt-encoder-tank/splats.irpa (100%) rename iree_tests/pytorch/models/{ => sd-clip}/sdxl-prompt-encoder-tank/test_cases.json (100%) diff --git a/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json b/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json index bb8e02f44..e580cf2a3 100644 --- a/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json +++ b/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json @@ -2,7 +2,12 @@ "config_name": "cpu_llvm_task", "iree_compile_flags" : [ "--iree-hal-target-backends=llvm-cpu", - "--iree-llvmcpu-target-cpu-features=host" + "--iree-llvmcpu-target-cpu-features=host", + "--iree-llvmcpu-fail-on-out-of-bounds-stack-allocation=false", + "--iree-llvmcpu-distribution-size=32", + "--iree-opt-const-eval=false", + "--iree-llvmcpu-enable-ukernels=all", + "--iree-global-opt-enable-quantized-matmul-reassociation" ], "iree_run_module_flags": [ "--device=local-task", diff --git a/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json b/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json index a0ff72dc6..9ce209882 100644 --- a/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json +++ b/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json @@ -22,6 +22,8 @@ ], "skip_compile_tests": [], "skip_run_tests": [], - "expected_compile_failures": [], + "expected_compile_failures": [ + "sd3-prompt-encoder", + ], "expected_run_failures": [] } diff --git a/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/model.mlirbc b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/model.mlirbc new file mode 100644 index 000000000..acb2cc88a --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/model.mlirbc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cc1456e1b5e2e5138db5bc00421b9f8e46710745b5f847579c1036b1df56f7c +size 969269 diff --git a/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/real_weights_data_flags.txt new file mode 100644 index 000000000..e3ddd86e0 --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/real_weights_data_flags.txt @@ -0,0 +1,9 @@ +--input=1x77x2xi64=@inference_input.0.bin +--input=1x77x2xi64=@inference_input.1.bin +--input=1x77x2xi64=@inference_input.2.bin +--input=1x77x2xi64=@inference_input.3.bin +--input=1x77x2xi64=@inference_input.4.bin +--input=1x77x2xi64=@inference_input.5.bin +--expected_output=2x154x4096xf32=@inference_output.0.bin +--expected_output=2x2048xf32=@inference_output.1.bin +--expected_f32_threshold=0.15f diff --git a/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splat_data_flags.txt b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splat_data_flags.txt new file mode 100644 index 000000000..774808296 --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splat_data_flags.txt @@ -0,0 +1,7 @@ +--input="1x77x2xi64" +--input="1x77x2xi64" +--input="1x77x2xi64" +--input="1x77x2xi64" +--input="1x77x2xi64" +--input="1x77x2xi64" +--parameters=splats.irpa diff --git a/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splats.irpa b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splats.irpa new file mode 100644 index 000000000..78d9b4541 --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/splats.irpa @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a7802c447ebd5951ffab12c8a5c60d75c4946393cd7af99e7b7539c4ebbe7a +size 327680 diff --git a/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/test_cases.json b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/test_cases.json new file mode 100644 index 000000000..ce0452114 --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip/sd3-prompt-encoder/test_cases.json @@ -0,0 +1,25 @@ +{ + "file_format": "test_cases_v0", + "test_cases": [ + { + "name": "splats", + "runtime_flagfile": "splat_data_flags.txt", + "remote_files": [] + }, + { + "name": "real_weights", + "runtime_flagfile": "real_weights_data_flags.txt", + "remote_files": [ + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_input.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_input.1.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_input.2.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_input.3.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_input.4.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_input.5.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_output.0.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/inference_output.1.bin", + "https://sharkpublic.blob.core.windows.net/sharkpublic/sai/sd3-prompt-encoder/real_weights.irpa" + ] + } + ] +} diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/model.mlirbc b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/model.mlirbc similarity index 100% rename from iree_tests/pytorch/models/sdxl-prompt-encoder-tank/model.mlirbc rename to iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/model.mlirbc diff --git a/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json new file mode 100644 index 000000000..69bd5a8b3 --- /dev/null +++ b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json @@ -0,0 +1,24 @@ +{ +"stream-aggregate": { + "global": { + "constant-count": 436, + "constant-size": 1280, + "variable-count": 0, + "variable-size": 0 + }, + "synchronization": { + "await-count": 0 + }, + "execution": { + "submission-count": 3, + "transient-memory-size": 4529224, + "fill-count": 2, + "copy-count": 7, + "dispatch-count": 1571, + "call-count": 0 + }, + "executable": { + "executable-count": 40 + } +} +} diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/real_weights_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sdxl-prompt-encoder-tank/real_weights_data_flags.txt rename to iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/real_weights_data_flags.txt diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splat_data_flags.txt b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/splat_data_flags.txt similarity index 100% rename from iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splat_data_flags.txt rename to iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/splat_data_flags.txt diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splats.irpa b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/splats.irpa similarity index 100% rename from iree_tests/pytorch/models/sdxl-prompt-encoder-tank/splats.irpa rename to iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/splats.irpa diff --git a/iree_tests/pytorch/models/sdxl-prompt-encoder-tank/test_cases.json b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/test_cases.json similarity index 100% rename from iree_tests/pytorch/models/sdxl-prompt-encoder-tank/test_cases.json rename to iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/test_cases.json From 306d2a061791032f709f221189d54efb24c50611 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 04:38:28 -0700 Subject: [PATCH 11/14] delete file pushed by accident --- .../prompt_encoder_info.json | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json diff --git a/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json b/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json deleted file mode 100644 index 69bd5a8b3..000000000 --- a/iree_tests/pytorch/models/sd-clip/sdxl-prompt-encoder-tank/prompt_encoder_info.json +++ /dev/null @@ -1,24 +0,0 @@ -{ -"stream-aggregate": { - "global": { - "constant-count": 436, - "constant-size": 1280, - "variable-count": 0, - "variable-size": 0 - }, - "synchronization": { - "await-count": 0 - }, - "execution": { - "submission-count": 3, - "transient-memory-size": 4529224, - "fill-count": 2, - "copy-count": 7, - "dispatch-count": 1571, - "call-count": 0 - }, - "executable": { - "executable-count": 40 - } -} -} From c34665d92111e8790c0ff6d6cec4cc5e23f72f82 Mon Sep 17 00:00:00 2001 From: saienduri Date: Sat, 15 Jun 2024 04:52:04 -0700 Subject: [PATCH 12/14] changes to adapt to all the changes made to the testing suite --- .github/workflows/test_iree.yml | 24 +++++++++---------- iree_tests/benchmarks/benchmark_sdxl_rocm.py | 6 ++--- .../configs/config_gpu_rocm_models.json | 5 +++- .../config_pytorch_models_cpu_llvm_task.json | 5 +++- ...nfig_sd_prompt_encoder_cpu_llvm_task.json} | 0 ...=> config_sd_prompt_encoder_gpu_rocm.json} | 0 ...nfig_sd_scheduled_unet_cpu_llvm_task.json} | 0 ...=> config_sd_scheduled_unet_gpu_rocm.json} | 0 ...> config_sd_vae_decode_cpu_llvm_task.json} | 0 ...son => config_sd_vae_decode_gpu_rocm.json} | 0 10 files changed, 23 insertions(+), 17 deletions(-) rename iree_tests/configs/{config_sdxl_prompt_encoder_cpu_llvm_task.json => config_sd_prompt_encoder_cpu_llvm_task.json} (100%) rename iree_tests/configs/{config_sdxl_prompt_encoder_gpu_rocm.json => config_sd_prompt_encoder_gpu_rocm.json} (100%) rename iree_tests/configs/{config_sdxl_scheduled_unet_cpu_llvm_task.json => config_sd_scheduled_unet_cpu_llvm_task.json} (100%) rename iree_tests/configs/{config_sdxl_scheduled_unet_gpu_rocm.json => config_sd_scheduled_unet_gpu_rocm.json} (100%) rename iree_tests/configs/{config_sdxl_vae_decode_cpu_llvm_task.json => config_sd_vae_decode_cpu_llvm_task.json} (100%) rename iree_tests/configs/{config_sdxl_vae_decode_gpu_rocm.json => config_sd_vae_decode_gpu_rocm.json} (100%) diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml index 518dda5e1..2f24842e6 100644 --- a/.github/workflows/test_iree.yml +++ b/.github/workflows/test_iree.yml @@ -101,16 +101,16 @@ jobs: - name: cpu_llvm_task runs-on: nodai-amdgpu-w7900-x86-64 models-config-file: config_pytorch_models_cpu_llvm_task.json - sdxl-prompt-encoder-config-file: config_sdxl_prompt_encoder_cpu_llvm_task.json - sdxl-unet-config-file: config_sdxl_scheduled_unet_cpu_llvm_task.json - sdxl-vae-decode-config-file: config_sdxl_vae_decode_cpu_llvm_task.json + sdxl-prompt-encoder-config-file: config_sd_prompt_encoder_cpu_llvm_task.json + sdxl-unet-config-file: config_sd_scheduled_unet_cpu_llvm_task.json + sdxl-vae-decode-config-file: config_sd_vae_decode_cpu_llvm_task.json backend: cpu - name: gpu_mi250_rocm runs-on: nodai-amdgpu-mi250-x86-64 models-config-file: config_gpu_rocm_models.json - sdxl-prompt-encoder-config-file: config_sdxl_prompt_encoder_gpu_rocm.json - sdxl-unet-config-file: config_sdxl_scheduled_unet_gpu_rocm.json - sdxl-vae-decode-config-file: config_sdxl_vae_decode_gpu_rocm.json + sdxl-prompt-encoder-config-file: config_sd_prompt_encoder_gpu_rocm.json + sdxl-unet-config-file: config_sd_scheduled_unet_gpu_rocm.json + sdxl-vae-decode-config-file: config_sd_vae_decode_gpu_rocm.json backend: rocm env: VENV_DIR: ${{ github.workspace }}/.venv @@ -165,12 +165,12 @@ jobs: --durations=0 \ --config-files=${MODELS_CONFIG_FILE_PATH} - - name: "Running real weights SDXL prompt encoder tests" + - name: "Running real weights SDXL + SD3 prompt encoder tests" id: prompt_encoder if: ${{ !cancelled() }} run: | source ${VENV_DIR}/bin/activate - pytest iree_tests/pytorch/models/sdxl-prompt-encoder-tank \ + pytest iree_tests/pytorch/models/sd-clip \ -rpfE \ -k real_weights \ --no-skip-tests-missing-files \ @@ -180,12 +180,12 @@ jobs: --durations=0 \ --config-files=${SDXL_PROMPT_ENCODER_CONFIG_FILE_PATH} - - name: "Running real weights SDXL scheduled unet tests" + - name: "Running real weights SDXL + SD3 scheduled unet/mmdit tests" id: unet if: ${{ !cancelled() }} run: | source ${VENV_DIR}/bin/activate - pytest iree_tests/pytorch/models/sdxl-scheduled-unet-3-tank \ + pytest iree_tests/pytorch/models/sd-unet \ -rpfE \ -k real_weights \ --no-skip-tests-missing-files \ @@ -197,12 +197,12 @@ jobs: env: IREE_TEST_BACKEND: ${{ matrix.backend }} - - name: "Running real weights SDXL vae decode tests" + - name: "Running real weights SDXL + SD3 vae decode tests" id: vae if: ${{ !cancelled() }} run: | source ${VENV_DIR}/bin/activate - pytest iree_tests/pytorch/models/sdxl-vae-decode-tank \ + pytest iree_tests/pytorch/models/sd-vae \ -rpfE \ -k real_weights \ --no-skip-tests-missing-files \ diff --git a/iree_tests/benchmarks/benchmark_sdxl_rocm.py b/iree_tests/benchmarks/benchmark_sdxl_rocm.py index 884442ea4..e38eb148b 100644 --- a/iree_tests/benchmarks/benchmark_sdxl_rocm.py +++ b/iree_tests/benchmarks/benchmark_sdxl_rocm.py @@ -12,9 +12,9 @@ benchmark_dir = os.path.dirname(os.path.realpath(__file__)) iree_root = os.path.dirname(benchmark_dir) -prompt_encoder_dir = f"{iree_root}/pytorch/models/sdxl-prompt-encoder-tank" -scheduled_unet_dir = f"{iree_root}/pytorch/models/sdxl-scheduled-unet-3-tank" -vae_decode_dir = f"{iree_root}/pytorch/models/sdxl-vae-decode-tank" +prompt_encoder_dir = f"{iree_root}/pytorch/models/sd-clip/sdxl-prompt-encoder-tank" +scheduled_unet_dir = f"{iree_root}/pytorch/models/sd-unet/sdxl-scheduled-unet-3-tank" +vae_decode_dir = f"{iree_root}/pytorch/models/sd-vae/sdxl-vae-decode-tank" def run_iree_command(args: Sequence[str] = ()): command = "Exec:", " ".join(args) diff --git a/iree_tests/configs/config_gpu_rocm_models.json b/iree_tests/configs/config_gpu_rocm_models.json index fce62c4b1..4d1795c0f 100644 --- a/iree_tests/configs/config_gpu_rocm_models.json +++ b/iree_tests/configs/config_gpu_rocm_models.json @@ -12,7 +12,10 @@ "skip_compile_tests": [ "sdxl-scheduled-unet-3-tank", "sdxl-vae-decode-tank", - "sdxl-prompt-encoder-tank" + "sdxl-prompt-encoder-tank", + "sd3-mmdit", + "sd3-vae-decode", + "sd3-prompt-encoder" ], "skip_run_tests": [], "expected_compile_failures": [ diff --git a/iree_tests/configs/config_pytorch_models_cpu_llvm_task.json b/iree_tests/configs/config_pytorch_models_cpu_llvm_task.json index 065a28854..6b9f49469 100644 --- a/iree_tests/configs/config_pytorch_models_cpu_llvm_task.json +++ b/iree_tests/configs/config_pytorch_models_cpu_llvm_task.json @@ -10,7 +10,10 @@ "skip_compile_tests": [ "sdxl-scheduled-unet-3-tank", "sdxl-vae-decode-tank", - "sdxl-prompt-encoder-tank" + "sdxl-prompt-encoder-tank", + "sd3-mmdit", + "sd3-vae-decode", + "sd3-prompt-encoder" ], "skip_run_tests": [], "expected_compile_failures": [ diff --git a/iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json b/iree_tests/configs/config_sd_prompt_encoder_cpu_llvm_task.json similarity index 100% rename from iree_tests/configs/config_sdxl_prompt_encoder_cpu_llvm_task.json rename to iree_tests/configs/config_sd_prompt_encoder_cpu_llvm_task.json diff --git a/iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json b/iree_tests/configs/config_sd_prompt_encoder_gpu_rocm.json similarity index 100% rename from iree_tests/configs/config_sdxl_prompt_encoder_gpu_rocm.json rename to iree_tests/configs/config_sd_prompt_encoder_gpu_rocm.json diff --git a/iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json b/iree_tests/configs/config_sd_scheduled_unet_cpu_llvm_task.json similarity index 100% rename from iree_tests/configs/config_sdxl_scheduled_unet_cpu_llvm_task.json rename to iree_tests/configs/config_sd_scheduled_unet_cpu_llvm_task.json diff --git a/iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json b/iree_tests/configs/config_sd_scheduled_unet_gpu_rocm.json similarity index 100% rename from iree_tests/configs/config_sdxl_scheduled_unet_gpu_rocm.json rename to iree_tests/configs/config_sd_scheduled_unet_gpu_rocm.json diff --git a/iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json b/iree_tests/configs/config_sd_vae_decode_cpu_llvm_task.json similarity index 100% rename from iree_tests/configs/config_sdxl_vae_decode_cpu_llvm_task.json rename to iree_tests/configs/config_sd_vae_decode_cpu_llvm_task.json diff --git a/iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json b/iree_tests/configs/config_sd_vae_decode_gpu_rocm.json similarity index 100% rename from iree_tests/configs/config_sdxl_vae_decode_gpu_rocm.json rename to iree_tests/configs/config_sd_vae_decode_gpu_rocm.json From 2e4c194e9983256a93f246d09fb05662adabc659 Mon Sep 17 00:00:00 2001 From: saienduri <77521230+saienduri@users.noreply.github.com> Date: Sat, 15 Jun 2024 05:24:21 -0700 Subject: [PATCH 13/14] spelling mistake --- iree_tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iree_tests/conftest.py b/iree_tests/conftest.py index ed656c652..808a1febb 100644 --- a/iree_tests/conftest.py +++ b/iree_tests/conftest.py @@ -320,7 +320,7 @@ def __init__(self, spec, **kwargs): self.run_args = ["iree-run-module", f"--module={vmfb_name}"] self.run_args.extend(self.spec.iree_run_module_flags) - # expand data flag file, so beter for logging and can use environmnet variables + # expand data flag file, so beter for logging and can use environment variables flag_file_path = f"{self.test_cwd}/{self.spec.data_flagfile_name}" file = open(flag_file_path) for line in file: From 0d52c6fbf74c6ed6951d0cded813629bcb1e608f Mon Sep 17 00:00:00 2001 From: saienduri <77521230+saienduri@users.noreply.github.com> Date: Sat, 15 Jun 2024 05:46:18 -0700 Subject: [PATCH 14/14] remove debug code --- iree_tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/iree_tests/conftest.py b/iree_tests/conftest.py index 808a1febb..dfbfbbe61 100644 --- a/iree_tests/conftest.py +++ b/iree_tests/conftest.py @@ -325,7 +325,6 @@ def __init__(self, spec, **kwargs): file = open(flag_file_path) for line in file: self.run_args.append(line.rstrip()) - logging.getLogger().info(self.run_args) def runtest(self): # TODO(scotttodd): log files needed by the test (remote files / git LFS)