diff --git a/.github/scripts/check-ut.py b/.github/scripts/check-ut.py index b90ba4353..c21f9ceda 100644 --- a/.github/scripts/check-ut.py +++ b/.github/scripts/check-ut.py @@ -138,7 +138,6 @@ def print_failures(failure_list=None): def generate_failures_log(): if not failures: - print("No failures found, skipping log file creation.") return for case in failures: @@ -245,6 +244,8 @@ def determine_category(ut): return 'op_extended' elif ut == 'op_transformers': return 'op_transformers' + elif ut == 'test_xpu': + return 'test_xpu' elif 'op_ut' in ut: return 'op_ut' else: @@ -296,7 +297,6 @@ def process_xml_file(xml_file): def generate_passed_log(): if not passed_cases: - print("No passed cases found, skipping log file creation.") return for category, category_passed in passed_by_category.items(): diff --git a/.github/scripts/ut_result_check.sh b/.github/scripts/ut_result_check.sh index efdd92dcb..0e6b95ec4 100644 --- a/.github/scripts/ut_result_check.sh +++ b/.github/scripts/ut_result_check.sh @@ -25,6 +25,10 @@ check_new_failed() { # Filter the same content from file_UT as file_known_issue echo "Filtering $file_known_issue for $file_UT" + if grep -q $'\r' "$file_UT"; then + echo "Detected log from windows" + sed -i 's/\r$//' "$file_UT" + fi grep -vFxf "$file_known_issue" "$file_UT" > "$output_file" echo -e "\n\033[1;31m[New failed cases Summary]\033[0m" @@ -54,6 +58,10 @@ check_passed_known_issues() { return 1 fi echo "Checking for known issues that are now passing in $file_passed_UT" + if grep -q $'\r' "$file_passed_UT"; then + echo "Detected log from windows" + sed -i 's/\r$//' "$file_passed_UT" + fi grep -Fxf "$file_passed_UT" "$file_known_issue" > "$output_file" echo -e "\n\033[1;32m[New passed cases Summary]\033[0m" if [[ -s "$output_file" ]]; then @@ -74,6 +82,7 @@ check_test_cases() { ["op_regression_dev1"]=1 ["op_transformers"]=237 ["op_ut"]=120408 + ["test_xpu"]=69 ) if [[ ! -f "$log_file" ]]; then @@ -115,7 +124,7 @@ check_test_cases() { } -if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' || "${ut_suite}" == 'op_ut' ]]; then +if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' || "${ut_suite}" == 'op_extended' || "${ut_suite}" == 'op_transformers' || "${ut_suite}" == 'op_ut' || "${ut_suite}" == 'test_xpu' ]]; then echo -e "=========================================================================" echo -e "Show Failed cases in ${ut_suite}" echo -e "=========================================================================" @@ -188,35 +197,6 @@ if [[ "${ut_suite}" == 'op_regression' || "${ut_suite}" == 'op_regression_dev1' fi fi -if [[ "${ut_suite}" == 'torch_xpu' ]]; then - echo "Pytorch XPU binary UT checking" - cd ../../pytorch || exit - for xpu_case in build/bin/*{xpu,sycl}*; do - if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then - case_name=$(basename "$xpu_case") - cd ../ut_log/torch_xpu || exit - grep -E "FAILED" binary_ut_"${ut_suite}"_"${case_name}"_test.log | awk '{print $2}' > ./binary_ut_"${ut_suite}"_"${case_name}"_failed.log - wc -l < "./binary_ut_${ut_suite}_${case_name}_failed.log" | tee -a ./binary_ut_"${ut_suite}"_failed_summary.log - grep -E "PASSED|Pass" binary_ut_"${ut_suite}"_"${case_name}"_test.log | awk '{print $2}' > ./binary_ut_"${ut_suite}"_"${case_name}"_passed.log - wc -l < "./binary_ut_${ut_suite}_${case_name}_passed.log" | tee -a ./binary_ut_"${ut_suite}"_passed_summary.log - cd - || exit - fi - done - echo -e "=========================================================================" - echo -e "Show Failed cases in ${ut_suite}" - echo -e "=========================================================================" - cd ../ut_log/torch_xpu || exit - cat "./binary_ut_${ut_suite}_${case_name}_failed.log" - num_failed_binary_ut=$(awk '{sum += $1};END {print sum}' binary_ut_"${ut_suite}"_failed_summary.log) - num_passed_binary_ut=$(awk '{sum += $1};END {print sum}' binary_ut_"${ut_suite}"_passed_summary.log) - ((num_failed=num_failed_binary_ut)) - if [[ $num_failed -gt 0 ]] || [[ $num_passed_binary_ut -le 0 ]]; then - echo -e "[ERROR] UT ${ut_suite} test Fail" - exit 1 - else - echo -e "[PASS] UT ${ut_suite} test Pass" - fi -fi if [[ "${ut_suite}" == 'xpu_distributed' ]]; then grep -E "^FAILED" xpu_distributed_test.log | awk '{print $2}' > ./"${ut_suite}"_xpu_distributed_test_failed.log grep "PASSED" xpu_distributed_test.log | awk '{print $1}' > ./"${ut_suite}"_xpu_distributed_test_passed.log diff --git a/.github/workflows/_windows_ut.yml b/.github/workflows/_windows_ut.yml index 14129446e..e650e32d1 100644 --- a/.github/workflows/_windows_ut.yml +++ b/.github/workflows/_windows_ut.yml @@ -112,7 +112,7 @@ jobs: call conda activate windows_ci cd ../pytorch pip install -r requirements.txt - pip install cmake setuptools==72.1.0 clang-format + pip install cmake setuptools clang-format pip install mkl-static mkl-include set USE_STATIC_MKL=1 copy "%CONDA_PREFIX%\Library\bin\libiomp*5md.dll" .\torch\lib @@ -123,7 +123,8 @@ jobs: set CMAKE_PREFIX_PATH="%CONDA_PREFIX%\Library" ) python setup.py clean - set MAX_JOBS=4 + set MAX_JOBS=32 + set TORCH_XPU_ARCH_LIST=mtl-h,bmg,lnl-m python setup.py bdist_wheel > build_torch_wheel_log.log echo "[INFO] begin to install torch whls" for /r C:\actions-runner\_work\torch-xpu-ops\pytorch\dist %%i in (torch*.whl) do ( @@ -141,24 +142,22 @@ jobs: python -c "import torch; print(torch.__config__.show())" python -c "import torch; print(torch.__config__.parallel_info())" python -c "import torch; print(torch.__config__.torch.xpu.device_count())" - - name: Upload Windows build log if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 with: name: Torch-XPU-Windows-Log-${{ github.event.pull_request.number || github.sha }} path: 'C:\actions-runner\_work\torch-xpu-ops\pytorch\build_torch_wheel_log.log' - - name: Upload Windows binary if: ${{ ! cancelled() }} uses: actions/upload-artifact@v4 with: name: Torch-XPU-Windows-Binary-${{ github.event.pull_request.number || github.sha }} path: 'C:\actions-runner\_work\torch-xpu-ops\pytorch\dist' - - name: Run XPU OP Extended UT if: contains(inputs.ut, 'op_extended') || github.event_name == 'schedule' shell: cmd + continue-on-error: true run: | call "C:\ProgramData\miniforge3\Scripts\activate.bat" call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" @@ -168,9 +167,12 @@ jobs: cd ../pytorch/third_party/torch-xpu-ops/test/xpu/extended/ python run_test_with_skip_mtl.py + if not exist "%GITHUB_WORKSPACE%\ut_log" mkdir "%GITHUB_WORKSPACE%\ut_log" + copy op_extended.xml %GITHUB_WORKSPACE%\ut_log /Y - name: Run Test XPU UT - if: contains(inputs.ut, 'torch_xpu') || github.event_name == 'schedule' + if: contains(inputs.ut, 'test_xpu') || github.event_name == 'schedule' shell: cmd + continue-on-error: true run: | call "C:\ProgramData\miniforge3\Scripts\activate.bat" call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" @@ -178,3 +180,103 @@ jobs: call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" cd ../pytorch/third_party/torch-xpu-ops/test/xpu/ python run_test_win_with_skip_mtl.py + + if not exist "%GITHUB_WORKSPACE%\ut_log" mkdir "%GITHUB_WORKSPACE%\ut_log" + copy test_xpu.xml %GITHUB_WORKSPACE%\ut_log /Y + - name: UT Test Results Summary + shell: cmd + run: | + call conda activate windows_ci + pip install junitparser + echo "GITHUB_WORKSPACE: %GITHUB_WORKSPACE%" + for %%i in ("%GITHUB_WORKSPACE%\ut_log\*.xml") do ( + python .\.github\scripts\check-ut.py "%%i" >> "%GITHUB_STEP_SUMMARY%" + ) + @echo off + + REM Check the failure logs + if exist "%GITHUB_WORKSPACE%\failures*.log" ( + echo Exist Failure logs + echo Found Failure logs as below: + for %%f in ("%GITHUB_WORKSPACE%\failures*.log") do ( + echo - %%f + copy "%%f" "%GITHUB_WORKSPACE%\ut_log\" + ) + echo Failure logs Copied + ) else ( + echo No Failure logs + ) + + REM Copied the passed logs + if exist "passed*.log" ( + copy "passed*.log" "%GITHUB_WORKSPACE%\ut_log\" + echo Passed logs Copied + ) else ( + echo No Passed logs + ) + + REM Copied the Summary logs + if exist "category*.log" ( + copy "category*.log" "%GITHUB_WORKSPACE%\ut_log\" + echo Category logs Copied + ) else ( + echo No Category logs + ) + - name: Upload Inductor XPU UT Log + if: ${{ ! cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-Windows + path: "${{ github.workspace }}/ut_log" + if-no-files-found: ignore + + summary: + needs: [ut_test] + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout torch-xpu-ops + uses: actions/checkout@v4 + - name: Download XPU UT Logs + uses: actions/download-artifact@v4 + with: + name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-Windows + path: ${{ github.workspace }}/ut_log + - name: Check UT Results + shell: bash + run: | + ls -al ${{ github.workspace }}/ut_log + cd ${{ github.workspace }}/ut_log + + # get skipped known issues + count=$(gh api "repos/${{ github.repository }}/issues?labels=skipped_windows" --jq 'length') + if [ "$count" -gt 0 ]; then + echo -e "$count issues with skipped label found" + gh api "repos/${{ github.repository }}/issues?labels=skipped_windows" \ + --jq '.[] | select(.pull_request == null) | "Issue #\(.number): \(.title)\n\(.body)\n"' > issues.log + fi + + cp ${{ github.workspace }}/.github/scripts/ut_result_check.sh ./ + for ut_name in $(echo ${{ inputs.ut }} |sed 's/,/ /g') + do + touch Known_issue.log + if [ -f "issues.log" ]; then + awk -v r="${ut_name}" 'BEGIN{ print_row = 0 }{ + if ( ! ( $0 ~ /[a-zA-Z0-9]/ ) ) { print_row = 0 }; + if ( print_row == 1 && $1 ~ r ) { print $0 }; + if ( $0 ~ /Cases:/ ) { print_row = 1 }; + }' issues.log > Known_issue.log + else + echo "Info: issues.log not found or empty, using empty Known_issue.log" + fi + bash ut_result_check.sh ${ut_name} + done + - name: Upload Inductor XPU UT Log + if: ${{ ! cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-Windows + path: ${{ github.workspace }}/ut_log + overwrite: true diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index 51ef59776..e5174af4f 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -205,7 +205,7 @@ jobs: needs: [Conditions-Filter] uses: ./.github/workflows/_windows_ut.yml with: - ut: 'op_extended,torch_xpu' + ut: 'op_extended,test_xpu' python: ${{ needs.Conditions-Filter.outputs.python }} src_changed: false has_label: true diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index bfb2d913a..8018abcf9 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -154,7 +154,7 @@ jobs: needs: [conditions-filter, preci-lint-check] uses: ./.github/workflows/_windows_ut.yml with: - ut: op_extended,torch_xpu + ut: op_extended,test_xpu runner: Windows_CI src_changed: ${{ needs.conditions-filter.outputs.src_changed }} has_label: ${{ needs.conditions-filter.outputs.has_label }} diff --git a/test/xpu/extended/run_test_with_skip_mtl.py b/test/xpu/extended/run_test_with_skip_mtl.py index e2683b64c..4b6eaa596 100644 --- a/test/xpu/extended/run_test_with_skip_mtl.py +++ b/test/xpu/extended/run_test_with_skip_mtl.py @@ -19,5 +19,6 @@ os.environ["PYTORCH_TEST_WITH_SLOW"] = "1" test_command = ["-k", skip_options, "test_ops_xpu.py", "-v"] +test_command.extend(["--junit-xml", "./op_extended.xml"]) res = pytest.main(test_command) sys.exit(res) diff --git a/test/xpu/run_test_win_with_skip_mtl.py b/test/xpu/run_test_win_with_skip_mtl.py index 4ba56dba3..06a4849a6 100644 --- a/test/xpu/run_test_win_with_skip_mtl.py +++ b/test/xpu/run_test_win_with_skip_mtl.py @@ -18,6 +18,7 @@ sys.stdout = StringIO() test_command = ["-k", skip_options, "../../../../test/test_xpu.py", "-v"] +test_command.extend(["--junit-xml", "./test_xpu.xml"]) res = pytest.main(test_command) output = sys.stdout.getvalue()