From 02aeab593ce0e2e5fe99717d286a8e1776de5b9d Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Fri, 23 Jul 2021 09:03:21 +0200 Subject: [PATCH 119/N] CI: clean up the build; enforce some tests This now enforces success of a subset of tests (see mingw_ignorefile.txt) and runs smoketests on the build as well as after installation. Remove everything that is no longer needed after the previous patches: * the build uses -O3 by default * it sets fwrapv by default * __USE_MINGW_ANSI_STDIO is not needed anymore with c99 * it sets_WIN32_WINNT anyway * ncurses is fixed by patching the build * debug is handled by autotools * all ac_cv_* overrides are now handled in configure.ac * venvlauncher is handled by the main makefile * setting MSYS2_ARG_CONV_EXCL is now handled by the makefile * the import library is now installed by the makefile * _sysconfig_data overrides are handled at runtime anyway * we don't care about permissions.. (I think?) --- .github/workflows/mingw.yml | 118 +++++++++++------------------------- mingw_ignorefile.txt | 34 +++++++++++ 2 files changed, 68 insertions(+), 84 deletions(-) create mode 100644 mingw_ignorefile.txt diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index b2f0d6a..16a96b5 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -51,45 +51,16 @@ jobs: run: | set -ex - CFLAGS="-O2" - - PREFIX_WIN=$(cygpath -wm ${MINGW_PREFIX}) if [ ${{ matrix.msystem }} == "CLANG64" ] then export CC=clang export CXX=clang++ fi autoreconf -vfi - CFLAGS+=" -fwrapv -D__USE_MINGW_ANSI_STDIO=1 -D_WIN32_WINNT=0x0601" - CXXFLAGS+=" -fwrapv -D__USE_MINGW_ANSI_STDIO=1 -D_WIN32_WINNT=0x0601" - CPPFLAGS+=" -I${PREFIX_WIN}/include/ncurses " - CFLAGS+=" -DNDEBUG " - CXXFLAGS+=" -DNDEBUG " - _extra_config+=("--enable-optimizations") - - # Workaround for conftest error on 64-bit builds - export ac_cv_working_tzset=no - - # Workaround for when dlfcn exists on Windows, which causes - # some conftests to succeed when they shouldn't (we don't use dlfcn). - export ac_cv_header_dlfcn_h=no - export ac_cv_lib_dl_dlopen=no - export ac_cv_have_decl_RTLD_GLOBAL=no - export ac_cv_have_decl_RTLD_LAZY=no - export ac_cv_have_decl_RTLD_LOCAL=no - export ac_cv_have_decl_RTLD_NOW=no - export ac_cv_have_decl_RTLD_DEEPBIND=no - export ac_cv_have_decl_RTLD_MEMBER=no - export ac_cv_have_decl_RTLD_NODELETE=no - export ac_cv_have_decl_RTLD_NOLOAD=no - - export CFLAGS - export CXXFLAGS - export CPPFLAGS - + rm -Rf _build && mkdir _build && cd _build - - MSYSTEM=MINGW ../configure \ + + ../configure \ --prefix=${MINGW_PREFIX} \ --host=${MINGW_CHOST} \ --build=${MINGW_CHOST} \ @@ -101,90 +72,69 @@ jobs: --without-ensurepip \ --without-c-locale-coercion \ --enable-loadable-sqlite-extensions \ - "${_extra_config[@]}" \ - OPT="" + --enable-optimizations + # We patch importlib which is embedded in C headers, so regenerate them make regen-importlib make -j8 - - # Add missing venvlauncher files (issue #7014) - # FIXME: build these from PC/launcher.c instead - cp python.exe venvlauncher.exe - cp pythonw.exe venvwlauncher.exe - - name: Install + - name: Run Smoke Test (build) shell: msys2 {0} run: | - set -ex + SMOKETESTS="$(pwd)/mingw_smoketests.py" + cd _build + ./python.exe "$SMOKETESTS" + MSYSTEM= ./python.exe "$SMOKETESTS" + + - name: Run tests + shell: msys2 {0} + run: | + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + cd _build + MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" -W + - name: Run broken tests + continue-on-error: true + shell: msys2 {0} + run: | + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" cd _build - _pybasever=$(./python.exe -c "import sys; print(sys.winver);") + MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" -W - # now install things + - name: Install + shell: msys2 {0} + run: | + set -ex + cd _build pkgdir=python_pkgdir - PREFIX_WIN=$(cygpath -wm ${MINGW_PREFIX}) - MSYSTEM=MINGW \ - MSYS2_ARG_CONV_EXCL="--prefix=;--install-scripts=;--install-platlib=" \ make -j1 install DESTDIR="${pkgdir}" - # gdb pretty printers for debugging Python itself; to use: - # python - # sys.path.append('C:/msys64/mingw64/share/gdb/python3') - # import python_gdb - # reload(python_gdb) - # end - - cp -f "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${_pybasever}/libpython${_pybasever}.dll.a "${pkgdir}${MINGW_PREFIX}"/lib/libpython${_pybasever}.dll.a - - # Need for building boost python module - cp -f "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${_pybasever}/libpython${_pybasever}.dll.a "${pkgdir}${MINGW_PREFIX}"/lib/libpython${_pybasever}.dll.a - + # Fix shebangs + _pybasever=$(./python.exe -c "import sys; print(sys.winver);") for fscripts in 2to3 2to3-${_pybasever} idle3 idle${_pybasever} pydoc3 pydoc${_pybasever}; do sed -i "s|$(cygpath -w ${MINGW_PREFIX} | sed 's|\\|\\\\|g')/bin/python${_pybasever}.exe|/usr/bin/env python${_pybasever}.exe|g" "${pkgdir}${MINGW_PREFIX}"/bin/${fscripts} done + sed -i "s|#!${pkgdir}${MINGW_PREFIX}/bin/python${_pybasever}.exe|#!/usr/bin/env python${_pybasever}.exe|" "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${_pybasever}/python-config.py - + # Create version-less aliases cp "${pkgdir}${MINGW_PREFIX}"/bin/python3.exe "${pkgdir}${MINGW_PREFIX}"/bin/python.exe cp "${pkgdir}${MINGW_PREFIX}"/bin/python3w.exe "${pkgdir}${MINGW_PREFIX}"/bin/pythonw.exe cp "${pkgdir}${MINGW_PREFIX}"/bin/python3-config "${pkgdir}${MINGW_PREFIX}"/bin/python-config cp "${pkgdir}${MINGW_PREFIX}"/bin/idle3 "${pkgdir}${MINGW_PREFIX}"/bin/idle cp "${pkgdir}${MINGW_PREFIX}"/bin/pydoc3 "${pkgdir}${MINGW_PREFIX}"/bin/pydoc - sed -i "s|#!${pkgdir}${MINGW_PREFIX}/bin/python${_pybasever}.exe|#!/usr/bin/env python${_pybasever}.exe|" "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${_pybasever}/python-config.py - - # fix permissons - find ${pkgdir}${MINGW_PREFIX} -type f \( -name "*.dll" -o -name "*.exe" \) | xargs chmod 0755 - - # replace paths in sysconfig - sed -i "s|${pkgdir}${MINGW_PREFIX}|${MINGW_PREFIX}|g" \ - "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/lib-dynload/_sysconfigdata*.py \ - "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/smtpd.py - - # install venv launchers - mkdir -p "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/venv/scripts/nt - cp venvlauncher.exe "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/venv/scripts/nt/python.exe - cp venvwlauncher.exe "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/venv/scripts/nt/pythonw.exe - - - name: Run Smoke Test + - name: Run Smoke Test (installed) shell: msys2 {0} run: | - SMOKETESTS="$(pwd)/.github/workflows/smoketests.py" + SMOKETESTS="$(pwd)/mingw_smoketests.py" cd _build cd python_pkgdir/${MINGW_PREFIX}/bin ./python.exe "$SMOKETESTS" MSYSTEM= ./python.exe "$SMOKETESTS" - - name: Run tests - continue-on-error: true - shell: msys2 {0} - run: | - cd _build - cd python_pkgdir/${MINGW_PREFIX}/bin - ./python.exe -m test -j4 - - name: Compress if: always() shell: msys2 {0} diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt new file mode 100644 index 0000000..dc3802e --- /dev/null +++ b/mingw_ignorefile.txt @@ -0,0 +1,34 @@ +ctypes.test.test_loading.LoaderTest.test_load_dll_with_flags +distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built +distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions +distutils.tests.test_util.UtilTestCase.test_change_root +test.datetimetester.TestLocalTimeDisambiguation_Fast.* +test.datetimetester.TestLocalTimeDisambiguation_Pure.* +test.test_cmath.CMathTests.test_specific_values +test.test_cmd_line_script.CmdLineTest.test_consistent_sys_path_for_direct_execution +test.test_compileall.CommandLineTestsNoSourceEpoch.* +test.test_compileall.CommandLineTestsWithSourceEpoch.* +test.test_compileall.CompileallTestsWithoutSourceEpoch.* +test.test_compileall.CompileallTestsWithSourceEpoch.* +test.test_import.ImportTests.test_dll_dependency_import +test.test_math.MathTests.* +test.test_ntpath.NtCommonTest.test_import +test.test_os.StatAttributeTests.test_stat_block_device +test.test_os.TestScandir.test_attributes +test.test_os.UtimeTests.test_large_time +test.test_platform.PlatformTest.test_architecture_via_symlink +test.test_regrtest.ProgramsTestCase.test_pcbuild_rt +test.test_regrtest.ProgramsTestCase.test_tools_buildbot_test +test.test_site._pthFileTests.* +test.test_site.HelperFunctionsTests.* +test.test_site.StartupImportTests.* +test.test_ssl.* +test.test_strptime.CalculationTests.* +test.test_strptime.StrptimeTests.test_weekday +test.test_strptime.TimeRETests.test_compile +test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date +test.test_venv.BasicTest.* +test.test_venv.EnsurePipTest.* +# flaky +test.test__xxsubinterpreters.* +test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe \ No newline at end of file -- 2.32.0