From 3a33db28df3b60d7127396f76366e2081eccd386 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Fri, 18 Jun 2021 17:51:59 +0530 Subject: [PATCH 096/N] Add CI to Build and Test --- .github/workflows/mingw.yml | 229 ++++++++++++++++++++++++++++++++ .github/workflows/smoketests.py | 132 ++++++++++++++++++ 2 files changed, 361 insertions(+) create mode 100644 .github/workflows/mingw.yml create mode 100644 .github/workflows/smoketests.py diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml new file mode 100644 index 0000000..fff5fa5 --- /dev/null +++ b/.github/workflows/mingw.yml @@ -0,0 +1,229 @@ +name: Build +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + msystem: ['MINGW64','MINGW32','UCRT64','CLANG64'] + include: + - msystem: MINGW64 + prefix: mingw-w64-x86_64 + - msystem: MINGW32 + prefix: mingw-w64-i686 + - msystem: UCRT64 + prefix: mingw-w64-ucrt-x86_64 + - msystem: CLANG64 + prefix: mingw-w64-clang-x86_64 + #- msystem: CLANG32 + # prefix: mingw-w64-clang-i686 + steps: + - name: Setup git + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - uses: actions/checkout@v2 + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + release: false + install: >- + make + binutils + autoconf-archive + ${{ matrix.prefix }}-toolchain + ${{ matrix.prefix }}-expat + ${{ matrix.prefix }}-bzip2 + ${{ matrix.prefix }}-libffi + ${{ matrix.prefix }}-mpdecimal + ${{ matrix.prefix }}-ncurses + ${{ matrix.prefix }}-openssl + ${{ matrix.prefix }}-sqlite3 + ${{ matrix.prefix }}-tcl + ${{ matrix.prefix }}-tk + ${{ matrix.prefix }}-zlib + ${{ matrix.prefix }}-xz + + - name: Build Python + shell: msys2 {0} + run: | + set -ex + pkgver='3.9.5' + _pybasever='3.9' + + PREFIX_WIN=$(cygpath -wm ${MINGW_PREFIX}) + if [ ${{ matrix.msystem }} == "CLANG64" ] + then + export CC=clang + export CXX=clang++ + fi + autoreconf -vfi + CFLAGS+=" -fwrapv -D__USE_MINGW_ANSI_STDIO=1 -D_WIN32_WINNT=0x0601" + CXXFLAGS+=" -fwrapv -D__USE_MINGW_ANSI_STDIO=1 -D_WIN32_WINNT=0x0601" + CPPFLAGS+=" -I${PREFIX_WIN}/include/ncurses " + CFLAGS+=" -DNDEBUG " + CXXFLAGS+=" -DNDEBUG " + _extra_config+=("--enable-optimizations") + # Fix MS_DLL_ID undeclared in PC/dl_nt.c + f3f=0 + re='^[0-9]+$' + part3=${pkgver##*.} + if ! [[ $part3 =~ $re ]]; then + c= + while test -n "$part3"; do + b=${part3:0:1} + if ! [[ $b =~ $re ]]; then + break + fi + c+=${b} + part3=${part3:1} + done + f3f=$((f3f + 1000 * $((${c})))) + part3Len=${#part3} + if [ "$part3Len" -ge "1" ]; then + f3f=$((f3f + 10 * $((0x${part3:0:1})))) + if [ "$part3Len" -ge "2" ]; then + f3f=$((f3f + ${part3:1:1})) + fi + fi + else + f3f=$((f3f + 1000 * ${part3})) + fi + CFLAGS+=" -DMS_DLL_ID=\\\"${_pybasever}\\\"" + CFLAGS+=" -DFIELD3=${f3f}" + CFLAGS+=" -DORIGINAL_FILENAME=\\\"libpython${_pybasever}.dll\\\"" + rc_flags=" -DMS_DLL_ID=\\\\\\\"${_pybasever}\\\\\\\" -DFIELD3=${f3f} -DORIGINAL_FILENAME=\\\\\\\"libpython${_pybasever}.dll\\\\\\\"" + + # Workaround for conftest error on 64-bit builds + export ac_cv_working_tzset=no + + # Workaround for when dlfcn exists on Windows, which causes + # some conftests to succeed when they shouldn't (we don't use dlfcn). + export ac_cv_header_dlfcn_h=no + export ac_cv_lib_dl_dlopen=no + export ac_cv_have_decl_RTLD_GLOBAL=no + export ac_cv_have_decl_RTLD_LAZY=no + export ac_cv_have_decl_RTLD_LOCAL=no + export ac_cv_have_decl_RTLD_NOW=no + export ac_cv_have_decl_RTLD_DEEPBIND=no + export ac_cv_have_decl_RTLD_MEMBER=no + export ac_cv_have_decl_RTLD_NODELETE=no + export ac_cv_have_decl_RTLD_NOLOAD=no + + export CFLAGS + export CXXFLAGS + export CPPFLAGS + + + MSYSTEM=MINGW ./configure \ + --prefix=${MINGW_PREFIX} \ + --host=${MINGW_CHOST} \ + --build=${MINGW_CHOST} \ + --enable-shared \ + --with-nt-threads \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ + --without-ensurepip \ + --without-c-locale-coercion \ + --enable-loadable-sqlite-extensions \ + "${_extra_config[@]}" \ + RCFLAGS="$rc_flags" \ + OPT="" + # We patch importlib which is embedded in C headers, so regenerate them + make regen-importlib + + make -j8 + + # Add missing venvlauncher files (issue #7014) + # FIXME: build these from PC/launcher.c instead + cp python.exe venvlauncher.exe + cp pythonw.exe venvwlauncher.exe + + - name: Install + shell: msys2 {0} + run: | + set -ex + + pkgver='3.9.5' + _pybasever='3.9' + srcdir=. + + # now install things + + pkgdir=python_pkgdir + + PREFIX_WIN=$(cygpath -wm ${MINGW_PREFIX}) + MSYSTEM=MINGW \ + MSYS2_ARG_CONV_EXCL="--prefix=;--install-scripts=;--install-platlib=" \ + make -j1 install DESTDIR="${pkgdir}" + VERABI=${_pybasever} + + # gdb pretty printers for debugging Python itself; to use: + # python + # sys.path.append('C:/msys64/mingw64/share/gdb/python3') + # import python_gdb + # reload(python_gdb) + # end + + cp -f "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${VERABI}/libpython${VERABI}.dll.a "${pkgdir}${MINGW_PREFIX}"/lib/libpython${VERABI}.dll.a + + # Need for building boost python module + cp -f "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${VERABI}/libpython${VERABI}.dll.a "${pkgdir}${MINGW_PREFIX}"/lib/libpython${_pybasever}.dll.a + + for fscripts in 2to3 2to3-${_pybasever} idle3 idle${_pybasever} pydoc3 pydoc${_pybasever}; do + sed -i "s|$(cygpath -w ${MINGW_PREFIX} | sed 's|\\|\\\\|g')/bin/python${_pybasever}.exe|/usr/bin/env python${_pybasever}.exe|g" "${pkgdir}${MINGW_PREFIX}"/bin/${fscripts} + done + + + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3.exe "${pkgdir}${MINGW_PREFIX}"/bin/python.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3w.exe "${pkgdir}${MINGW_PREFIX}"/bin/pythonw.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3-config "${pkgdir}${MINGW_PREFIX}"/bin/python-config + cp "${pkgdir}${MINGW_PREFIX}"/bin/idle3 "${pkgdir}${MINGW_PREFIX}"/bin/idle + cp "${pkgdir}${MINGW_PREFIX}"/bin/pydoc3 "${pkgdir}${MINGW_PREFIX}"/bin/pydoc + + sed -i "s|#!${pkgdir}${MINGW_PREFIX}/bin/python${VERABI}.exe|#!/usr/bin/env python${_pybasever}.exe|" "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${VERABI}/python-config.py + + # fix permissons + find ${pkgdir}${MINGW_PREFIX} -type f \( -name "*.dll" -o -name "*.exe" \) | xargs chmod 0755 + + # replace paths in sysconfig + sed -i "s|${pkgdir}${MINGW_PREFIX}|${MINGW_PREFIX}|g" \ + "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/lib-dynload/_sysconfigdata*.py \ + "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/smtpd.py + + # install venv launchers + mkdir -p "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/venv/scripts/nt + cp venvlauncher.exe "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/venv/scripts/nt/python.exe + cp venvwlauncher.exe "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/venv/scripts/nt/pythonw.exe + + - name: Run Smoke Test + shell: msys2 {0} + run: | + SMOKETESTS="$(pwd)/.github/workflows/smoketests.py" + cd python_pkgdir/${MINGW_PREFIX}/bin + ./python.exe "$SMOKETESTS" + MSYSTEM= ./python.exe "$SMOKETESTS" + + - name: Run tests + continue-on-error: true + shell: msys2 {0} + run: | + cd python_pkgdir/${MINGW_PREFIX}/bin + ./python.exe -m test -j4 + + - name: Compress + if: always() + shell: msys2 {0} + run: | + tar -I 'zstd --ultra -20' -cf python.tar.zst python_pkgdir/ + + - name: Upload + uses: actions/upload-artifact@v2 + if: always() + with: + name: build-${{ matrix.msystem }} + path: python.tar.zst + diff --git a/.github/workflows/smoketests.py b/.github/workflows/smoketests.py new file mode 100644 index 0000000..9277c14 --- /dev/null +++ b/.github/workflows/smoketests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# Copyright 2017 Christoph Reiter +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The goal of this test suite is collect tests for update regressions +and to test msys2 related modifications like for path handling. +Feel free to extend. +""" + +import os +import unittest + +if "MSYSTEM" in os.environ: + SEP = "/" +else: + SEP = "\\" + + +class Tests(unittest.TestCase): + + def test_sep(self): + self.assertEqual(os.sep, SEP) + + def test_module_file_path(self): + import asyncio + import zlib + self.assertEqual(zlib.__file__, os.path.normpath(zlib.__file__)) + self.assertEqual(asyncio.__file__, os.path.normpath(asyncio.__file__)) + + def test_importlib_frozen_path_sep(self): + import importlib._bootstrap_external + self.assertEqual(importlib._bootstrap_external.path_sep, SEP) + + def test_os_commonpath(self): + self.assertEqual( + os.path.commonpath( + [os.path.join("C:", os.sep, "foo", "bar"), + os.path.join("C:", os.sep, "foo")]), + os.path.join("C:", os.sep, "foo")) + + def test_pathlib(self): + import pathlib + + p = pathlib.Path("foo") / pathlib.Path("foo") + self.assertEqual(str(p), os.path.normpath(p)) + + def test_modules_import(self): + import sqlite3 + import ssl + import ctypes + + def test_socket_inet_ntop(self): + import socket + self.assertTrue(hasattr(socket, "inet_ntop")) + + def test_socket_inet_pton(self): + import socket + self.assertTrue(hasattr(socket, "inet_pton")) + + def test_multiprocessing_queue(self): + from multiprocessing import Queue + Queue(0) + + #def test_socket_timout_normal_error(self): + # import urllib.request + # from urllib.error import URLError + + # try: + # urllib.request.urlopen( + # 'http://localhost', timeout=0.0001).close() + # except URLError: + # pass + + def test_threads(self): + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor(1) as pool: + for res in pool.map(lambda *x: None, range(10000)): + pass + + def test_sysconfig(self): + import sysconfig + # This should be able to execute without exceptions + sysconfig.get_config_vars() + + def test_sqlite_enable_load_extension(self): + # Make sure --enable-loadable-sqlite-extensions is used + import sqlite3 + self.assertTrue(sqlite3.Connection.enable_load_extension) + + def test_venv_creation(self): + import tempfile + import venv + import subprocess + import shutil + with tempfile.TemporaryDirectory() as tmp: + builder = venv.EnvBuilder() + builder.create(tmp) + assert os.path.exists(os.path.join(tmp, "bin", "activate")) + assert os.path.exists(os.path.join(tmp, "bin", "python.exe")) + assert os.path.exists(os.path.join(tmp, "bin", "python3.exe")) + subprocess.check_call([shutil.which("bash.exe"), os.path.join(tmp, "bin", "activate")]) + + def test_has_mktime(self): + from time import mktime, gmtime + mktime(gmtime()) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') -- 2.32.0