diff --git a/CHANGELOG.md b/CHANGELOG.md index 74735410..9ed7305e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [Unreleased] +## 5.14.2.1 ### Added * [#39](https://github.com/stlehmann/PyQt5-stubs/pull/39) Add this changelog file +* [#36](https://github.com/stlehmann/PyQt5-stubs/pull/36), [#41](https://github.com/stlehmann/PyQt5-stubs/pull/41 +) New build script for upstream stubs includes extra + packages +like QtWebEngine and Qt3D ### Changed - * [#38](https://github.com/stlehmann/PyQt5-stubs/pull/38) Changed license to GPLv3 to be compilient with PyQt5 license ### Fixed \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7e372a8f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,274 @@ +# Careful, the version and build date are both dates, but different formats +ARG ARCH_VERSION="20200407" +ARG BUILD_DATE="2020/04/22" + +ARG PYQT_VERSION="5.14.2" +ARG PYQT_3D_VERSION="5.14.0" +ARG PYQT_CHART_VERSION="5.14.0" +ARG PYQT_DATA_VISUALIZATION_VERSION="5.14.0" +ARG PYQT_PURCHASING_VERSION="5.14.0" +ARG PYQT_WEB_ENGINE_VERSION="5.14.0" + +################################################################################ +# Build dependencies +################################################################################ + +FROM archlinux:${ARCH_VERSION} AS build-dep + +# Reuse argument from previous build scope +ARG BUILD_DATE + +# Use Arch archive to freeze packages to a certain date +RUN echo "Server=https://archive.archlinux.org/repos/${BUILD_DATE}/\$repo/os/\$arch" \ + | tee /etc/pacman.d/mirrorlist && \ + pacman -Syyuu --noconfirm + +# Install build dependencies and Qt Modules +RUN pacman --noconfirm -S \ + # Build stuff + base-devel wget \ + # PyQt stuff + pyqt-builder python-sip sip5 \ + # Used to build other PyQt modules in later build stages + python-pyqt5 \ + # Qt core + qt5-base \ + # Qt modules not included in qt5-base + qt5-3d \ + qt5-charts \ + qt5-connectivity \ + qt5-datavis3d \ + qt5-declarative \ + qt5-gamepad \ + qt5-graphicaleffects \ + qt5-imageformats \ + qt5-location \ + qt5-multimedia \ + qt5-purchasing \ + qt5-networkauth \ + qt5-remoteobjects \ + qt5-script \ + qt5-sensors \ + qt5-serialport \ + qt5-svg \ + qt5-tools \ + qt5-wayland \ + qt5-webchannel \ + qt5-webengine \ + qt5-webkit \ + qt5-websockets \ + qt5-x11extras \ + qt5-xmlpatterns \ + # Required for QtDBus + python-dbus + +################################################################################ +# PyQt5 core stubs +################################################################################ + +FROM build-dep AS pyqt5 + +# Reuse argument from previous build scope +ARG PYQT_VERSION + +# Download source tar +RUN wget --no-verbose \ + --output-document upstream.tar.gz \ + https://pypi.io/packages/source/p/pyqt5/PyQt5-${PYQT_VERSION}.tar.gz +RUN mkdir /upstream/ && \ + tar -xf \ + upstream.tar.gz \ + --directory /upstream/ \ + --strip-components 1 + +# Build PyQt with stubs +# TODO: Find way to build only stubs. This takes way too long +WORKDIR /upstream/ +RUN sip-install \ + --qmake /usr/bin/qmake-qt5 \ + --confirm-license \ + --pep484-pyi \ + --build-dir ./build \ + --verbose + +# Copy all .pyi files to output dir +WORKDIR /output/ +RUN find /upstream/ -name \*.pyi -exec cp {} . \; + +################################################################################ +# PyQt3D +################################################################################ + +FROM build-dep AS pyqt-3d + +# Reuse argument from previous build scope +ARG PYQT_3D_VERSION + +# Download source tar +RUN wget --no-verbose \ + --output-document upstream.tar.gz \ + https://pypi.io/packages/source/p/pyqt3d/PyQt3D-${PYQT_3D_VERSION}.tar.gz +RUN mkdir /upstream/ && \ + tar -xf \ + upstream.tar.gz \ + --directory /upstream/ \ + --strip-components 1 + +# Build PyQt3D with stubs +# TODO: Find way to build only stubs +WORKDIR /upstream/ +RUN sip-install \ + --qmake /usr/bin/qmake-qt5 \ + --pep484-pyi \ + --build-dir ./build \ + --verbose + +# Copy all .pyi files to output dir +WORKDIR /output/ +RUN find /upstream/ -name \*.pyi -exec cp {} . \; + +################################################################################ +# PyQtChart +################################################################################ + +FROM build-dep AS pyqt-chart + +# Reuse argument from previous build scope +ARG PYQT_CHART_VERSION + +# Download source tar +RUN wget --no-verbose \ + --output-document upstream.tar.gz \ + https://pypi.io/packages/source/p/pyqtchart/PyQtChart-${PYQT_CHART_VERSION}.tar.gz +RUN mkdir /upstream/ && \ + tar -xf \ + upstream.tar.gz \ + --directory /upstream/ \ + --strip-components 1 + +# Build PyQtChart with stubs +# TODO: Find way to build only stubs +WORKDIR /upstream/ +RUN sip-install \ + --qmake /usr/bin/qmake-qt5 \ + --pep484-pyi \ + --build-dir ./build \ + --verbose + +# Copy all .pyi files to output dir +WORKDIR /output/ +RUN find /upstream/ -name \*.pyi -exec cp {} . \; + +################################################################################ +# PyQtDataVisualization +################################################################################ + +FROM build-dep AS pyqt-data-visualization + +# Reuse argument from previous build scope +ARG PYQT_DATA_VISUALIZATION_VERSION + +# Download source tar +RUN wget --no-verbose \ + --output-document upstream.tar.gz \ + https://pypi.io/packages/source/p/pyqtdatavisualization/PyQtDataVisualization-${PYQT_DATA_VISUALIZATION_VERSION}.tar.gz +RUN mkdir /upstream/ && \ + tar -xf \ + upstream.tar.gz \ + --directory /upstream/ \ + --strip-components 1 + +# Build PyQtDataVisualization with stubs +# TODO: Find way to build only stubs +WORKDIR /upstream/ +RUN sip-install \ + --qmake /usr/bin/qmake-qt5 \ + --pep484-pyi \ + --build-dir ./build \ + --verbose + +# Copy all .pyi files to output dir +WORKDIR /output/ +RUN find /upstream/ -name \*.pyi -exec cp {} . \; + +################################################################################ +# PyQtPurchasing +################################################################################ + +FROM build-dep AS pyqt-purchasing + +# Reuse argument from previous build scope +ARG PYQT_PURCHASING_VERSION + +# Download source tar +RUN wget --no-verbose \ + --output-document upstream.tar.gz \ + https://pypi.io/packages/source/p/pyqtpurchasing/PyQtPurchasing-${PYQT_PURCHASING_VERSION}.tar.gz +RUN mkdir /upstream/ && \ + tar -xf \ + upstream.tar.gz \ + --directory /upstream/ \ + --strip-components 1 + +# Build PyQtPurchasing with stubs +# TODO: Find way to build only stubs +WORKDIR /upstream/ +RUN sip-install \ + --qmake /usr/bin/qmake-qt5 \ + --pep484-pyi \ + --build-dir ./build \ + --verbose + +# Copy all .pyi files to output dir +WORKDIR /output/ +RUN find /upstream/ -name \*.pyi -exec cp {} . \; + +################################################################################ +# PyQtWebEngine +################################################################################ + +FROM build-dep AS pyqt-web-engine + +# Reuse argument from previous build scope +ARG PYQT_WEB_ENGINE_VERSION + +# Download source tar +RUN wget --no-verbose \ + --output-document upstream.tar.gz \ + https://pypi.io/packages/source/p/pyqtwebengine/PyQtWebEngine-${PYQT_WEB_ENGINE_VERSION}.tar.gz +RUN mkdir /upstream/ && \ + tar -xf \ + upstream.tar.gz \ + --directory /upstream/ \ + --strip-components 1 + +# Build PyQtWebEngine with stubs +# TODO: Find way to build only stubs +WORKDIR /upstream/ +RUN sip-install \ + --qmake /usr/bin/qmake-qt5 \ + --pep484-pyi \ + --build-dir ./build \ + --verbose + +# Copy all .pyi files to output dir +WORKDIR /output/ +RUN find /upstream/ -name \*.pyi -exec cp {} . \; + +################################################################################ +# Output +################################################################################ + +FROM scratch AS output + +# Get all the outputs from the build layers +WORKDIR /output/ +COPY --from=pyqt5 /output/* ./ +COPY --from=pyqt-3d /output/* ./ +COPY --from=pyqt-chart /output/* ./ +COPY --from=pyqt-data-visualization /output/* ./ +COPY --from=pyqt-purchasing /output/* ./ +COPY --from=pyqt-web-engine /output/* ./ + +# Required to run the image (which we need to do to get the files) +CMD /bin/true diff --git a/PyQt5-stubs/__init__.pyi b/PyQt5-stubs/__init__.pyi index 3d582a0d..339f7ce3 100644 --- a/PyQt5-stubs/__init__.pyi +++ b/PyQt5-stubs/__init__.pyi @@ -1 +1 @@ -__version__ = "5.14.2.0" +__version__ = "5.14.2.1" diff --git a/README.md b/README.md index fcb8c45b..63c29931 100644 --- a/README.md +++ b/README.md @@ -38,3 +38,29 @@ The following modules are supported by PyQt5-stubs: * QtTest * QtXml * sip + +# Building upstream stubs +The Dockerfile is used to build all of the stubs for the upstream PyQt5 modules. +The Dockerfile consists of multiple build layers: +* core: `PyQt5` +* `PyQt3D` +* `PyQtChart` +* `PyQtDataVisualization` +* `PyQtPurchasing` +* `PyQtWebEngine` +* an output layer + +Each module build layer deposits its stub files within `/output/` in its +filesystem. The output layer then collects the contents of each into its own +`/output/` dir for export to the host computer. Build args are provided to +change the version of each module. + +A convenience script, `build_upstream.py`, is provided. It builds the stubs and +copies them to the host computer. Make sure you install `docker-py` to use it. +It builds `$PWD/Dockerfile` (overridden with `--dockerfile`) and outputs the +stubs to `$PWD/PyQt5-stubs` (overridden with `--output-dir`). + +\* There are a few missing modules: `QtAxContainer`, `QtAndroidExtras`, +`QtMacExtras`, and `QtWindowsExtras`. The current project understanding is that +they need to be built on the target platform, something a Linux-based docker +image cannot do. The deprecated `Enginio` module is also missing. diff --git a/build_upstream.py b/build_upstream.py new file mode 100644 index 00000000..3d74494f --- /dev/null +++ b/build_upstream.py @@ -0,0 +1,118 @@ +import argparse +import io +import re +import tarfile +from pathlib import Path +import typing + +import docker +from docker import DockerClient +from docker.models.images import Image +from docker.utils.json_stream import json_stream + +DEFAULT_DOCKERFILE = Path("Dockerfile") +DEFAULT_OUTPUT_DIR = Path("PyQt5-stubs") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build PyQt stubs in Docker") + + # noinspection PyTypeChecker + parser.add_argument('-d', '--dockerfile', type=Path, + default=DEFAULT_DOCKERFILE, + help="Dockerfile to build") + + # noinspection PyTypeChecker + parser.add_argument('-o', '--output-dir', type=Path, + default=DEFAULT_OUTPUT_DIR, + help="Directory to find package(s) to be built. " + "Defaults to ./pkg") + + return parser.parse_args() + + +def main(): + args = parse_args() + + docker_client = docker.from_env() + + image_id = build_image(docker_client, args.dockerfile) + + extract_output(docker_client, image_id, args.output_dir) + + +def build_image(docker_client: DockerClient, dockerfile: Path) -> str: + image_name = "pyqt5-stubs" + + # Using low-level API so that we can log as it occurs instead of only + # after build has finished/failed + resp = docker_client.api.build( + path=str(dockerfile.parent), + rm=True, + tag=image_name) + + image_id: str = typing.cast(str, None) + for chunk in json_stream(resp): + if 'error' in chunk: + message = f"Error while building Dockerfile for " \ + f"{image_name}:\n{chunk['error']}" + print(message) + raise DockerBuildError(message) + + elif 'stream' in chunk: + print(chunk['stream'].rstrip('\n')) + # Taken from the high level API implementation of build + match = re.search(r'(^Successfully built |sha256:)([0-9a-f]+)$', + chunk['stream']) + if match: + image_id = match.group(2) + + if not image_id: + message = f"Unknown Error while building Dockerfile for " \ + f"{image_name}. Build did not return an image ID" + raise DockerBuildError(message) + + return image_id + + +def extract_output(docker_client: DockerClient, image_id: str, + output_dir: Path) -> None: + image = docker_client.images.get(image_id) + container = docker_client.containers.create(image) + + # Get archive tar bytes from the container as a sequence of bytes + package_tar_byte_gen: typing.Generator[bytes, None, None] + package_tar_byte_gen, _ = container.get_archive("/output/", + chunk_size=None) + + # Concat all the chunks together + package_tar_bytes: bytes + package_tar_bytes = b"".join(package_tar_byte_gen) + + # Create a tarfile from the tar bytes + tar_file_object = io.BytesIO(package_tar_bytes) + package_tar = tarfile.open(fileobj=tar_file_object) + + # Extract the files from the tarfile to the disk + for tar_deb_info in package_tar.getmembers(): + # Ignore directories + if not tar_deb_info.isfile(): + continue + + # Directory that will contain the output files + output_dir.mkdir(parents=True, exist_ok=True) + + # Filename (without outer directory) + tar_deb_info.name = Path(tar_deb_info.name).name + + # Extract + package_tar.extract(tar_deb_info, output_dir) + + +class DockerBuildError(RuntimeError): + def __init__(self, message): + self.message = message + + +if __name__ == '__main__': + main() diff --git a/buildenv/Dockerfile b/buildenv/Dockerfile deleted file mode 100644 index 9df52283..00000000 --- a/buildenv/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu -MAINTAINER Stefan Lehmann "stlm@posteo.de" -RUN apt-get update -RUN apt-get install -y build-essential wget qt5-default python3 python3-pip dos2unix -COPY ./build_pyqt5.sh . -RUN dos2unix build_pyqt5.sh -CMD bash build_pyqt5.sh diff --git a/buildenv/README.md b/buildenv/README.md deleted file mode 100644 index 911d32a8..00000000 --- a/buildenv/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Using the buildenv - -The buildenv enables you to create new PyQt5-stubs from scratch. - -## Usage - -### 1. Update build_pyq5.sh to the latest versions: - -Set PYQT5_VERSION and SIP_VERSION according to the current version number on riverbankcomputing.com: - -PyQt5: https://www.riverbankcomputing.com/software/pyqt/download5/ -SIP: https://www.riverbankcomputing.com/software/sip/download - -### 2. Build docker image - -Build the image with the following command: - -```bash -$ docker build -t pyqt-stubs . -``` - -### 3. Run the docker image - -```bash -$ docker run -it --rm pyqt5-stubs -``` - -If everything went fine you will find the stub files in the buildenv/stubs directory. - -### 4. Create a new branch and replace older modified stub files - -Change to PyQt5-stubs directory and create a branch with the current version -number and *.org* suffix (e.g. 5.13.1.org). Replace the existing stub file with -the freshly created ones. \ No newline at end of file diff --git a/buildenv/build.py b/buildenv/build.py deleted file mode 100644 index 3c63f3ce..00000000 --- a/buildenv/build.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -:author: Stefan Lehmann -:license: MIT, see license file or https://opensource.org/licenses/MIT - -:created on: 2019-07-22 14:41:45 -:last modified by: Stefan Lehmann -:last modified time: 2019-07-23 08:38:42 - -""" -import sys -import pathlib -from subprocess import call - - -p = pathlib.Path() -stubs_path = p / "stubs" - -# build docker image -call(["docker", "build", "-tpyqt5-buildenv", "."]) - -# run docker image -call( - [ - "docker", - "run", - "-it", - "--name=pyqt5-buildenv", - "-v" + str(stubs_path.resolve()) + ":/root/stubs", - "pyqt5-buildenv", - ] -) diff --git a/buildenv/build_pyqt5.sh b/buildenv/build_pyqt5.sh deleted file mode 100644 index f13c6448..00000000 --- a/buildenv/build_pyqt5.sh +++ /dev/null @@ -1,29 +0,0 @@ -# @Author: Stefan Lehmann -# @Date: 2018-12-06 11:24:07 -# @Last Modified by: Stefan Lehmann -# @Last Modified time: 2019-07-23 08:43:52 -PYQT_VERSION="5.13.1" -SIP_VERSION="4.19.19" - -# build sip -cd /root -wget https://www.riverbankcomputing.com/static/Downloads/sip/$SIP_VERSION/sip-$SIP_VERSION.tar.gz -O sip.tar.gz -tar -xvzf sip.tar.gz -cd sip-* -python3 configure.py --sip-module PyQt5.sip -make -make install -cd /root - -# build PyQt5 -wget https://www.riverbankcomputing.com/static/Downloads/PyQt5/$PYQT_VERSION/PyQt5_gpl-$PYQT_VERSION.tar.gz -O PyQt5.tar.gz -tar -xvzf PyQt5.tar.gz -cd PyQt5_gpl-$PYQT_VERSION -python3 configure.py --confirm-license -make -make install - -# copy stubs -cd /root -mkdir /root/stubs/$PYQT_VERSION -cp /root/PyQt5_gpl-$PYQT_VERSION/*.pyi /root/stubs/$PYQT_VERSION/ diff --git a/setup.py b/setup.py index 520cbab5..8e9ad781 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ def find_version(*file_paths): package_data={"PyQt5-stubs": ['*.pyi']}, packages=["PyQt5-stubs"], tests_require=["PyQt5==5.14.*"], + extras_requires={"build": ["docker==4.2.0"]}, classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers",