Merge pull request #219 from GoSecure/docker-faster-iterations

Better use of the docker cache

.github/workflows/ci.yml

@@ -15,9 +15,13 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Build Docker image
-      run: docker build .
+      run: docker build -t pyrdp .
+    - name: Smoke test docker image
+      run: docker run pyrdp pyrdp-player.py -h
     - name: Build slim Docker image
-      run: docker build -f Dockerfile.slim .
+      run: docker build -f Dockerfile.slim -t pyrdp .
+    - name: Smoke test docker slim image
+      run: docker run pyrdp pyrdp-mitm.py -h
 
   install-and-test-ubuntu:
     runs-on: ubuntu-latest

Dockerfile

@@ -16,22 +16,28 @@ RUN python3 -m venv /opt/venv
 # Make sure we use the virtualenv:
 ENV PATH="/opt/venv/bin:$PATH"
 
-# Copy only what is required for the install
-COPY setup.py /pyrdp/setup.py
-COPY bin/ /pyrdp/bin/
+# Install dependencies only (speeds repetitive builds)
+COPY requirements.txt /pyrdp/requirements.txt
+RUN cd /pyrdp && pip3 --no-cache-dir install -r requirements.txt
+
+# Compile only our C extension and install
+# This way changes to source tree will not trigger full images rebuilds
 COPY ext/rle.c /pyrdp/ext/rle.c
-COPY pyrdp/ /pyrdp/pyrdp/
-# Install in the virtualenv
+COPY setup.py /pyrdp/setup.py
 RUN cd /pyrdp \
-    && pip3 --no-cache-dir install .[full] -U
+    && python setup.py build_ext \
+    && python setup.py install_lib
 
 
 # Handles runtime only (minimize size for distribution)
 FROM ubuntu:18.04 AS docker-image
 
 # Install runtime dependencies except pre-built venv
 RUN apt-get update && apt-get install -y --no-install-recommends python3 \
+        # To generate certificates
         openssl \
+        # Required for the setup.py install
+        python3-distutils \
         # GUI and notifications stuff
         libgl1-mesa-glx \
         notify-osd dbus-x11 libxkbcommon-x11-0 \
@@ -42,10 +48,24 @@ COPY --from=compile-image /opt/venv /opt/venv
 
 # Create user
 RUN useradd --create-home --home-dir /home/pyrdp pyrdp
+
+# Make sure we use the virtualenv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install python source and package
+# NOTE: we are no longer doing this in the compile image to avoid long image rebuilds in development
+COPY --from=compile-image /pyrdp /pyrdp
+COPY bin/ /pyrdp/bin/
+COPY pyrdp/ /pyrdp/pyrdp/
+COPY setup.py /pyrdp/setup.py
+RUN cd /pyrdp \
+    && python setup.py install
+
 USER pyrdp
 
 # UTF-8 support on the console output (for pyrdp-player)
 ENV PYTHONIOENCODING=utf-8
-# Make sure we use the virtualenv:
+# Make sure we use the virtualenv
 ENV PATH="/opt/venv/bin:$PATH"
+
 WORKDIR /home/pyrdp

Dockerfile.slim

@@ -18,32 +18,53 @@ RUN python3 -m venv /opt/venv
 # Make sure we use the virtualenv:
 ENV PATH="/opt/venv/bin:$PATH"
 
-# Copy only what is required for the install
-COPY setup.py /pyrdp/setup.py
-COPY bin/ /pyrdp/bin/
+# Install dependencies only (speeds repetitive builds)
+COPY requirements-slim.txt /pyrdp/requirements.txt
+RUN cd /pyrdp && pip3 --no-cache-dir install -r requirements.txt
+
+# Compile only our C extension and install
+# This way changes to source tree will not trigger full images rebuilds
 COPY ext/rle.c /pyrdp/ext/rle.c
-COPY pyrdp/ /pyrdp/pyrdp/
-# Install in the virtualenv
+COPY setup.py /pyrdp/setup.py
 RUN cd /pyrdp \
-    && pip3 --no-cache-dir install . -U
+    && python setup.py build_ext \
+    && python setup.py install_lib
+
 
 # Handles runtime only (minimize size for distribution)
 FROM ubuntu:18.04 AS docker-image
 
 # Install runtime dependencies except pre-built venv
 RUN apt-get update && apt-get install -y --no-install-recommends python3 \
-    openssl \
-    && rm -rf /var/lib/apt/lists/*
+        # To generate certificates
+        openssl \
+        # Required for the setup.py install
+        python3-distutils \
+        && rm -rf /var/lib/apt/lists/*
 
 # Copy preinstalled dependencies from compile image
 COPY --from=compile-image /opt/venv /opt/venv
 
 # Create user
 RUN useradd --create-home --home-dir /home/pyrdp pyrdp
+
+# Make sure we use the virtualenv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install python source and package
+# NOTE: we are no longer doing this in the compile image to avoid long image rebuilds in development
+COPY --from=compile-image /pyrdp /pyrdp
+COPY bin/ /pyrdp/bin/
+COPY pyrdp/ /pyrdp/pyrdp/
+COPY setup.py /pyrdp/setup.py
+RUN cd /pyrdp \
+    && python setup.py install
+
 USER pyrdp
 
 # UTF-8 support on the console output (for pyrdp-player)
 ENV PYTHONIOENCODING=utf-8
-# Make sure we use the virtualenv:
+# Make sure we use the virtualenv
 ENV PATH="/opt/venv/bin:$PATH"
+
 WORKDIR /home/pyrdp

docs/devel.adoc

@@ -23,6 +23,6 @@ NOTE: This is a work in progress
 * linkify issues with vim's: `%s/{uri-issue}\d\+\[#\(\d\+\)\]/#\1/gc`
 
 .Post-release
-* Update version in `setup.py` (+1 bugfix, append 'dev') and commit
+* Update version in `setup.py` (+1 bugfix, append '.dev0') and commit
 * commit msg: Begin development on next release
 

requirements-slim.txt

@@ -0,0 +1,23 @@
+appdirs==1.4.3
+attrs==19.3.0
+Automat==20.2.0
+cffi==1.14.0
+constantly==15.1.0
+cryptography==2.9
+hyperlink==19.0.0
+idna==2.9
+incremental==17.5.0
+names==0.3.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodome==3.9.7
+PyHamcrest==2.0.2
+pyOpenSSL==19.1.0
+pytz==2019.3
+rsa==4.0
+scapy==2.4.3
+service-identity==18.1.0
+six==1.14.0
+Twisted==20.3.0
+zope.interface==5.1.0

requirements.txt

@@ -0,0 +1,27 @@
+appdirs==1.4.3
+attrs==19.3.0
+Automat==20.2.0
+cffi==1.14.0
+constantly==15.1.0
+cryptography==2.9
+dbus-python==1.2.16
+hyperlink==19.0.0
+idna==2.9
+incremental==17.5.0
+names==0.3.0
+notify2==0.3.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodome==3.9.7
+PyHamcrest==2.0.2
+pyOpenSSL==19.1.0
+PySide2==5.14.2
+pytz==2019.3
+rsa==4.0
+scapy==2.4.3
+service-identity==18.1.0
+shiboken2==5.14.2
+six==1.14.0
+Twisted==20.3.0
+zope.interface==5.1.0

setup.py

@@ -3,7 +3,7 @@
 
 #
 # This file is part of the PyRDP project.
-# Copyright (C) 2019 GoSecure Inc.
+# Copyright (C) 2019, 2020 GoSecure Inc.
 # Licensed under the GPLv3 or later.
 #
 
@@ -12,7 +12,7 @@
 from distutils.core import Extension, setup
 
 setup(name='pyrdp',
-    version='0.4.2dev',
+    version='0.4.2.dev0',
     description='Remote Desktop Protocol Man-in-the-Middle and library for Python 3',
     long_description="""Remote Desktop Protocol Man-in-the-Middle and library for Python 3""",
     author='Émilio Gonzalez, Francis Labelle',
@@ -30,19 +30,23 @@
         'bin/pyrdp-player.py'
     ],
     install_requires=[
-        'appdirs',
-        'cryptography',
-        'names',
-        'pyasn1',
-        'pycryptodome',
-        'pyopenssl>=19,<20',
+        'appdirs>=1,<2',
+        'cryptography>=2.2,<3',
+        'names>=0,<1',
+        'pyasn1>=0,<1',
+        'pycryptodome>=3.5,<4',
+        'pyopenssl>=18',
         'pytz',
-        'rsa',
-        'scapy',
-        'service_identity',
-        'twisted',
+        'rsa>=4,<5',
+        'scapy>=2.4,<3',
+        'service_identity>=18',
+        'twisted>=18',
     ],
     extras_require={
-        "full": ['PySide2', 'dbus-python;platform_system!="Windows"', 'notify2;platform_system!="Windows"']
+        "full": [
+            'PySide2>=5.12,<6',
+            'dbus-python>=1.2<1.3;platform_system!="Windows"',
+            'notify2>=0.3,<1;platform_system!="Windows"'
+        ]
     }
 )