[feat] implement fallback to webkit for bundles

in the end, depending on webengine didn't seem to be such a great idea:
it makes bundles more complex, heavy and brittle.

I tried compiling qt5 and the maintained qtwebkit fork, but that had
also its own set of problems (lastly, pyinstaller didn't work properly;
pyqt also adds a lot of garbage even if you disabled lots of modules you
don't need).

so, this is a fallback to a stretch-based build. it fixes the libGL
library by adding the right version into the bundle.

in this commit I also add a dockerfile for an image that can build
bundles by installing pip dependencies directly in the system. this has
the advantage that the build process is very fast, but the downside that
a new image needs to be pushed to the registry each time that new
dependencies are added; we'll also miss any bug due to introducing new
dependncies (but hopefully it will be catched by other steps in the CI).
The good side of it is that it reduces the need for some hacks in the
build process when bundling from a virtualenv.

6 files changed, 102 insertions, 12 deletions

diff --git a/pkg/build_bundle_with_venv.sh b/pkg/build_bundle_with_venv.sh
index 7e963ff6..e6f05f01 100755
--- a/pkg/build_bundle_with_venv.sh
+++ b/pkg/build_bundle_with_venv.sh
@@ -27,7 +27,7 @@ $VIRTUAL_ENV/bin/pip install appdirs packaging
 # dev version
 # $VIRTUAL_ENV/bin/pip install -U git+https://github.com/pyinstaller/pyinstaller.git@develop
 # try bjones WIP Qt5 branch
-$VIRTUAL_ENV/bin/pip install -U git+https://github.com/bjones1/pyinstaller.git@pyqt5_testing
+$VIRTUAL_ENV/bin/pip install -U git+https://github.com/bjones1/pyinstaller.git@pyqt5_fix
 # ------------------------------------------------------------------------------------------------
 $VIRTUAL_ENV/bin/pip install zope.interface zope.proxy
 
diff --git a/pkg/bundles/build.mk b/pkg/bundles/build.mk
index d92d34d8..bc78fa3f 100644
--- a/pkg/bundles/build.mk
+++ b/pkg/bundles/build.mk
@@ -50,13 +50,14 @@ bundle_linux_helpers:
 	cp src/leap/bitmask/vpn/helpers/linux/se.leap.bitmask.bundle.policy $(DIST_VERSION)/apps/helpers/
 	# now we copy some missing qt stuff... this might be fixed by pyinstaller at some point
 	# this is "/mesa/libGL.so" in ubuntu
-	cp /usr/lib/x86_64-linux-gnu/libGL.so.1 $(DIST_VERSION)/lib/libGL.so.1 || echo "libGL version not found"
+	ls -la /usr/lib/x86_64-linux-gnu/libGL.so.1.2.0
+	cp /usr/lib/x86_64-linux-gnu/libGL.so.1.2.0 $(DIST_VERSION)/lib/libGL.so.1 || echo "Cannot copy libGL"
 	# workaround for https://github.com/pyinstaller/pyinstaller/issues/2737
-	cp /usr/lib/x86_64-linux-gnu/nss/libsoftokn3.so $(DIST_VERSION)/lib/ || echo "libsoftokn3 not found"
-	cp /usr/lib/x86_64-linux-gnu/nss/libfreeblpriv3.so $(DIST_VERSION)/lib/ || echo "libfreeblpriv3 not found"
-	cp /usr/lib/x86_64-linux-gnu/qt5/libexec/QtWebEngineProcess $(DIST_VERSION)/lib/ || echo "QtWebEngineProcess not found"
-	cp /usr/share/qt5/resources/icudtl.dat $(DIST_VERSION)/lib/ || echo "icudtl.dat not found"
-	cp /usr/share/qt5/resources/qtwebengine_resources.pak $(DIST_VERSION)/lib/ || echo "qtwebengine_resources.pak not found"
+	#cp /usr/lib/x86_64-linux-gnu/nss/libsoftokn3.so $(DIST_VERSION)/lib/ || echo "libsoftokn3 not found"
+	#cp /usr/lib/x86_64-linux-gnu/nss/libfreeblpriv3.so $(DIST_VERSION)/lib/ || echo "libfreeblpriv3 not found"
+	#cp /usr/lib/x86_64-linux-gnu/qt5/libexec/QtWebEngineProcess $(DIST_VERSION)/lib/ || echo "QtWebEngineProcess not found"
+	#cp /usr/share/qt5/resources/icudtl.dat $(DIST_VERSION)/lib/ || echo "icudtl.dat not found"
+	#cp /usr/share/qt5/resources/qtwebengine_resources.pak $(DIST_VERSION)/lib/ || echo "qtwebengine_resources.pak not found"
 
 
 bundle_osx_helpers:
diff --git a/pkg/docker_bundle/Dockerfile b/pkg/docker_bundle/Dockerfile
index 30ed6859..b2974dfe 100644
--- a/pkg/docker_bundle/Dockerfile
+++ b/pkg/docker_bundle/Dockerfile
@@ -16,7 +16,8 @@ RUN apt update && apt upgrade -y
 RUN apt install -y --no-install-recommends \
   build-essential virtualenv libpython-dev \
   libsqlcipher-dev libssl-dev libffi-dev \
-  python-pyqt5 python-pyqt5.qtwebengine python-pyqt5.qtwebchannel libqt5webengine-data \
+  python-pyqt5 python-pyqt5.qtwebkit \
+  libqt5printsupport5 \
   qttranslations5-l10n libgl1-mesa-glx \
   libusb-0.1-4 patchelf wget \
-  gnupg1 git
+  gnupg1 git libgl1-mesa-glx
diff --git a/pkg/docker_bundleapt/Dockerfile b/pkg/docker_bundleapt/Dockerfile
new file mode 100644
index 00000000..31484156
--- /dev/null
+++ b/pkg/docker_bundleapt/Dockerfile
@@ -0,0 +1,73 @@
+FROM python:2.7-slim-stretch
+MAINTAINER kali <kali@leap.se>
+
+LABEL Description="Image for building Bitmask bundle based on python:2.7-slim-stretch" Vendor="LEAP" Version="1.0"
+
+RUN apt update && apt upgrade -y
+RUN pip install -U pip
+
+# Install bitmask-dev build deps --------------------------------
+RUN apt install -y --no-install-recommends \
+  build-essential virtualenv libpython-dev \
+  libsqlcipher-dev libssl-dev libffi-dev \
+  libsqlite3-dev libzmq3-dev \
+  python-pyqt5 python-pyqt5.qtwebkit \
+  libqt5printsupport5 \
+  qttranslations5-l10n libgl1-mesa-glx \
+  libusb-0.1-4 patchelf wget \
+  gnupg1 git libgl1-mesa-glx
+
+
+# Pyinstaller custom versions
+#ARG PYINSTALLER_TAG=v3.2
+ARG PYINSTALLER_TAG=pyqt5_fix
+#
+# TODO 
+# change to pyinstaller repo when pyqt5_fix is merged
+#
+#RUN git clone --depth 1 --single-branch --branch $PYINSTALLER_TAG https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller
+RUN git clone --depth 1 --single-branch --branch $PYINSTALLER_TAG https://github.com/bjones1/pyinstaller.git /tmp/pyinstaller
+RUN cd /tmp/pyinstaller && pip install --force-reinstall .
+
+
+# Get Bitmask code --------------------------------------------
+RUN mkdir -p /src/leap
+WORKDIR /src/leap
+
+#ARG BITMASK_BRANCH=master
+#ARG BITMASK_REPO=https://0xacab.org/leap/bitmask-dev
+ARG BITMASK_BRANCH=feat/webkit-fallback
+ARG BITMASK_REPO=https://0xacab.org/kali/bitmask-dev
+#RUN git clone https://0xacab.org/leap/bitmask-dev
+RUN git clone --depth 1 --single-branch --branch $BITMASK_BRANCH $BITMASK_REPO
+
+WORKDIR /src/leap/bitmask-dev
+RUN pip install pysqlcipher --install-option="--bundled"
+RUN pip install leap.bitmask_js
+RUN pip install -r pkg/requirements.pip
+
+RUN pip install ".[mail]"
+RUN make install_pixelated
+
+# TODO -- build on different containers and orchestrate them
+# build gnupg and openvpn binaries --------------------------
+# RUN cd pkg/thirdparty/gnupg && ./build_gnupg.sh
+# RUN cd pkg/thirdparty/openvpn && ./build_openvpn.sh
+# -----------------------------------------------------------
+
+
+# Some hacks to make dist-packages visible from the pip installation path in /usr/local
+RUN ln -s /usr/lib/python2.7/dist-packages/PyQt5/ /usr/local/lib/python2.7/site-packages/PyQt5
+RUN cd /usr/local/lib/python2.7/site-packages/PyQt5 && ln -s QtCore.x86_64-linux-gnu.so QtCore.so
+RUN cd /usr/local/lib/python2.7/site-packages/PyQt5 && ln -s QtGui.x86_64-linux-gnu.so QtGui.so
+RUN cd /usr/local/lib/python2.7/site-packages/PyQt5 && ln -s QtWidgets.x86_64-linux-gnu.so QtWidgets.so
+RUN cd /usr/local/lib/python2.7/site-packages/PyQt5 && ln -s QtWebKit.x86_64-linux-gnu.so QtWebKit.so
+RUN cd /usr/local/lib/python2.7/site-packages/PyQt5 && ln -s QtWebKitWidgets.x86_64-linux-gnu.so QtWebKitWidgets.so
+RUN cd /usr/local/lib/python2.7/site-packages/PyQt5 && ln -s QtNetwork.x86_64-linux-gnu.so QtNetwork.so
+RUN cd /usr/local/lib/python2.7/site-packages/PyQt5 && ln -s QtPrintSupport.x86_64-linux-gnu.so QtPrintSupport.so
+RUN ln -s /usr/lib/python2.7/dist-packages/sip.x86_64-linux-gnu.so /usr/local/lib/python2.7/site-packages/sip.so
+RUN ln -s /usr/lib/python2.7/dist-packages/sipconfig.py /usr/local/lib/python2.7/site-packages/
+RUN ln -s /usr/lib/python2.7/dist-packages/sipconfig_nd.py /usr/local/lib/python2.7/site-packages/
+RUN ln -s /usr/lib/python2.7/dist-packages/sip.pyi /usr/local/lib/python2.7/site-packages/
+
+RUN apt-get install bash
diff --git a/pkg/docker_bundleapt/Makefile b/pkg/docker_bundleapt/Makefile
new file mode 100644
index 00000000..a9eaa7c7
--- /dev/null
+++ b/pkg/docker_bundleapt/Makefile
@@ -0,0 +1,16 @@
+build:
+	docker build -t 0xacab.org:4567/leap/docker/bitmask-bundler:latest .
+push:
+	docker push 0xacab.org:4567/leap/docker/bitmask-bundler:latest
+login:
+	docker login 0xacab.org:4567
+
+# local tests
+
+bundler:
+	docker build -t bundler .
+shell:
+	docker run -i -t bundler bash
+bundle:
+	echo "cd /src/leap/bitmask-dev && VIRTUAL_ENV=/usr/local make bundle && cp -r dist/* /dist" | docker run -i -v /srv/bitmask-builds:/dist -w /dist bundler bash
+
diff --git a/pkg/pyinst/app.spec b/pkg/pyinst/app.spec
index d00d540f..41fec22b 100644
--- a/pkg/pyinst/app.spec
+++ b/pkg/pyinst/app.spec
@@ -50,10 +50,9 @@ elif IS_MAC:
     excludes = ['IPython', 'PySide']
 else:
     hiddenimports.extend(
-        ['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWebEngineWidgets',
+        ['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWebKit', 'PyQt5.QtWebKitWidgets', 'PyQt5.QtPrintSupport',
          'leap.bitmask.gui.icons_rc'])
-    excludes = ['IPython', 'PySide', 
-                'PyQt5.QtPrintSupport', 'PyQt5.QtNetwork.py']
+    excludes = ['IPython', 'PySide', 'PyQt5.QtNetwork.py']
 
 
 VENV = os.environ.get('VIRTUAL_ENV', '')