diff --git a/.travis.yml b/.travis.yml index 1dc69a4dc..ab2e994bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ env: script: - docker info - docker run --rm --privileged multiarch/qemu-user-static:register --reset - - docker build --build-arg VCS_REF=$TRAVIS_COMMIT --build-arg BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ") -t $DOCKER_REPO:$TRAVIS_BRANCH-$TARGET -f extra/Dockerfile${EXT} . + - travis_wait 40 docker build --build-arg VCS_REF=$TRAVIS_COMMIT --build-arg BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ") -t $DOCKER_REPO:$TRAVIS_BRANCH-$TARGET -f extra/Dockerfile${EXT} . - docker run --rm $DOCKER_REPO:$TRAVIS_BRANCH-$TARGET uname -a - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then echo $DOCKER_PASSWORD | docker login -u "$DOCKER_USERNAME" --password-stdin; fi' - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then docker push $DOCKER_REPO:$TRAVIS_BRANCH-$TARGET; fi' diff --git a/extra/Dockerfile.armv7-armhf b/extra/Dockerfile.armv7-armhf index 3768243d3..64293066e 100644 --- a/extra/Dockerfile.armv7-armhf +++ b/extra/Dockerfile.armv7-armhf @@ -1,3 +1,22 @@ +FROM multiarch/debian-debootstrap:armhf-stretch as builder +ENV PYTHONUNBUFFERED 1 +RUN apt-get update && \ + apt-get upgrade --yes && \ + DEBIAN_FRONTEND="noninteractive" apt-get --yes --option Dpkg::Options::="--force-confnew" --no-install-recommends install \ + python-wheel \ + build-essential \ + python-dev \ + libxslt1.1 \ + libxml2-dev \ + libxslt-dev \ + python-pip \ + zlib1g-dev \ + python-setuptools +WORKDIR /wheels +RUN pip wheel webdavclient3 + + + FROM multiarch/debian-debootstrap:armhf-stretch LABEL maintainer="Marcus Klein " @@ -15,6 +34,8 @@ LABEL org.label-schema.build-date=$BUILD_DATE \ COPY . /tmp/motioneye ENV LD_LIBRARY_PATH /opt/vc/lib +COPY --from=builder /wheels /wheels + RUN apt-get update && \ apt-get upgrade --yes && \ DEBIAN_FRONTEND="noninteractive" apt-get --yes --option Dpkg::Options::="--force-confnew" --no-install-recommends install \ @@ -35,8 +56,7 @@ RUN apt-get update && \ python-tz \ python-wheel \ tzdata \ - v4l-utils \ - zlib1g-dev && \ + v4l-utils && \ git clone --depth 1 https://github.com/Hexxeh/rpi-firmware.git /tmp/rpi-firmware && \ cp -rv /tmp/rpi-firmware/vc/hardfp/opt/vc /opt && \ rm -rf /tmp/rpi-firmware && \ @@ -44,13 +64,20 @@ RUN apt-get update && \ curl -L --output /tmp/motion.deb https://github.com/Motion-Project/motion/releases/download/release-4.2.2/pi_stretch_motion_4.2.2-1_armhf.deb && \ dpkg -i /tmp/motion.deb && \ rm /tmp/motion.deb && \ + pip install webdavclient3 -f /wheels && \ + rm -rf /wheels \ pip install /tmp/motioneye && \ rm -rf /tmp/motioneye && \ + rm -rf /root/.cache/pip/* && \ apt-get purge --yes \ git \ python-pip \ python-setuptools \ - python-wheel && \ + python-wheel \ + build-essential \ + python-dev \ + libxml2-dev \ + libxslt-dev && \ apt-get autoremove --yes && \ apt-get --yes clean && rm -rf /var/lib/apt/lists/* && rm -f /var/cache/apt/*.bin diff --git a/motioneye/config.py b/motioneye/config.py index f822ee9b5..e7d8e5605 100644 --- a/motioneye/config.py +++ b/motioneye/config.py @@ -391,7 +391,8 @@ def get_camera(camera_id, as_lines=False): camera_config = _conf_to_dict(lines, no_convert=['@network_share_name', '@network_smb_ver', '@network_server', '@network_username', '@network_password', '@storage_device', - '@upload_server', '@upload_username', '@upload_password']) + '@upload_server', '@upload_username', '@upload_password', + '@upload_url']) if utils.is_local_motion_camera(camera_config): # determine the enabled status @@ -716,6 +717,7 @@ def motion_camera_ui_to_dict(ui, prev_config=None): '@upload_service': ui['upload_service'], '@upload_server': ui['upload_server'], '@upload_port': ui['upload_port'], + '@upload_url': ui['upload_url'], '@upload_method': ui['upload_method'], '@upload_location': ui['upload_location'], '@upload_subfolders': ui['upload_subfolders'], @@ -1082,6 +1084,7 @@ def motion_camera_dict_to_ui(data): 'upload_service': data['@upload_service'], 'upload_server': data['@upload_server'], 'upload_port': data['@upload_port'], + 'upload_url': data['@upload_url'], 'upload_method': data['@upload_method'], 'upload_location': data['@upload_location'], 'upload_subfolders': data['@upload_subfolders'], @@ -1873,6 +1876,7 @@ def _set_default_motion_camera(camera_id, data): data.setdefault('@upload_service', 'ftp') data.setdefault('@upload_server', '') data.setdefault('@upload_port', '') + data.setdefault('@upload_url', '') data.setdefault('@upload_method', 'POST') data.setdefault('@upload_location', '') data.setdefault('@upload_subfolders', True) diff --git a/motioneye/static/js/main.js b/motioneye/static/js/main.js index 4a29348c2..826323338 100644 --- a/motioneye/static/js/main.js +++ b/motioneye/static/js/main.js @@ -1893,6 +1893,7 @@ function cameraUi2Dict() { 'upload_movie': $('#uploadMovieSwitch')[0].checked, 'upload_service': $('#uploadServiceSelect').val(), 'upload_server': $('#uploadServerEntry').val(), + 'upload_url': $('#uploadUrlEntry').val(), 'upload_port': $('#uploadPortEntry').val(), 'upload_method': $('#uploadMethodSelect').val(), 'upload_location': $('#uploadLocationEntry').val(), @@ -2215,6 +2216,7 @@ function dict2CameraUi(dict) { $('#uploadMovieSwitch')[0].checked = dict['upload_movie']; markHideIfNull('upload_movie', 'uploadMovieSwitch'); $('#uploadServiceSelect').val(dict['upload_service']); markHideIfNull('upload_service', 'uploadServiceSelect'); $('#uploadServerEntry').val(dict['upload_server']); markHideIfNull('upload_server', 'uploadServerEntry'); + $('#uploadUrlEntry').val(dict['upload_url']); markHideIfNull('upload_url', 'uploadUrlEntry'); $('#uploadPortEntry').val(dict['upload_port']); markHideIfNull('upload_port', 'uploadPortEntry'); $('#uploadMethodSelect').val(dict['upload_method']); markHideIfNull('upload_method', 'uploadMethodSelect'); $('#uploadLocationEntry').val(dict['upload_location']); markHideIfNull('upload_location', 'uploadLocationEntry'); @@ -2920,7 +2922,7 @@ function doRestore() { } function doTestUpload() { - var q = $('#uploadPortEntry, #uploadLocationEntry, #uploadServerEntry'); + var q = $('#uploadPortEntry, #uploadLocationEntry, #uploadServerEntry, #uploadUrlEntry'); var valid = true; q.each(function() { this.validate(); @@ -2940,6 +2942,7 @@ function doTestUpload() { service: $('#uploadServiceSelect').val(), server: $('#uploadServerEntry').val(), port: $('#uploadPortEntry').val(), + url: $('#uploadUrlEntry').val(), method: $('#uploadMethodSelect').val(), location: $('#uploadLocationEntry').val(), subfolders: $('#uploadSubfoldersSwitch')[0].checked, diff --git a/motioneye/templates/main.html b/motioneye/templates/main.html index db6609031..e1afd124b 100644 --- a/motioneye/templates/main.html +++ b/motioneye/templates/main.html @@ -419,16 +419,23 @@ + + ? - + Server Address ? - + + Server URL + + ? + + Server Port ? @@ -458,12 +465,12 @@ ? - + Username ? - + Password ? diff --git a/motioneye/uploadservices.py b/motioneye/uploadservices.py index b4ebc1104..3ff53e262 100644 --- a/motioneye/uploadservices.py +++ b/motioneye/uploadservices.py @@ -14,28 +14,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import StringIO +import datetime import ftplib import json import logging import mimetypes import os import os.path -import StringIO import time import urllib import urllib2 +from abc import ABCMeta, abstractmethod +from inspect import isabstract + import pycurl +import webdav3.client import settings import utils -import config -import datetime _STATE_FILE_NAME = 'uploadservices.json' _services = None -class UploadService(object): +class UploadService: + __metaclass__ = ABCMeta + MAX_FILE_SIZE = 1024 * 1024 * 1024 # 1GB NAME = 'base' @@ -50,8 +55,9 @@ def __str__(self): def get_authorize_url(cls): return '/' + @abstractmethod def test_access(self): - return True + pass def upload_file(self, target_dir, filename, camera_name): ctime = os.path.getctime(filename) @@ -87,29 +93,32 @@ def upload_file(self, target_dir, filename, camera_name): raise Exception(msg) try: - f = open(filename) + with open(filename) as f: + data = f.read() except Exception as e: msg = 'failed to open file "%s": %s' % (filename, e) self.error(msg) - raise Exception(msg) + raise - data = f.read() self.debug('size of "%s" is %.3fMB' % (filename, len(data) / 1024.0 / 1024)) mime_type = mimetypes.guess_type(filename)[0] or 'image/jpeg' self.debug('mime type of "%s" is "%s"' % (filename, mime_type)) - self.upload_data(rel_filename, mime_type, data, ctime, camera_name) + self.upload_data(filename, rel_filename, mime_type, data, ctime, camera_name) self.debug('file "%s" successfully uploaded' % filename) - def upload_data(self, filename, mime_type, data, ctime, camera_name): + @abstractmethod + def upload_data(self, full_fname, filename, mime_type, data, ctime, camera_name): pass + @abstractmethod def dump(self): - return {} + pass + @abstractmethod def load(self, data): pass @@ -137,12 +146,22 @@ def error(self, message, **kwargs): def clean_cloud(self, cloud_dir, local_folders): pass - @staticmethod - def get_service_classes(): - return {c.NAME: c for c in UploadService.__subclasses__()} + @classmethod + def get_service_classes(cls): + sub_classes = cls.__subclasses__() + service_classes = {} + # search sub classes for Concrete implementations of UploadService + for sub_cls in sub_classes: + service_classes.update(sub_cls.get_service_classes()) + # add own class to service list if not abstract + if not isabstract(cls): + service_classes[cls.NAME] = cls + # end recursion + return service_classes class GoogleBase: + __metaclass__ = ABCMeta AUTH_URL = 'https://accounts.google.com/o/oauth2/auth' TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' @@ -200,7 +219,7 @@ def _request(self, url, body=None, headers=None, retry_auth=True, method=None): if not self._authorization_key: msg = 'missing authorization key' self.error(msg) - raise Exception(msg) + raise self.debug('requesting credentials') try: @@ -348,7 +367,7 @@ def get_authorize_url(cls): def test_access(self): return self._test_access() - def upload_data(self, filename, mime_type, data, ctime, camera_name): + def upload_data(self, full_fname, filename, mime_type, data, ctime, camera_name): path = os.path.dirname(filename) filename = os.path.basename(filename) @@ -546,7 +565,7 @@ def get_authorize_url(cls): def test_access(self): return self._test_access() - def upload_data(self, filename, mime_type, data, ctime, camera_name): + def upload_data(self, full_fname, filename, mime_type, data, ctime, camera_name): path = os.path.dirname(filename) filename = os.path.basename(filename) dayinfo = datetime.datetime.fromtimestamp(ctime).strftime('%Y-%m-%d') @@ -710,7 +729,7 @@ def test_access(self): return msg - def upload_data(self, filename, mime_type, data, ctime, camera_name): + def upload_data(self, full_fname, filename, mime_type, data, ctime, camera_name): metadata = { 'path': os.path.join(self._clean_location(), filename), 'mode': 'add', @@ -833,6 +852,94 @@ def _request_credentials(self, authorization_key): } +class WebDav(UploadService): + NAME = 'webdav' + + def __init__(self, camera_id): + super(WebDav, self).__init__(camera_id) + self._url = None + self._location = None + self._username = None + self._password = None + + def test_access(self): + client = self._webdav_client() + try: + self.debug('testing access') + self._make_dirs(client, self._location) + client.list(self._location) + return True + except Exception as e: + self.error(str(e), exc_info=True) + return False + + def upload_data(self, full_fname, filename, mime_type, data, ctime, camera_name): + client = self._webdav_client() + self._make_dirs(client, self._location + '/' + os.path.dirname(filename)) + client.upload_file(self._location + '/' + filename, full_fname) + + def dump(self): + return { + 'url': self._url, + 'location': self._location, + 'username': self._username, + 'password': self._password + } + + def load(self, data): + if data.get('url'): + self._url = data['url'] + if data.get('location'): + self._location = data['location'] + if data.get('username'): + self._username = data['username'] + if data.get('password'): + self._password = data['password'] + + def _webdav_client(self): + options = { + 'webdav_hostname': self._url, + 'webdav_login': self._username, + 'webdav_password': self._password + } + return webdav3.client.Client(options) + + def _make_dirs(self, client, remote_dir): + # strip trailing '/' + remote_dir = remote_dir.rstrip('/') + self.debug('testing if remote dir "{}" exists'.format(remote_dir)) + if client.check(remote_dir): + self.debug('remote dir {} exists'.format(remote_dir)) + return + if remote_dir in ['', '.']: + self.debug('Root directory reached. Stop recursion and create directories from here.') + return + # create directories recursively + parent_dir = os.path.dirname(remote_dir) + self._make_dirs(client, parent_dir) + self.debug('creating remote directory {}'.format(remote_dir)) + client.mkdir(remote_dir) + + +class Nextcloud(WebDav): + NAME = 'nextcloud' + + @property + def _dav_url(self): + return "{url}/remote.php/dav/files/{user}".format( + url=self._url.rstrip('/'), + user=self._username + ) + + def _webdav_client(self): + options = { + 'webdav_hostname': self._dav_url, + 'webdav_login': self._username, + 'webdav_password': self._password + } + return webdav3.client.Client(options) + + class FTP(UploadService): NAME = 'ftp' CONN_LIFE_TIME = 60 # don't keep an FTP connection for more than 1 minute @@ -868,7 +975,7 @@ def test_access(self): return str(e) - def upload_data(self, filename, mime_type, data, ctime, camera_name): + def upload_data(self, full_fname, filename, mime_type, data, ctime, camera_name): path = os.path.dirname(filename) filename = os.path.basename(filename) @@ -985,7 +1092,7 @@ def test_access(self): return str(e) - def upload_data(self, filename, mime_type, data, ctime, camera_name): + def upload_data(self, full_fname, filename, mime_type, data, ctime, camera_name): conn = self._get_conn(filename) conn.setopt(pycurl.READFUNCTION, StringIO.StringIO(data).read) @@ -1079,12 +1186,12 @@ def get(camera_id, service_name): def test_access(camera_id, service_name, data): logging.debug('testing access to %s' % service_name) - service = get(camera_id, service_name) - service.load(data) if not service: - return 'unknown upload service %s' % service_name - + msg = 'unknown upload service %s' % service_name + logging.debug(msg) + return msg + service.load(data) return service.test_access() @@ -1176,6 +1283,7 @@ def _save(services): finally: f.close() + def clean_cloud(local_dir, data, info): camera_id = info['camera_id'] service_name = info['service_name'] @@ -1190,6 +1298,7 @@ def clean_cloud(local_dir, data, info): service.load(data) service.clean_cloud(cloud_dir, local_folders) + def exist_in_local(folder, local_folders): if not local_folders: local_folders = [] @@ -1199,6 +1308,7 @@ def exist_in_local(folder, local_folders): return folder in local_folders + def get_local_folders(dir): folders = next(os.walk(dir))[1] return folders diff --git a/setup.py b/setup.py index 5a011f489..a39e8043c 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ def apply_patches(self, base_dir): packages=['motioneye'], - install_requires=['tornado>=3.1,<6', 'jinja2', 'pillow', 'pycurl'], + install_requires=['tornado>=3.1,<6', 'jinja2', 'pillow', 'pycurl', 'webdavclient3==0.12'], package_data={ 'motioneye': [