webrecorder · ikreymer · Jun 8, 2020 · Feb 2, 2018 · Feb 2, 2018 · Feb 5, 2018
diff --git a/.dockerignore b/.dockerignore
@@ -1,8 +1,6 @@
 build/
 dist/
 karma-tests/
-sample_archive/
-tests/
 tests_disabled/
 venv/
 collections/

diff --git a/.travis.yml b/.travis.yml
@@ -5,6 +5,7 @@ python:
   - "3.5"
   - "3.6"
   - "3.7"
+  - "3.8"
 
 dist: xenial
 
@@ -39,6 +40,7 @@ after_success:
 matrix:
   allow_failures:
     - env: WR_TEST=yes
+    - python: "2.7"
 
   exclude:
     - env: WR_TEST=yes

diff --git a/.travis/install.sh b/.travis/install.sh
@@ -2,11 +2,11 @@
 set -e
 
 pip install --upgrade pip setuptools
+pip install 'Markupsafe<2.0.0'
 python setup.py -q install
 pip install -r extra_requirements.txt
 pip install coverage pytest-cov coveralls
 pip install codecov
-npm install
 
 if [ "$WR_TEST" = "yes" ]; then
     git clone https://github.com/webrecorder/webrecorder-tests.git

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,3 +1,39 @@
+pywb 2.4.0 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+This release includes significant update, specifically merging of https://github.com/ukwa/pywb branch into this release.
+A few selected improvements:
+
+* New Access Control System: https://pywb.readthedocs.io/en/latest/manual/access-control.html
+
+* Support for Localization, configuring multiple languages (not enabled by default): https://github.com/ukwa/ukwa-pywb/blob/master/docs/localization.md
+
+* Support for OpenWayback-style XML-based index source (xmlquery)
+
+* Support for loading from WebHDFS via `webhdfs://` scheme.
+
+* Initial support for a new embeds/transclusions replay system, in combination with warcit: https://github.com/webrecorder/warcit/wiki/Warcit-Video-Audio-Conversion
+
+* Proxy mode improvements: handle OPTIONS requests and CORS `#520 <https://github.com/webrecorder/pywb/pull/520>`_
+
+* Memento Prefer header: support for experimental `Prefer` header to select 'raw' or 'rewritten' memento
+
+* Other memento fixes: fix timemap including invalid mementos, correct timegate behavior on top frame `#564 <https://github.com/webrecorder/pywb/pull/564>`_
+
+* Fixes for collection metadata display: `#509 <https://github.com/webrecorder/pywb/pull/520>`_
+
+* Fix for incorrected WARC record length due to re-serialized headers: `#561 <https://github.com/webrecorder/pywb/pull/561>`_
+
+* Filter invalid WARC records `#536 <https://github.com/webrecorder/pywb/pull/536>`_
+
+* Updated fuzzy matching rules and wombat client-side rewriting.
+
+
+For the full changelist, see this PR: `#565 <https://github.com/webrecorder/pywb/pull/565>`_
+
+* Access Control System
+
+
 pywb 2.3.5 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,26 @@
+include LICENSE
+include *.rst
+include *requirements.txt
+include *.yaml
+include *.yml
+include .gitmodules
+include .dockerignore
+include Dockerfile
+include Vagrantfile
+include uwsgi.ini
+include run-tests.py
+include *.sh
+recursive-include static *.js
+recursive-include pywb *.ini
+recursive-include pywb *.md
+recursive-include pywb *.py
+recursive-include pywb *.yaml
+recursive-include sample_archive *.aclj
+recursive-include tests *.po
+recursive-include tests *.yaml
+recursive-include tests_disabled *.py
+recursive-include tests_disabled *.yaml
+recursive-include docs *.bat
+recursive-include docs *.py
+recursive-include docs *.rst
+recursive-include docs Makefile
diff --git a/README.rst b/README.rst
@@ -1,4 +1,4 @@
-Webrecorder pywb 2.3
+Webrecorder pywb 2.4
 ====================
 
 .. image:: https://travis-ci.org/webrecorder/pywb.svg?branch=master
@@ -39,7 +39,11 @@ The 2.x release included a major overhaul of pywb and introduces many new featur
 
 * Standalone, modular `client-side rewriting system (wombat.js) <https://github.com/webrecorder/wombat>`_ to handle most modern web sites.
 
-* Improved 'calendar' query UI, grouping results by year and month, and updated replay banner.
+* Improved 'calendar' query UI with incremental loading, grouping results by year and month, and updated replay banner.
+
+* New in 2.4: Extensible UI customizations system for modifying all aspects of the UI.
+
+* New in 2.4: Robust access control system for blocking or excluding URLs, by prefix or by exact match.
 
 
 Please see the `full documentation <https://pywb.readthedocs.org>`_ for more detailed info on all these features.
@@ -67,12 +71,6 @@ Contributions & Bug Reports
 
 Users are encouraged to fork and contribute to this project to keep improving web archiving tools.
 
-A few key features are high on list of priorities, but have not yet been implemented, including:
-
-* Url Exclusion System
-
-* UI Improvements
-
 If you are interested in contributing, especially to any of these areas, please let us know!
 
 Otherwise, please take a look at `list of current issues <https://github.com/webrecorder/pywb/issues>`_ and feel free to open new ones about any aspect of pywb, including the new documentation.

diff --git a/appveyor.yml b/appveyor.yml
@@ -3,26 +3,29 @@ environment:
     CMD_IN_ENV: "cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd"
 
   matrix:
-    - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python27-x64"
     - PYTHON: "C:\\Python35"
     - PYTHON: "C:\\Python35-x64"
     - PYTHON: "C:\\Python36"
     - PYTHON: "C:\\Python36-x64"
     - PYTHON: "C:\\Python37"
     - PYTHON: "C:\\Python37-x64"
+    - PYTHON: "C:\\Python38"
+    - PYTHON: "C:\\Python38-x64"
+
 
 
 install:        
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "python -m pip install --upgrade pip"
   - "pip install -U setuptools"
+  - "pip install MarkupSafe==1.1.1"
   - "pip install coverage pytest-cov"
   - "pip install cffi"
   - "pip install pyopenssl"
   - "pip install pypiwin32"
   - "pip install certauth boto3 youtube-dl pysocks"
   - "pip install codecov"
+  - "pip install wheel"
 
 build_script:
   - "python setup.py install"

diff --git a/docs/code/pywb.manager.rst b/docs/code/pywb.manager.rst
@@ -4,6 +4,14 @@ pywb\.manager package
 Submodules
 ----------
 
+pywb\.manager\.aclmanager module
+--------------------------------
+
+.. automodule:: pywb.manager.aclmanager
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pywb\.manager\.autoindex module
 -------------------------------
 

diff --git a/docs/code/pywb.rewrite.rst b/docs/code/pywb.rewrite.rst
@@ -100,6 +100,14 @@ pywb\.rewrite\.rewrite\_hls module
     :undoc-members:
     :show-inheritance:
 
+pywb\.rewrite\.rewrite\_js\_workers module
+------------------------------------------
+
+.. automodule:: pywb.rewrite.rewrite_js_workers
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pywb\.rewrite\.rewriteinputreq module
 -------------------------------------
 

diff --git a/docs/code/pywb.rst b/docs/code/pywb.rst
@@ -14,6 +14,18 @@ Subpackages
     pywb.utils
     pywb.warcserver
 
+Submodules
+----------
+
+pywb\.version module
+--------------------
+
+.. automodule:: pywb.version
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
 Module contents
 ---------------
 

diff --git a/docs/code/pywb.utils.rst b/docs/code/pywb.utils.rst
@@ -60,6 +60,14 @@ pywb\.utils\.memento module
     :undoc-members:
     :show-inheritance:
 
+pywb\.utils\.merge module
+-------------------------
+
+.. automodule:: pywb.utils.merge
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pywb\.utils\.wbexception module
 -------------------------------
 

diff --git a/docs/code/pywb.warcserver.rst b/docs/code/pywb.warcserver.rst
@@ -12,6 +12,22 @@ Subpackages
 Submodules
 ----------
 
+pywb\.warcserver\.access\_checker module
+----------------------------------------
+
+.. automodule:: pywb.warcserver.access_checker
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+pywb\.warcserver\.amf module
+----------------------------
+
+.. automodule:: pywb.warcserver.amf
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 pywb\.warcserver\.basewarcserver module
 ---------------------------------------
 

diff --git a/docs/index.rst b/docs/index.rst
@@ -16,6 +16,8 @@ A subset of features provides the basic functionality of a "Wayback Machine".
 
    manual/usage
    manual/configuring
+   manual/access-control
+   manual/ui-customization
    manual/architecture
    manual/apis
    code/pywb