Merge pull request #52 from bodleian/develop

Release 2022-06-23

.github/workflows/test-build.yml

@@ -0,0 +1,24 @@
+name: Build
+on: [push]
+jobs:
+  Run-Tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: '${{ matrix.python-version }}'
+      - name: Install apt dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install exiftool libstdc++6
+      - name: Install python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Run pytest
+        run: pytest

.gitignore

@@ -2,6 +2,7 @@
 .DS_Store
 .idea/
 venv/
+.venv/
 tests/output
 .cache
 /dist/

.readthedocs.yaml

@@ -0,0 +1,29 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.8"
+    # You can also specify other tool versions:
+    # nodejs: "16"
+    # rust: "1.55"
+    # golang: "1.17"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+# formats:
+#    - pdf
+
+# Optionally declare the Python requirements required to build your docs
+python:
+  install:
+    - requirements: requirements.txt

README.rst

@@ -6,9 +6,8 @@ Image-processing is a Python library that converts a source image (TIFF or JPEG)
 
 At the Bodleian we use it to generate the derivative image files we ingest into Digital Bodleian for both delivery and long-term preservation.
 
-
-.. image:: https://travis-ci.org/bodleian/image-processing.svg?branch=master
-    :target: https://travis-ci.org/bodleian/image-processing
+.. image:: https://github.com/bodleian/image-processing/actions/workflows/test-build.yml/badge.svg
+    :target: https://github.com/bodleian/image-processing/actions/workflows/test-build.yml
     :alt: Build Status
 .. image:: https://readthedocs.org/projects/image-processing/badge/?version=latest
     :target: https://image-processing.readthedocs.io/?badge=latest
@@ -26,7 +25,7 @@ Installation
 
 ``pip install git+https://github.com/bodleian/image-processing.git``
 
-- Compatible with both Python 2.7 and 3.5+
+- Tested on Python 3.8, but should work with most 3.x versions
 
 Dependencies
 ~~~~~~~~~~~~
@@ -38,6 +37,8 @@ Dependencies
 - `Pillow`_ prerequisites before pip install
     - May need some image packages installed before pip installation (may not need lcms2 depending on which TIFF formats you'll be processing)
     - ``yum install lcms2 lcms2-devel libtiff libtiff-devel libjpeg libjpeg-devel``
+        - If Pillow raises "decoder jpeg2k not available" errors while running the unit tests, try installing `openjpeg2`. This should only affect the unit tests, not normal running
+        - You may need to delete and recreate the Python virtual environment for Pillow to properly link to these packages
     - The virtual environment's python binary needs to match the Python.h used by GCC. If necessary, use ``export C_INCLUDE_PATH=/usr/local/include/python2.7/``
 - `Jpylyzer`_ prerequisites before pip install
     - Needs a relatively recent pip version to install - it fails on 1.4.
@@ -88,6 +89,12 @@ To just use Kakadu directly through the wrapper:
     kdu.kdu_compress("input.tif", "output.jp2", kakadu_options=kakadu.DEFAULT_LOSSLESS_COMPRESS_OPTIONS)
 
 
+Development and Testing
+-----------------------
+
+We run a subset of our unit tests on Python versions 3.7+ using Github Actions. Tests that require Kakadu are skipped, as we cannot access the proprietary Kakadu executables from a public repository. Any changes should be tested locally, with Kakadu installed, rather than relying on the CI testing alone. You can use requirements.txt to set up a Python virtual environment.
+
+
 .. inclusion-marker-intro-end
 
 More information

docs/Makefile

@@ -1,10 +1,10 @@
 # Minimal makefile for Sphinx documentation
 #
 
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-SPHINXPROJ    = image_processing
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
 
@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -2,9 +2,9 @@
 #
 # Configuration file for the Sphinx documentation builder.
 #
-# This file does only contain a selection of the most common options. For a
-# full list see the documentation:
-# http://www.sphinx-doc.org/en/stable/config
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
 
 # -- Path setup --------------------------------------------------------------
 
@@ -14,27 +14,22 @@
 #
 import os
 import sys
+import sphinx_rtd_theme
 sys.path.insert(0, os.path.abspath('..'))
 
 
 # -- Project information -----------------------------------------------------
 
-project = u'image_processing'
-copyright = u'2018 Bodleian Libraries, University of Oxford'
-author = u'Mel Mason'
+project = 'image_processing'
+copyright = '2022 Bodleian Libraries, University of Oxford'
+author = 'Mel Mason'
 
-# The short X.Y version
-version = u''
 # The full version, including alpha/beta/rc tags
-release = u'1.6'
+release = '1.10.0'
 
 
 # -- General configuration ---------------------------------------------------
 
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
@@ -44,6 +39,7 @@
     'sphinx.ext.coverage',
     'sphinx.ext.ifconfig',
     #'sphinx.ext.viewcode',
+    'sphinx_rtd_theme'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -58,17 +54,11 @@
 # The master toctree document.
 master_doc = 'index'
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path .
-exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
@@ -79,7 +69,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-#html_theme = 'alabaster'
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -90,7 +80,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = ['_static']
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.

docs/make.bat

@@ -9,9 +9,6 @@ if "%SPHINXBUILD%" == "" (
 )
 set SOURCEDIR=.
 set BUILDDIR=_build
-set SPHINXPROJ=image_processing
-
-if "%1" == "" goto help
 
 %SPHINXBUILD% >NUL 2>NUL
 if errorlevel 9009 (

requirements.txt

@@ -1,7 +1,7 @@
-py==1.4.31
-pytest==2.9.0
+py==1.11.0
+pytest==7.1.2
 uuid==1.30
-Pillow >= 5
-jpylyzer
-sphinx
-sphinx-autobuild
+Pillow==9.1.0
+jpylyzer==2.0.1
+Sphinx==5.0.2
+sphinx-rtd-theme==1.0.0

setup.py

@@ -4,7 +4,7 @@
     long_description = f.read()
 
 setup(name='image_processing',
-      version='1.9.1',
+      version='1.10.0',
       description='Digital Bodleian image processing library',
       url='http://github.com/bodleian/image-processing',
       license="MIT",

tests/data/standard_adobe_jpg.xmp

@@ -1,5 +1,5 @@
 <?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
-<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='Image::ExifTool 10.10'>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='Image::ExifTool 11.88'>
 <rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#'>
 
  <rdf:Description rdf:about=''

tests/test_conversion.py

@@ -2,8 +2,13 @@
 import logging
 import os
 import sys
+
+from pytest import mark
+
 from image_processing import conversion, derivative_files_generator, validation, exceptions, kakadu
 import pytest
+
+from image_processing.utils import cmd_is_executable
 from .test_utils import temporary_folder, filepaths, image_files_match
 from PIL import Image, ImageCms
 
@@ -22,6 +27,7 @@ def test_converts_jpg_to_tiff(self):
             assert os.path.isfile(tiff_file)
             assert image_files_match(tiff_file, filepaths.TIF_FROM_STANDARD_JPG)
 
+    @mark.skipif(not cmd_is_executable('/opt/kakadu/kdu_compress'), reason="requires kakadu installed")
     def test_converts_tif_to_jpeg2000(self):
         with temporary_folder() as output_folder:
             output_file = os.path.join(output_folder, 'output.jp2')
@@ -38,6 +44,7 @@ def test_converts_tif_to_jpeg(self):
             assert os.path.isfile(output_file)
             assert image_files_match(output_file, filepaths.HIGH_QUALITY_JPG_FROM_STANDARD_TIF)
 
+    @mark.skipif(not cmd_is_executable('/opt/kakadu/kdu_compress'), reason="requires kakadu installed")
     def test_converts_tif_to_lossy_jpeg2000(self):
         with temporary_folder() as output_folder:
             output_file = os.path.join(output_folder, 'output.jp2')
@@ -48,6 +55,7 @@ def test_converts_tif_to_lossy_jpeg2000(self):
             # lossy conversions to jp2 don't seem to produce deterministic results, even if we only look at the pixels
             # validation.check_visually_identical(output_file, filepaths.LOSSY_JP2_FROM_STANDARD_TIF)
 
+    @mark.skipif(not cmd_is_executable('/opt/kakadu/kdu_compress'), reason="requires kakadu installed")
     def test_kakadu_errors_are_raised(self):
         with temporary_folder() as output_folder:
             output_file = os.path.join(output_folder, 'output.jp2')

tests/test_derivatives_generator.py

@@ -4,7 +4,10 @@
 import shutil
 import sys
 import pytest
+from pytest import mark
+
 from image_processing import derivative_files_generator, validation, exceptions
+from image_processing.utils import cmd_is_executable
 from .test_utils import temporary_folder, filepaths, image_files_match, xmp_files_match
 
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
@@ -14,6 +17,7 @@ def get_derivatives_generator():
     return derivative_files_generator.DerivativeFilesGenerator(kakadu_base_path=filepaths.KAKADU_BASE_PATH)
 
 
+@mark.skipif(not cmd_is_executable('/opt/kakadu/kdu_compress'), reason="requires kakadu installed")
 class TestDerivativeGenerator(object):
 
     def test_creates_high_quality_jpg(self):