diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000000..5f574f4f59 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,5 @@ +Contributing +============ + +Please see the [project documentation](http://zarr.readthedocs.io/en/stable/contributing.html) for information about contributing to Zarr. + diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..35144271d0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,27 @@ +For bug reports, please follow the template below. For enhancement proposals, feel free +to use whatever template makes sense. + +#### Minimal, reproducible code sample, a copy-pastable example if possible + +```python +# Your code here + +``` + +#### Problem description + +Explain why the current behavior is a problem, what the expected output/behaviour +is, and why the expected output/behaviour is a better solution. + +#### Version and installation information + +Please provide the following: + +* Value of ``zarr.__version__`` +* Value of ``numcodecs.__version__`` +* Version of Python interpreter +* Operating system (Linux/Windows/Mac) +* How Zarr was installed (e.g., "using pip into virtual environment", or "using conda") + +Also, if you think it might be relevant, please provide the output from ``pip freeze`` or +``conda env export`` depending on which was used to install Zarr. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..e31c477477 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +[Description of PR] + +TODO: +* [ ] Add unit tests and/or doctests in docstrings +* [ ] Unit tests and doctests pass locally under Python 3.6 (e.g., run ``tox -e py36`` or + ``pytest -v --doctest-modules zarr``) +* [ ] Unit tests pass locally under Python 2.7 (e.g., run ``tox -e py27`` or + ``pytest -v zarr``) +* [ ] PEP8 checks pass (e.g., run ``tox -e py36`` or ``flake8 --max-line-length=100 zarr``) +* [ ] Add docstrings and API docs for any new/modified user-facing classes and functions +* [ ] New/modified features documented in docs/tutorial.rst +* [ ] Doctests in tutorial pass (e.g., run ``tox -e py36`` or ``python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst``) +* [ ] Changes documented in docs/release.rst +* [ ] Docs build locally (e.g., run ``tox -e docs``) +* [ ] AppVeyor and Travis CI passes +* [ ] Test coverage to 100% (Coveralls passes) diff --git a/LICENSE b/LICENSE index bed26082d7..5bb4df8cf8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Alistair Miles +Copyright (c) 2015-2018 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index e65537900e..a43e87652b 100644 --- a/README.rst +++ b/README.rst @@ -9,11 +9,11 @@ more information. .. image:: https://readthedocs.org/projects/zarr/badge/?version=latest :target: http://zarr.readthedocs.io/en/latest/?badge=latest -.. image:: https://travis-ci.org/alimanfoo/zarr.svg?branch=master - :target: https://travis-ci.org/alimanfoo/zarr +.. image:: https://travis-ci.org/zarr-developers/zarr.svg?branch=master + :target: https://travis-ci.org/zarr-developers/zarr .. image:: https://ci.appveyor.com/api/projects/status/7d4iko59k8hpbbik?svg=true - :target: https://ci.appveyor.com/project/alimanfoo/zarr + :target: https://ci.appveyor.com/project/zarr-developers/zarr -.. image:: https://coveralls.io/repos/github/alimanfoo/zarr/badge.svg?branch=master - :target: https://coveralls.io/github/alimanfoo/zarr?branch=master +.. image:: https://coveralls.io/repos/github/zarr-developers/zarr/badge.svg?branch=master + :target: https://coveralls.io/github/zarr-developers/zarr?branch=master diff --git a/docs/conf.py b/docs/conf.py index 58fe548cd2..5947013200 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,7 +60,7 @@ def __getattr__(cls, name): numpydoc_show_class_members = False numpydoc_class_members_toctree = False -issues_github_path = 'alimanfoo/zarr' +issues_github_path = 'zarr-developers/zarr' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -78,8 +78,8 @@ def __getattr__(cls, name): # General information about the project. project = 'zarr' -copyright = '2016, Alistair Miles' -author = 'Alistair Miles' +copyright = '2018, Zarr Developers' +author = 'Zarr Developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -258,7 +258,7 @@ def __getattr__(cls, name): # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'zarr.tex', 'zarr Documentation', - 'Alistair Miles', 'manual'), + 'Zarr Developers', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000000..d471b092c7 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,346 @@ +Contributing to Zarr +==================== + +Zarr is a community maintained project. We welcome contributions in the form of bug +reports, bug fixes, documentation, enhancement proposals and more. This page provides +information on how best to contribute. + +Asking for help +--------------- + +If you have a question about how to use Zarr, please post your question on +StackOverflow using the `"zarr" tag `_. +If you don't get a response within a day or two, feel free to raise a `GitHub issue +`_ including a link to your StackOverflow +question. We will try to respond to questions as quickly as possible, but please bear +in mind that there may be periods where we have limited time to answer questions +due to other commitments. + +Bug reports +----------- + +If you find a bug, please raise a `GitHub issue +`_. Please include the following items in +a bug report: + +1. A minimal, self-contained snippet of Python code reproducing the problem. You can + format the code nicely using markdown, e.g.:: + + + ```python + import zarr + g = zarr.group() + # etc. + ``` + +2. An explanation of why the current behaviour is wrong/not desired, and what you + expect instead. + +3. Information about the version of Zarr, along with versions of dependencies and the + Python interpreter, and installation information. The version of Zarr can be obtained + from the ``zarr.__version__`` property. Please also state how Zarr was installed, + e.g., "installed via pip into a virtual environment", or "installed using conda". + Information about other packages installed can be obtained by executing ``pip freeze`` + (if using pip to install packages) or ``conda env export`` (if using conda to install + packages) from the operating system command prompt. The version of the Python + interpreter can be obtained by running a Python interactive session, e.g.:: + + $ python + Python 3.6.1 (default, Mar 22 2017, 06:17:05) + [GCC 6.3.0 20170321] on linux + +Enhancement proposals +--------------------- + +If you have an idea about a new feature or some other improvement to Zarr, please raise a +`GitHub issue `_ first to discuss. + +We very much welcome ideas and suggestions for how to improve Zarr, but please bear in +mind that we are likely to be conservative in accepting proposals for new features. The +reasons for this are that we would like to keep the Zarr code base lean and focused on +a core set of functionalities, and available time for development, review and maintenance +of new features is limited. But if you have a great idea, please don't let that stop +you from posting it on GitHub, just please don't be offended if we respond cautiously. + +Contributing code and/or documentation +-------------------------------------- + +Forking the repository +~~~~~~~~~~~~~~~~~~~~~~ + +The Zarr source code is hosted on GitHub at the following location: + +* `https://github.com/zarr-developers/zarr `_ + +You will need your own fork to work on the code. Go to the link above and hit +the "Fork" button. Then clone your fork to your local machine:: + + $ git clone git@github.com:your-user-name/zarr.git + $ cd zarr + $ git remote add upstream git@github.com:zarr-developers/zarr.git + +Creating a development environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To work with the Zarr source code, it is recommended to set up a Python virtual +environment and install all Zarr dependencies using the same versions as are used by +the core developers and continuous integration services. Assuming you have a Python +3 interpreter already installed, and have also installed the virtualenv package, and +you have cloned the Zarr source code and your current working directory is the root of +the repository, you can do something like the following:: + + $ mkdir -p ~/pyenv/zarr-dev + $ virtualenv --no-site-packages --python=/usr/bin/python3.6 ~/pyenv/zarr-dev + $ source ~/pyenv/zarr-dev/bin/activate + $ pip install -r requirements_dev.txt + $ pip install -r requirements_dev_optional.txt + $ pip install -e . + +To verify that your development environment is working, you can run the unit tests:: + + $ pytest -v zarr + +Creating a branch +~~~~~~~~~~~~~~~~~ + +Before you do any new work or submit a pull request, please open an issue on GitHub to +report the bug or propose the feature you'd like to add. + +It's best to synchronize your fork with the upstream repository, then create a +new, separate branch for each piece of work you want to do. E.g.:: + + git checkout master + git fetch upstream + git rebase upstream/master + git push + git checkout -b shiny-new-feature + git push -u origin shiny-new-feature + +This changes your working directory to the 'shiny-new-feature' branch. Keep any changes in +this branch specific to one bug or feature so it is clear what the branch brings to +Zarr. + +To update this branch with latest code from Zarr, you can retrieve the changes from +the master branch and perform a rebase:: + + git fetch upstream + git rebase upstream/master + +This will replay your commits on top of the latest Zarr git master. If this leads to +merge conflicts, these need to be resolved before submitting a pull request. +Alternatively, you can merge the changes in from upstream/master instead of rebasing, +which can be simpler:: + + git fetch upstream + git merge upstream/master + +Again, any conflicts need to be resolved before submitting a pull request. + +Running the test suite +~~~~~~~~~~~~~~~~~~~~~~ + +Zarr includes a suite of unit tests, as well as doctests included in function and class +docstrings and in the tutorial and storage spec. The simplest way to run the unit tests +is to invoke:: + + $ pytest -v zarr + +To also run the doctests within docstrings, run:: + + $ pytest -v --doctest-modules zarr + +To run the doctests within the tutorial and storage spec, run:: + + $ python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst + +Tests can be run under different Python versions using tox. E.g. (assuming you have the +corresponding Python interpreters installed on your system):: + + $ tox -e py27,py34,py35,py36 + +Zarr currently supports Python 2.7 and Python 3.4-3.6, so the above command must +succeed before code can be accepted into the main code base. Note that only the py36 +tox environment runs the doctests, i.e., doctests only need to succeed under Python 3.6. + +All tests are automatically run via Travis (Linux) and AppVeyor (Windows) continuous +integration services for every pull request. Tests must pass under both services before +code can be accepted. Test coverage is also collected automatically via the Coveralls +service, and total coverage over all builds must be 100% (although individual builds +may be lower due to Python 2/3 or other differences). + +Code standards +~~~~~~~~~~~~~~ + +All code must conform to the PEP8 standard. Regarding line length, lines up to 100 +characters are allowed, although please try to keep under 90 wherever possible. +Conformance can be checked by running:: + + $ flake8 --max-line-length=100 zarr + +This is automatically run when invoking ``tox -e py36``. + +Test coverage +~~~~~~~~~~~~~ + +Zarr maintains 100% test coverage under the latest Python stable release (currently +Python 3.6). Both unit tests and docstring doctests are included when computing +coverage. Running ``tox -e py36`` will automatically run the test suite with coverage +and produce a coverage report. This should be 100% before code can be accepted into the +main code base. + +When submitting a pull request, coverage will also be collected across all supported +Python versions via the Coveralls service, and will be reported back within the pull +request. Coveralls coverage must also be 100% before code can be accepted. + +Documentation +~~~~~~~~~~~~~ + +Docstrings for user-facing classes and functions should follow the `numpydoc +`_ standard, +including sections for Parameters and Examples. All examples should run and pass as doctests +under Python 3.6 only. + +Zarr uses Sphinx for documentation, hosted on readthedocs.org. Documentation is +written in the RestructuredText markup language (.rst files) in the ``docs`` folder. +The documentation consists both of prose and API documentation. All user-facing classes +and functions should be included in the API documentation, under the ``docs/api`` +folder. Any new features or important usage information should be included in the +tutorial (``docs/tutorial.rst``). Any changes should also be included in the release +notes (``docs/release.rst``). + +The documentation can be built by running:: + + $ tox -e docs + +The resulting built documentation will be available in the ``.tox/docs/tmp/html`` folder. + +Development best practices, policies and procedures +--------------------------------------------------- + +The following information is mainly for core developers, but may also be of interest to +contributors. + +Merging pull requests +~~~~~~~~~~~~~~~~~~~~~ + +Pull requests submitted by an external contributor should be reviewed and approved by at least +one core developers before being merged. Ideally, pull requests submitted by a core developer +should be reviewed and approved by at least one other core developers before being merged. + +Pull requests should not be merged until all CI checks have passed (Travis, AppVeyor, +Coveralls) against code that has had the latest master merged in. + +Compatibility and versioning policies +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because Zarr is a data storage library, there are two types of compatibility to +consider: API compatibility and data format compatibility. + +API compatibility +""""""""""""""""" + +All functions, classes and methods that are included in the API +documentation (files under ``docs/api/*.rst``) are considered as part of the Zarr **public API**, +except if they have been documented as an experimental feature, in which case they are part of +the **experimental API**. + +Any change to the public API that does **not** break existing third party +code importing Zarr, or cause third party code to behave in a different way, is a +**backwards-compatible API change**. For example, adding a new function, class or method is usually +a backwards-compatible change. However, removing a function, class or method; removing an argument +to a function or method; adding a required argument to a function or method; or changing the +behaviour of a function or method, are examples of **backwards-incompatible API changes**. + +If a release contains no changes to the public API (e.g., contains only bug fixes or +other maintenance work), then the micro version number should be incremented (e.g., +2.2.0 -> 2.2.1). If a release contains public API changes, but all changes are +backwards-compatible, then the minor version number should be incremented +(e.g., 2.2.1 -> 2.3.0). If a release contains any backwards-incompatible public API changes, +the major version number should be incremented (e.g., 2.3.0 -> 3.0.0). + +Backwards-incompatible changes to the experimental API can be included in a minor release, +although this should be minimised if possible. I.e., it would be preferable to save up +backwards-incompatible changes to the experimental API to be included in a major release, and to +stabilise those features at the same time (i.e., move from experimental to public API), rather than +frequently tinkering with the experimental API in minor releases. + +Data format compatibility +""""""""""""""""""""""""" + +The data format used by Zarr is defined by a specification document, which should be +platform-independent and contain sufficient detail to construct an interoperable +software library to read and/or write Zarr data using any programming language. The +latest version of the specification document is available from the :ref:`spec` page. + +Here, **data format compatibility** means that all software libraries that implement a +particular version of the Zarr storage specification are interoperable, in the sense +that data written by any one library can be read by all others. It is obviously +desirable to maintain data format compatibility wherever possible. However, if a change +is needed to the storage specification, and that change would break data format +compatibility in any way, then the storage specification version number should be +incremented (e.g., 2 -> 3). + +The versioning of the Zarr software library is related to the versioning of the storage +specification as follows. A particular version of the Zarr library will +implement a particular version of the storage specification. For example, Zarr version +2.2.0 implements the Zarr storage specification version 2. If a release of the Zarr +library implements a different version of the storage specification, then the major +version number of the Zarr library should be incremented. E.g., if Zarr version 2.2.0 +implements the storage spec version 2, and the next release of the Zarr library +implements storage spec version 3, then the next library release should have version +number 3.0.0. Note however that the major version number of the Zarr library may not +always correspond to the spec version number. For example, Zarr versions 2.x, 3.x, and +4.x might all implement the same version of the storage spec and thus maintain data +format compatibility, although they will not maintain API compatibility. The version number +of the storage specification that is currently implemented is stored under the +``zarr.meta.ZARR_FORMAT`` variable. + +Note that the Zarr test suite includes a data fixture and tests to try and ensure that +data format compatibility is not accidentally broken. See the +:func:`test_format_compatibility` function in the :mod:`zarr.tests.test_storage` module +for details. + +When to make a release +~~~~~~~~~~~~~~~~~~~~~~ + +Ideally, any bug fixes that don't change the public API should be released as soon as +possible. It is fine for a micro release to contain only a single bug fix. + +When to make a minor release is at the discretion of the core developers. There are no +hard-and-fast rules, e.g., it is fine to make a minor release to make a single new +feature available; equally, it is fine to make a minor release that includes a number of +changes. + +Major releases obviously need to be given careful consideration, and should be done as +infrequently as possible, as they will break existing code and/or affect data +compatibility in some way. + +Release procedure +~~~~~~~~~~~~~~~~~ + +Checkout and update the master branch:: + + $ git checkout master + $ git pull + +Verify all tests pass on all supported Python versions, and docs build:: + + $ tox + +Tag the version (where "X.X.X" stands for the version number, e.g., "2.2.0"):: + + $ version=X.X.X + $ git tag -a v$version -m v$version + $ git push --tags + +Release source code to PyPI:: + + $ python setup.py register sdist + $ twine upload dist/zarr-${version}.tar.gz + +Obtain checksum for release to conda-forge:: + + $ openssl sha256 dist/zarr-${version}.tar.gz + +Release to conda-forge by making a pull request against the zarr-feedstock conda-forge +repository, incrementing the version number. diff --git a/docs/index.rst b/docs/index.rst index 585dd7111c..5ddd3368bb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,7 +22,8 @@ Status ------ Zarr is still a young project. Feedback and bug reports are very welcome, please get in touch via -the `GitHub issue tracker `_. +the `GitHub issue tracker `_. See +:doc:`contributing` for further information about contributing to Zarr. Installation ------------ @@ -42,7 +43,7 @@ Alternatively, install Zarr via conda:: To work with Zarr source code in development, install from GitHub:: - $ git clone --recursive https://github.com/alimanfoo/zarr.git + $ git clone --recursive https://github.com/zarr-developers/zarr.git $ cd zarr $ python setup.py install @@ -61,6 +62,13 @@ Contents api spec release + contributing + +Projects using Zarr +------------------- + +If you are using Zarr, we would `love to hear about it +`_. Acknowledgments --------------- diff --git a/docs/release.rst b/docs/release.rst index dbc107d628..27dee0290b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,7 +18,7 @@ Enhancements properties that enable a selection of items in an array to be retrieved or updated. See the :ref:`tutorial_indexing` tutorial section for more information. There is also a `notebook - `_ + `_ with extended examples and performance benchmarks. :issue:`78`, :issue:`89`, :issue:`112`, :issue:`172`. @@ -124,7 +124,7 @@ Enhancements continue to work, however a warning will be raised to encourage use of the ``object_codec`` parameter. :issue:`208`, :issue:`212`. -* **Added support for ``datetime64`` and ``timedelta64`` data types**; +* **Added support for datetime64 and timedelta64 data types**; :issue:`85`, :issue:`215`. * **Array and group attributes are now cached by default** to improve performance with @@ -197,8 +197,8 @@ Maintenance Acknowledgments ~~~~~~~~~~~~~~~ -Code was contributed to this release by :user:`John Kirkham ` and -:user:`Prakhar Goel `. +Code was contributed to this release by :user:`Alistair Miles `, :user:`John +Kirkham ` and :user:`Prakhar Goel `. Documentation was contributed to this release by :user:`Mamy Ratsimbazafy ` and :user:`Charles Noyes `. @@ -340,8 +340,8 @@ Storage ~~~~~~~ The main motivation for re-organizing the code was to create an -abstraction layer between the core array logic and data storage (`#21 -`_). In this release, any +abstraction layer between the core array logic and data storage (:issue:`21`). +In this release, any object that implements the ``MutableMapping`` interface can be used as an array store. See the tutorial sections on :ref:`tutorial_persist` and :ref:`tutorial_storage`, the :ref:`spec_v1`, and the @@ -392,8 +392,7 @@ performance improvement for multi-threaded compression operations. The :mod:`zarr.blosc` extension now automatically detects whether it is running within a single-threaded or multi-threaded program and -adapts its internal behaviour accordingly (`#27 -`_). There is no need for +adapts its internal behaviour accordingly (:issue:`27`). There is no need for the user to make any API calls to switch Blosc between contextual and non-contextual (global lock) mode. See also the tutorial section on :ref:`tutorial_tips_blosc`. @@ -409,8 +408,7 @@ option present in the previous release, and this has been removed. The memory layout within chunks can now be set as either "C" (row-major) or "F" (column-major), which can help to provide better -compression for some data (`#7 -`_). See the tutorial +compression for some data (:issue:`7`). See the tutorial section on :ref:`tutorial_chunks_order` for more information. A bug has been fixed within the ``__getitem__`` and ``__setitem__`` @@ -430,7 +428,7 @@ Thanks to :user:`Matthew Rocklin `, :user:`Stephan Hoyer `, ----- See `v0.4.0 release notes on GitHub -`_. +`_. .. _release_0.3.0: @@ -438,6 +436,6 @@ See `v0.4.0 release notes on GitHub ----- See `v0.3.0 release notes on GitHub -`_. +`_. .. _NumCodecs: http://numcodecs.readthedocs.io/ diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 0fd2f1b88d..7f720860e9 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -423,7 +423,7 @@ Groups also have the :func:`zarr.hierarchy.Group.tree` method, e.g.:: If you're using Zarr within a Jupyter notebook, calling ``tree()`` will generate an interactive tree representation, see the `repr_tree.ipynb notebook -`_ +`_ for more examples. .. _tutorial_attrs: @@ -860,7 +860,7 @@ groups and datasets, use :func:`zarr.convenience.copy_all`, e.g.:: └── spam (100,) int64 If you need to copy data between two Zarr groups, the -func:`zarr.convenience.copy` and :func:`zarr.convenience.copy_all` functions can +:func:`zarr.convenience.copy` and :func:`zarr.convenience.copy_all` functions can be used and provide the most flexibility. However, if you want to copy data in the most efficient way possible, without changing any configuration options, the :func:`zarr.convenience.copy_store` function can be used. This function diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt new file mode 100644 index 0000000000..ad6f7064c6 --- /dev/null +++ b/requirements_dev_optional.txt @@ -0,0 +1,2 @@ +bsddb3==6.2.5 +lmdb==0.93 diff --git a/setup.py b/setup.py index 5ae09c7ef2..7f19f50709 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup -DESCRIPTION = 'A minimal implementation of chunked, compressed, ' \ +DESCRIPTION = 'An implementation of chunked, compressed, ' \ 'N-dimensional arrays for Python.' with open('README.rst') as f: @@ -46,10 +46,8 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', ], - author='Alistair Miles', - author_email='alimanfoo@googlemail.com', maintainer='Alistair Miles', maintainer_email='alimanfoo@googlemail.com', - url='https://github.com/alimanfoo/zarr', + url='https://github.com/zarr-developers/zarr', license='MIT', ) diff --git a/tox.ini b/tox.ini index 9d0db40a81..3c0e931260 100644 --- a/tox.ini +++ b/tox.ini @@ -20,10 +20,10 @@ commands = py36: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst py36: flake8 --max-line-length=100 zarr deps = + py27: backports.lzma -rrequirements_dev.txt # linux only - bsddb3==6.2.5 - lmdb==0.93 + -rrequirements_dev_optional.txt [testenv:docs] diff --git a/zarr/convenience.py b/zarr/convenience.py index 263ae3d0bc..19de7b2826 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -403,6 +403,12 @@ def tree(grp, expand=False, level=None): -------- zarr.hierarchy.Group.tree + Notes + ----- + Please note that this is an experimental feature. The behaviour of this + function is still evolving and the default output and/or parameters may change + in future versions. + """ return TreeViewer(grp, expand=expand, level=level) @@ -547,6 +553,12 @@ def copy_store(source, dest, source_path='', dest_path='', excludes=None, array([ 0, 1, 2, ..., 97, 98, 99]) >>> store2.close() # zip stores need to be closed + Notes + ----- + Please note that this is an experimental feature. The behaviour of this + function is still evolving and the default behaviour and/or parameters may change + in future versions. + """ # normalize paths @@ -751,6 +763,12 @@ def copy(source, dest, name=None, shallow=False, without_attrs=False, log=None, dry run: 3 copied, 1 skipped (3, 1, 0) + Notes + ----- + Please note that this is an experimental feature. The behaviour of this + function is still evolving and the default behaviour and/or parameters may change + in future versions. + """ # value checks @@ -1021,6 +1039,12 @@ def copy_all(source, dest, shallow=False, without_attrs=False, log=None, └── spam (100,) int64 >>> source.close() + Notes + ----- + Please note that this is an experimental feature. The behaviour of this + function is still evolving and the default behaviour and/or parameters may change + in future versions. + """ # value checks diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 466af203ca..e9565caa13 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -603,6 +603,12 @@ def tree(self, expand=False, level=None): └── quux └── baz (100,) float64 + Notes + ----- + Please note that this is an experimental feature. The behaviour of this + function is still evolving and the default output and/or parameters may change + in future versions. + """ return TreeViewer(self, expand=expand, level=level) diff --git a/zarr/tests/test_convenience.py b/zarr/tests/test_convenience.py index a786a0fa3e..c77006c4f6 100644 --- a/zarr/tests/test_convenience.py +++ b/zarr/tests/test_convenience.py @@ -587,7 +587,7 @@ def temp_h5f(): return h5f -@pytest.mark.skipif(h5py is None, reason='h5py not installed') +@unittest.skipIf(h5py is None, 'h5py is not installed') class TestCopyHDF5ToZarr(TestCopy): def __init__(self, *args, **kwargs): @@ -598,7 +598,7 @@ def __init__(self, *args, **kwargs): self.new_dest = group -@pytest.mark.skipif(h5py is None, reason='h5py not installed') +@unittest.skipIf(h5py is None, 'h5py is not installed') class TestCopyZarrToHDF5(TestCopy): def __init__(self, *args, **kwargs): @@ -609,7 +609,7 @@ def __init__(self, *args, **kwargs): self.new_dest = temp_h5f -@pytest.mark.skipif(h5py is None, reason='h5py not installed') +@unittest.skipIf(h5py is None, 'h5py is not installed') class TestCopyHDF5ToHDF5(TestCopy): def __init__(self, *args, **kwargs): diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index d0c46af3b7..83c64545f6 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -1251,7 +1251,7 @@ def test_nbytes_stored(self): bsddb3 = None -@pytest.mark.skipif(bsddb3 is None, reason='bsddb3 not installed') +@unittest.skipIf(bsddb3 is None, 'bsddb3 is not installed') class TestArrayWithDBMStoreBerkeleyDB(TestArray): @staticmethod @@ -1276,7 +1276,7 @@ def test_nbytes_stored(self): lmdb = None -@pytest.mark.skipif(lmdb is None, reason='lmdb is not installed') +@unittest.skipIf(lmdb is None, 'lmdb is not installed') class TestArrayWithLMDBStore(TestArray): @staticmethod @@ -1295,7 +1295,7 @@ def test_nbytes_stored(self): pass # not implemented -@pytest.mark.skipif(lmdb is None, reason='lmdb is not installed') +@unittest.skipIf(lmdb is None, 'lmdb is not installed') class TestArrayWithLMDBStoreNoBuffers(TestArray): @staticmethod @@ -1421,45 +1421,47 @@ def test_hexdigest(self): assert '95d40c391f167db8b1290e3c39d9bf741edacdf6' == z.hexdigest() -# TODO rely on backports and remove the PY2 exclusion -if not PY2: # pragma: py2 no cover +try: + from numcodecs import LZMA +except ImportError: # pragma: no cover + LZMA = None - from zarr.codecs import LZMA - class TestArrayWithLZMACompressor(TestArray): +@unittest.skipIf(LZMA is None, 'LZMA codec not available') +class TestArrayWithLZMACompressor(TestArray): - def create_array(self, read_only=False, **kwargs): - store = dict() - compressor = LZMA(preset=1) - kwargs.setdefault('compressor', compressor) - cache_metadata = kwargs.pop('cache_metadata', True) - cache_attrs = kwargs.pop('cache_attrs', True) - init_array(store, **kwargs) - return Array(store, read_only=read_only, cache_metadata=cache_metadata, - cache_attrs=cache_attrs) + def create_array(self, read_only=False, **kwargs): + store = dict() + compressor = LZMA(preset=1) + kwargs.setdefault('compressor', compressor) + cache_metadata = kwargs.pop('cache_metadata', True) + cache_attrs = kwargs.pop('cache_attrs', True) + init_array(store, **kwargs) + return Array(store, read_only=read_only, cache_metadata=cache_metadata, + cache_attrs=cache_attrs) - def test_hexdigest(self): - # Check basic 1-D array - z = self.create_array(shape=(1050,), chunks=100, dtype='i4') - assert '93ecaa530a1162a9d48a3c1dcee4586ccfc59bae' == z.hexdigest() + def test_hexdigest(self): + # Check basic 1-D array + z = self.create_array(shape=(1050,), chunks=100, dtype='i4') + assert '93ecaa530a1162a9d48a3c1dcee4586ccfc59bae' == z.hexdigest() - # Check basic 1-D array with different type - z = self.create_array(shape=(1050,), chunks=100, dtype='f4') - assert '04a9755a0cd638683531b7816c7fa4fbb6f577f2' == z.hexdigest() + # Check basic 1-D array with different type + z = self.create_array(shape=(1050,), chunks=100, dtype='f4') + assert '04a9755a0cd638683531b7816c7fa4fbb6f577f2' == z.hexdigest() - # Check basic 2-D array - z = self.create_array(shape=(20, 35,), chunks=10, dtype='i4') - assert 'b93b163a21e8500519250a6defb821d03eb5d9e0' == z.hexdigest() + # Check basic 2-D array + z = self.create_array(shape=(20, 35,), chunks=10, dtype='i4') + assert 'b93b163a21e8500519250a6defb821d03eb5d9e0' == z.hexdigest() - # Check basic 1-D array with some data - z = self.create_array(shape=(1050,), chunks=100, dtype='i4') - z[200:400] = np.arange(200, 400, dtype='i4') - assert 'cde499f3dc945b4e97197ff8e3cf8188a1262c35' == z.hexdigest() + # Check basic 1-D array with some data + z = self.create_array(shape=(1050,), chunks=100, dtype='i4') + z[200:400] = np.arange(200, 400, dtype='i4') + assert 'cde499f3dc945b4e97197ff8e3cf8188a1262c35' == z.hexdigest() - # Check basic 1-D array with attributes - z = self.create_array(shape=(1050,), chunks=100, dtype='i4') - z.attrs['foo'] = 'bar' - assert 'e2cf3afbf66ad0e28a2b6b68b1b07817c69aaee2' == z.hexdigest() + # Check basic 1-D array with attributes + z = self.create_array(shape=(1050,), chunks=100, dtype='i4') + z.attrs['foo'] = 'bar' + assert 'e2cf3afbf66ad0e28a2b6b68b1b07817c69aaee2' == z.hexdigest() class TestArrayWithFilters(TestArray): diff --git a/zarr/tests/test_filters.py b/zarr/tests/test_filters.py index 1b03081213..4ac256a438 100644 --- a/zarr/tests/test_filters.py +++ b/zarr/tests/test_filters.py @@ -9,7 +9,6 @@ from numcodecs import (AsType, Delta, FixedScaleOffset, PackBits, Categorize, Zlib, Blosc, BZ2, Quantize) from zarr.creation import array -from zarr.compat import PY2 compressors = [ @@ -19,9 +18,11 @@ Blosc(), ] -# TODO rely on backports and remove PY2 exclusion -if not PY2: # pragma: py2 no cover - from zarr.codecs import LZMA +try: + from numcodecs import LZMA +except ImportError: # pragma: no cover + LZMA = None +else: compressors.append(LZMA()) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 1b89c4585e..f47012cf88 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -892,7 +892,7 @@ def create_store(): bsddb3 = None -@pytest.mark.skipif(bsddb3 is None, reason='bsddb3 is not installed') +@unittest.skipIf(bsddb3 is None, 'bsddb3 is not installed') class TestGroupWithDBMStoreBerkeleyDB(TestGroup): @staticmethod @@ -909,7 +909,7 @@ def create_store(): lmdb = None -@pytest.mark.skipif(lmdb is None, reason='lmdb is not installed') +@unittest.skipIf(lmdb is None, 'lmdb is not installed') class TestGroupWithLMDBStore(TestGroup): @staticmethod diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index fe5818be8d..f68f8a6ed6 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -90,10 +90,10 @@ def test_writeable_values(self): store = self.create_store() # __setitem__ should accept any value that implements buffer interface - store['foo'] = b'bar' - store['foo'] = bytearray(b'bar') - store['foo'] = array.array('B', b'bar') - store['foo'] = np.frombuffer(b'bar', dtype='u1') + store['foo1'] = b'bar' + store['foo2'] = bytearray(b'bar') + store['foo3'] = array.array('B', b'bar') + store['foo4'] = np.frombuffer(b'bar', dtype='u1') def test_update(self): store = self.create_store() @@ -777,7 +777,7 @@ def create_store(self): gdbm = None -@pytest.mark.skipif(gdbm is None, reason='gdbm is not installed') +@unittest.skipIf(gdbm is None, 'gdbm is not installed') class TestDBMStoreGnu(TestDBMStore): def create_store(self): @@ -787,13 +787,13 @@ def create_store(self): return store -if not PY2: +if not PY2: # pragma: py2 no cover try: import dbm.ndbm as ndbm except ImportError: # pragma: no cover ndbm = None - @pytest.mark.skipif(ndbm is None, reason='ndbm is not installed') + @unittest.skipIf(ndbm is None, 'ndbm is not installed') class TestDBMStoreNDBM(TestDBMStore): def create_store(self): @@ -809,7 +809,7 @@ def create_store(self): bsddb3 = None -@pytest.mark.skipif(bsddb3 is None, reason='bsddb3 is not installed') +@unittest.skipIf(bsddb3 is None, 'bsddb3 is not installed') class TestDBMStoreBerkeleyDB(TestDBMStore): def create_store(self): @@ -825,7 +825,7 @@ def create_store(self): lmdb = None -@pytest.mark.skipif(lmdb is None, reason='lmdb is not installed') +@unittest.skipIf(lmdb is None, 'lmdb is not installed') class TestLMDBStore(StoreTests, unittest.TestCase): def create_store(self):