diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..5677300 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,29 @@ +# https://coverage.readthedocs.io/en/latest/config.html +# .coveragerc to control coverage.py +[run] +branch = True +omit = + cartoreader_lite/output/* + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + #raise AssertionError + #raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +ignore_errors = True + +[html] +directory = coverage_html diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..1cf791a --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,59 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CI Tests + +on: + push: + branches: [ public ] + pull_request: + branches: [ public ] + +jobs: + + test_lib_pip_ubuntu: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install with pip + run: | + python -m pip install --upgrade pip + pip install -e .[tests] + python tests/generate_test_data.py + - name: Test with pytest + run: | + python -m pytest --cov-config=.coveragerc --cov=cartoreader_lite tests/ + bash <(curl -s https://codecov.io/bash) -t ${{ secrets.CODECOV_TOKEN }} + + test_lib_pip_windows: + + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install with pip + run: | + python -m pip install --upgrade pip + pip install -e .[tests] + python tests/generate_test_data.py + - name: Test with pytest + run: | + python -m pytest --cov-config=.coveragerc --cov=cartoreader_lite tests/ diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..f0e9f5d --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,36 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build -s + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb342fa --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*__pycache__* +*.pyc +*.egg-info* +docs/_* +cartoreader_lite/output/* +.pytest_cache* +.vscode* +coverage_html* +dist* +.coverage diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0ad25db --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..21a2127 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# CARTOreader - lite + +[![CI Tests](https://github.com/thomgrand/cartoreader-lite/actions/workflows/python-package.yml/badge.svg)](https://github.com/thomgrand/cartoreader-lite/actions/workflows/python-package.yml) +[![codecov](https://codecov.io/gh/thomgrand/cartoreader-lite/branch/main/graph/badge.svg?token=4A7DD8DWXW)](https://codecov.io/gh/thomgrand/cartoreader-lite) + +This repository is an inofficial reader to easily process exported [CARTO3 data](https://www.jnjmedicaldevices.com/en-US/product/carto-3-system) in Python. +It does not provide the more extensive capabilities to analyze the signals, such as [OpenEP](https://openep.io/), but is rather meant as a simple reader to import CARTO data. +The loaded time data is imported in [pandas](https://pandas.pydata.org) and the meshes in [VTK](https://vtk.org/) provided through [PyVista](https://www.pyvista.org), allowing for easy access, export and interoperatibility with existing software. + +# Installation + +To install `cartoreader_lite`, you have to clone the repository and install the libary using `pip`. + +```bash +git clone https://github.com/thomgrand/cartoreader-lite +cd cartoreader-lite +pip install -e . +``` + +# Usage + +To test the library, you first need to get CARTO3 data. +None is provided with this repository, but you can download the testing data provided by [OpenEP](https://openep) to quickly try out the library (make sure the libary was installed first): + +```bash +python tests/generate_test_data.py +``` + +```python +from cartoreader_lite import CartoStudy +study_dir = "openep-testingdata/Carto/Export_Study-1-11_25_2021-15-01-32" +study_name = "Study 1 11_25_2021 15-01-32.xml" +study = CartoStudy(study_dir, study_name, + carto_map_kwargs={"discard_invalid_points": False} #All points of the example are outside the WOI, which would be by default discarded + ) +ablation_points = pv.PolyData(np.stack(study.ablation_data.session_avg_data["pos"].to_numpy())) +ablation_points.point_data["RFIndex"] = study.ablation_data.session_avg_data["RFIndex"] +plotter = pv.Plotter() +plotter.add_mesh(ablation_points, cmap="jet") +plotter.add_mesh(study.maps[2].mesh) +plotter.show() +``` + +You should see the recorded map of the [OpenEP](https://openep.io) example, together with its recorded points like below. + +![openep-example](docs/figures/openep-example.png) + +# Documentation + +[https://cartoreader-lite.readthedocs.io/en/latest](https://cartoreader-lite.readthedocs.io/en/latest) + +# Citation + +If you use the library in your scientific projects, please cite the associated Zenodo archive: https://www.zenodo.org. + +```bibtex +%TODO: FIM +@software{thomas_grandits_2021_5594452, + author = {Thomas Grandits}, + title = {A Fast Iterative Method Python package}, + month = oct, + year = 2021, + publisher = {Zenodo}, + version = {1.1}, + doi = {10.5281/zenodo.5594452}, + url = {https://doi.org/10.5281/zenodo.5594452} +} +``` diff --git a/cartoreader_lite/__init__.py b/cartoreader_lite/__init__.py new file mode 100644 index 0000000..06940a7 --- /dev/null +++ b/cartoreader_lite/__init__.py @@ -0,0 +1,4 @@ +from .high_level.study import CartoStudy, CartoAuxMesh, CartoMap, CartoPointDetailData, AblationSites + +__version__ = "1.0.0" +__author__ = "Thomas Grandits" diff --git a/cartoreader_lite/high_level/__init__.py b/cartoreader_lite/high_level/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cartoreader_lite/high_level/study.py b/cartoreader_lite/high_level/study.py new file mode 100644 index 0000000..e07672d --- /dev/null +++ b/cartoreader_lite/high_level/study.py @@ -0,0 +1,353 @@ +from __future__ import annotations #recursive type hinting +import logging as log +import pickle +from typing import Dict, List, Tuple, IO, Union + +from cartoreader_lite.low_level.utils import convert_fname_to_handle, simplify_dataframe_dtypes, unify_time_data, xyz_to_pos_vec +from ..low_level.study import CartoLLStudy, CartoLLMap, CartoAuxMesh +import pandas as pd +import numpy as np +import re +import gzip +from os import PathLike +import os +import pyvista as pv +from ..postprocessing.geometry import project_points + +dtype_simplify_dict = {"InAccurateSeverity": np.int8, + "ChannelID": np.int32, + "MetalSeverity": np.int8, + "NeedZeroing": np.int8, + "ChannelID": np.int16, + "TagIndexStatus": np.int8, + "Valid": np.int8} + +#np.iinfo(np.int8) +#np.issubdtype(np.int8, np.integer) + +class AblationSites(): + """High level class for easily referencing ablation sites read from the visitag files. + Automatically generated from :class:`.CartoStudy`. + + Parameters + ---------- + visitag_data : Dict[str, pd.DataFrame] + Visitag data read by the low level function :ref:`.low_level.visitags.read_visitag_dir`. + resample_unified_time : bool, optional + If true, the visitag file with different time intervals and frequencies will be resampled into a single pandas Dataframe with unified time steps. + By default True + position_to_vec : bool, optional + If true, the entries X, Y, Z of the dataframes will be unified in the resulting tables to a single 3D vector pos. + By default, True + """ + + session_avg_data : pd.DataFrame #: Contains average data of each ablation session, such as :term:`RFIndex`, average force and position + session_time_data : List[Tuple[int, pd.DataFrame]] #: List of time data associated with each Ablation session. Each item contains the session ID + the ablation data over the course of the session + session_rf_data : List[Tuple[int, pd.DataFrame]] = None #: Ablation data (impedance, power, temperature, ...) provided by the low level classes. Only present if `resample_unified_time` was False + session_force_data : List[Tuple[int, pd.DataFrame]] = None #: Force data provided by the low level classes. Only present if `resample_unified_time` was False + + def __init__(self, visitag_data : Dict[str, pd.DataFrame], resample_unified_time=True, + position_to_vec=True) -> None: + + if resample_unified_time: + #Contact force data uses a different time label, but the timings look the same as the other data + contact_force_data = visitag_data["ContactForceData"].rename(columns={"Time": "TimeStamp"}) + time_data = unify_time_data([visitag_data["RawPositions"], visitag_data["AblationData"], contact_force_data], time_k="TimeStamp", + time_interval=100, kind="quadratic") + self.session_time_data = list(simplify_dataframe_dtypes(time_data, dtype_simplify_dict).groupby("Session")) + + else: + self.session_time_data = list(simplify_dataframe_dtypes(visitag_data["RawPositions"], dtype_simplify_dict).groupby("Session")) + self.session_rf_data = list(simplify_dataframe_dtypes(visitag_data["AblationData"], dtype_simplify_dict).groupby("Session")) + self.session_force_data = list(simplify_dataframe_dtypes(visitag_data["ContactForceData"], dtype_simplify_dict).groupby("Session")) + + self.session_avg_data = simplify_dataframe_dtypes(visitag_data["Sites"], dtype_simplify_dict) + + if position_to_vec: + self.session_avg_data = xyz_to_pos_vec(self.session_avg_data) + self.session_time_data = [(session_id, xyz_to_pos_vec(time_data)) for session_id, time_data in self.session_time_data] + +ecg_gain_re = re.compile("\s*Raw ECG to MV \(gain\)\s*\=\s*(-?\d+\.?\d*)\s*") +ecg_labels = ["I", "II", "III", "aVR", "aVL", "aVF"] + [f"V{i+1}" for i in range(6)] +ecg_labels_re = [re.compile(l + "\(\d+\)") for l in ecg_labels] +egm_label_re = re.compile("(.+)\((\d+)\)") + +class CartoPointDetailData(): + """Detailed data associated to a CARTO3 point. + + Parameters + ---------- + main_data : pd.Series + Metadata of the point, such as ID and mean position + raw_data : Tuple[Dict[str, Dict], Dict[str, Dict]] + Raw data provided by :class:`cartoreader_lite.low_level.study.CartoLLMap`, including details such as :term:`EGMs`. + remove_egm_header_numbers : bool, optional + If true, the :term:`EGM` header numbers will be discarded (e.g. I(110) -> I). + By default True + """ + + id : int #: Carto generated ID of the point + pos : np.ndarray #: Position of the point in 3D + cath_orientation : np.ndarray #: 3D-Orientation of the catheter while recording the point + cath_id : int #: ID of the Catheter + woi : np.ndarray #: :term:`WOI`. Only recordings located in this window are deemed valid. + start_time : int #: System start time of the recording + ref_annotation : int #: Reference annotation to synchronize all recordings + map_annotation : int #: Annotation of the activation of this point, also often called :term:`LAT` + uni_volt : float #: Unipolar voltage magnitdue + bip_volt : float #: Bipolar voltage magnitdue + connectors : List[str] #: List of the recorded connector names + ecg_gain : float #: Gain of the recorded :term:`ECGs` + ecg_metadata : object #: Additional provided metadata regarding the :term:`ECGs` or :term:`EGMs` + surface_ecg : pd.DataFrame #: Recorded surface :term:`ECG`. Has type np.int16 and needs to be multiplied by :attr:`~CartoPointDetailData.ecg_gain` to get the ECG in Volts + egm : pd.DataFrame #: Recorded electrograms at the point through the connectors. Naming and columns differ for each setup + + def __init__(self, main_data : pd.Series, raw_data : Tuple[Dict[str, Dict], Dict[str, Dict]], + remove_egm_header_numbers=True) -> None: + + #Read the easy metadata first + self.id = main_data["Id"] + self.pos = main_data["Position3D"] + self.cath_orientation = main_data["CathOrientation"] + self.cath_id = main_data["Cath_Id"] + + #Metadata is the first tuple object, actual data in the second tuple + #WOI + self.woi = np.array([float(raw_data[0]["WOI"]["From"]), float(raw_data[0]["WOI"]["To"])]) + self.start_time = int(raw_data[0]["Annotations"]["StartTime"]) + self.ref_annotation = int(raw_data[0]["Annotations"]["Reference_Annotation"]) + self.map_annotation = int(raw_data[0]["Annotations"]["Map_Annotation"]) + + #Voltages + self.uni_volt = float(raw_data[0]["Voltages"]["Unipolar"]) + self.bip_volt = float(raw_data[0]["Voltages"]["Bipolar"]) + + #Connectors + self.connectors = list(raw_data[1]["connector_data"].keys()) + + #Contact Forces + if "contact_force_data" in raw_data[1]: + self.contact_force_metadata, self.contact_force_data = raw_data[1]["contact_force_data"] + self.contact_force_data = simplify_dataframe_dtypes(self.contact_force_data, dtype_simplify_dict) + + #TODO: Not yet sure what the OnAnnotation file does + + #ECG + ecg_gain_str = raw_data[1]["ecg"][0][1] #Pass ECG metadata + self.ecg_gain = float(ecg_gain_re.match(ecg_gain_str).group(1)) + self.ecg_metadata = raw_data[1]["ecg"][0][2] + #self.ecg_export_version_number = data[1]["ecg"][0][0]"" + ecg_data = raw_data[1]["ecg"][1] + assert np.iinfo(np.int16).min <= ecg_data.min().min() and np.iinfo(np.int16).max >= ecg_data.max().max(), "ECG can not be simplified to np.in16" + ecg_data = ecg_data.astype(np.int16) + + #Find the surface ECGs in the data and split the data into surface and other EGMs + surface_ecg_inds = [[j for j, c in enumerate(ecg_data.columns) if l.match(c) is not None] for l in ecg_labels_re] + assert all([len(l) == 1 for l in surface_ecg_inds]), "12-lead ECG not present in the point data" + surface_ecg_inds = [l[0] for l in surface_ecg_inds] + self.surface_ecg = ecg_data.iloc[:, surface_ecg_inds] + self.egm = ecg_data.drop(ecg_data.columns[surface_ecg_inds], axis=1) + + #Remove the numbers after electrode description numbering + if remove_egm_header_numbers: + self.surface_ecg.columns = np.array(ecg_labels) + self.egm.columns = np.array([egm_label_re.match(col).group(1) for col in self.egm.columns]) + + @property + def main_point_pd_row(self): + pd_attrs = ["id", "pos", "cath_orientation", "cath_id", "woi", "start_time", "ref_annotation", + "map_annotation", "uni_volt", "bip_volt", "connectors", + ] #"surface_ecg", "egm"] + + #This represents a single row of the returning pandas DataFrame + return {**{k: getattr(self, k) for k in pd_attrs}, **{"detail": self}} + +class CartoMap(): + + """High level container for carto maps with the associated point data and mesh. + + Parameters + ---------- + ll_map : CartoLLMap + The low level study to load and simplify + """ + + points : pd.DataFrame #: Recorded point data associated with this map. The column `detail` returns the associated :class:`CartoPointDetailData` where the ECGs and EGMs can be found. + mesh : pv.UnstructuredGrid #: Mesh associated with the map + + def _simplify(self, ll_map : CartoLLMap, discard_invalid_points=True, remove_egm_header_numbers=True, + proj_points=True): + """Function to simplify the data given by the lower level ll_map. + + Parameters + ---------- + ll_map : CartoLLMap + Data of the low level map to be simplified into this high level map + discard_invalid_points : bool, optional + If true, points with :term:`LAT` outside the :term:`WOI` will be automatically discarded. + By default True + """ + self.name = ll_map.name + + #Point data + if len(ll_map.points_main_data) > 0: + self._points_raw = np.array([CartoPointDetailData(main_data, raw_data, remove_egm_header_numbers) for (row_i, main_data), raw_data in zip(ll_map.points_main_data.iterrows(), ll_map.point_raw_data)]) + self.points = pd.DataFrame([p.main_point_pd_row for p in self._points_raw]) + + if discard_invalid_points: + corrected_woi = np.stack((self.points.woi + self.points.ref_annotation).to_numpy()) + lat = self.points.map_annotation + valid_mask = (lat >= corrected_woi[..., 0]) & (lat <= corrected_woi[..., 1]) + log.info(f"Discarding {np.sum(~valid_mask)}/{valid_mask.size} invalid points in map {self.name} (LAT outside WOI)") + self._points_raw = self._points_raw[valid_mask] + self.points = self.points[valid_mask].reset_index(drop=True) + else: + self.points = self._points_raw = [] + + + #Mesh data + self.mesh = ll_map.mesh + self.mesh_affine = np.fromstring(ll_map.mesh_metadata["Matrix"], sep=" ").reshape([4, 4]) + assert self.mesh.n_points == int(ll_map.mesh_metadata["NumVertex"]), "Metadata and mesh mismatch" + assert self.mesh.n_cells == int(ll_map.mesh_metadata["NumTriangle"]), "Metadata and mesh mismatch" + + #Project points onto the geometry + if proj_points: + if len(self.points) > 0: + proj_points, proj_dist = project_points(self.mesh, np.stack(self.points["pos"].to_numpy()))[:2] + self.points["proj_pos"] = proj_points.tolist() + self.points["proj_dist"] = proj_dist + + elif type(self.points) == pd.DataFrame: #Add empty columns just to be consistent + self.points["proj_pos"] = [] + self.points["proj_dist"] = [] + + @property + def nr_points(self): + return len(self.points) + + def __init__(self, ll_map : CartoLLMap, *simplify_args, **simplify_kwargs) -> None: + #self.ll_map = ll_map + self._simplify(ll_map, *simplify_args, **simplify_kwargs) + #del self.ll_map + +class CartoStudy(): + """High level class to easily read Carto3 archives, directories or buffered studies. + + Parameters + ---------- + arg1 : str + A path to either + + * A directory containing the study + * A zip file with the study inside + * A path to a previously saved study + * A :class:`cartoreader_lite.low_level.study.CartoLLStudy` instance + arg2 : str, optional + The name of the study to load, contained inside the directory or zip file. + Has to be None when loading a pickled study. + Will default to either the zip name, bottom most directory name or None, depending on your choice of arg1. + ablation_sites_kwargs : Dict + Optional keyword arguments to be passed to :class:`AblationSites` + carto_map_kwargs : Dict + Optional keyword arguments to be passed to :class:`CartoMap` + """ + + name : str #: The name of the study + ablation_data : AblationSites #: Detailed information about the ablation sites and their readings over time + maps : List[CartoMap] #: All recorded maps associated with this study + aux_meshes : List[CartoAuxMesh] #: Auxiliary meshes generated by the CARTO system, not associated with any specific map, e.g. CT segmentations from `CARTOSeg`_. + aux_mesh_reg_mat : np.ndarray #: 4x4 affine registration matrix to map the auxiliary meshes. + + + def _simplify(self, ll_study : CartoLLStudy, ablation_sites_kwargs : Dict, carto_map_kwargs : Dict): + """Function to simplify the data given by the lower level ll_study. + + Parameters + ---------- + ll_study : CartoLLStudy + Data of the low level study to be simplified into this high level study + ablation_sites_kwargs : Dict + Optional keyword arguments to be passed to :class:`AblationSites` + carto_map_kwargs : Dict + Optional keyword arguments to be passed to :class:`CartoMap` + """ + self.ablation_data = AblationSites(ll_study.visitag_data, **ablation_sites_kwargs) + self.maps = [CartoMap(m, **carto_map_kwargs) for m in ll_study.maps] + self.name = ll_study.name + self.aux_meshes = ll_study.aux_meshes + self.aux_mesh_reg_mat = ll_study.aux_mesh_reg_mat + + def __init__(self, arg1, arg2 = None, ablation_sites_kwargs=None, carto_map_kwargs=None) -> None: + + if ablation_sites_kwargs is None: + ablation_sites_kwargs = {} + if carto_map_kwargs is None: + carto_map_kwargs = {} + + if issubclass(type(arg1), str) and os.path.isfile(arg1) and arg1.endswith(".pkl.gz") and arg2 is None: + loaded_study = CartoStudy.load_pickled_study(arg1) + + #https://stackoverflow.com/questions/2709800/how-to-pickle-yourself + self.__dict__.update(loaded_study.__dict__) + elif issubclass(type(arg1), CartoLLStudy) and arg2 is None: + ll_study = arg1 + self._simplify(ll_study, ablation_sites_kwargs, carto_map_kwargs) + + else: + ll_study = CartoLLStudy(arg1, arg2) + self._simplify(ll_study, ablation_sites_kwargs, carto_map_kwargs) + + @property + def nr_maps(self): + return len(self.maps) + + def save(self, file : Union[IO, PathLike] = None): + """Backup the current study into a pickled and compressed file or buffer. + + Parameters + ---------- + file : Union[IO, PathLike], optional + Target to write the study to. Can be on of the following: + + * Name of the file which the study will be written to + * A file, or buffer handle to write to + + Will default to the study name with the ending `.pkl.gz` + """ + if file is None: + file = self.name + ".pkl.gz" + + assert not issubclass(type(file), str) or file.endswith("pkl.gz"), "Only allowed file type is currently pkl.gz" + file, is_fname = convert_fname_to_handle(file, "wb") + + with gzip.GzipFile(fileobj=file, mode="wb", compresslevel=2) as g_h: + pickle.dump(self, g_h) + + if is_fname: + file.close() + + @staticmethod + def load_pickled_study(file : Union[IO, PathLike]) -> CartoStudy: + """Will load a pickled study either from + + Parameters + ---------- + file : Union[IO, PathLike] + [description] + + Returns + ------- + CartoStudy + [description] + """ + assert not issubclass(type(file), str) or file.endswith("pkl.gz"), "Only allowed file type is currently pkl.gz" + file, is_fname = convert_fname_to_handle(file, "rb") + with gzip.GzipFile(fileobj=file, mode="rb") as f: + data = pickle.load(f) + + if is_fname: + file.close() + + assert issubclass(type(data), CartoStudy), "Unexpected unpickling result" + return data diff --git a/cartoreader_lite/low_level/__init__.py b/cartoreader_lite/low_level/__init__.py new file mode 100644 index 0000000..dd81cf6 --- /dev/null +++ b/cartoreader_lite/low_level/__init__.py @@ -0,0 +1,3 @@ +"""This module implements the routines to read CARTO3 files on a low level. +Most classes here read the XML and associated data files into a pandas DataFrame. +""" \ No newline at end of file diff --git a/cartoreader_lite/low_level/read_mesh.py b/cartoreader_lite/low_level/read_mesh.py new file mode 100644 index 0000000..7f6df1c --- /dev/null +++ b/cartoreader_lite/low_level/read_mesh.py @@ -0,0 +1,147 @@ +"""This file provides routines to read .mesh files from CARTO3 +""" + +from io import StringIO +import os +from typing import Iterable, List, Tuple, Union +import pandas as pd +from pandas.errors import EmptyDataError +import numpy as np +import re +import pyvista as pv +import vtk +import logging as log + +section_re_expr_str = "\[(\S+)\]" +section_re_expr = re.compile(section_re_expr_str) + +attribute_re_expr_str = "(\S+)\s+=\s*(.*)" +attribute_re_expr = re.compile(attribute_re_expr_str) + +blank_line_expr = re.compile("\s+") + +def read_section(lines : Iterable[str]) -> pd.DataFrame: + """Reads a single section of the .mesh files + + Parameters + ---------- + lines : Iterable[str] + The lines of the section + + Returns + ------- + pd.DataFrame + The section data in a dataframe + """ + vertices_str = "\n".join(lines) + verts_io = StringIO(vertices_str) + df = pd.read_csv(verts_io, comment=";", header=None, sep="\s+") + valid_line_i = [i for i, l in enumerate(lines) if not l.strip().startswith(";")][0] #Find first non-comment line + assert valid_line_i > 0, "No header row found" + names = [name for name in re.split("\s+", lines[valid_line_i-1]) if name not in ["", ";"]] + return df, names + +def read_vertices(data : pd.DataFrame): + points = np.stack([data[n] for n in ["X", "Y", "Z"]], axis=-1) + normals = np.stack([data[f"Normal{n:s}"] for n in ["X", "Y", "Z"]], axis=-1) + group_id = np.array(data["GroupID"]) + + return points, normals, group_id + +def read_tris(data : pd.DataFrame) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Converts the dataframe of the TrianglesSection into multiple numpy array + + Parameters + ---------- + data : pd.DataFrame + The data containing the TrianglesSection + + Returns + ------- + Tuple[np.ndarray, np.ndarray, np.ndarray] + Returns the triplet (faces [Mx3], normals [Mx3], group-ID [M]) + """ + tris = np.stack([data[f"Vertex{i:d}"] for i in range(3)], axis=-1) + normals = np.stack([data[f"Normal{n:s}"] for n in ["X", "Y", "Z"]], axis=-1) + group_id = np.array(data["GroupID"]) + + return tris, normals, group_id + + +def read_mesh_file(fname : str) -> Tuple[Union[pv.UnstructuredGrid, pv.PolyData], dict]: + """Reads a single mesh file in CARTO3 mesh format and returns it as a pyvista object + + Parameters + ---------- + fname : str + Filename of the mesh file + + Returns + ------- + Tuple[Union[pv.UnstructuredGrid, pv.PolyData], dict] + Returns the constructred mesh from the data, along with the header data as a dictionary + """ + with open(fname, "r", errors="replace") as f: + lines = f.readlines() + + section_inds = [(i, section_re_expr.match(lines[i])) for i in np.arange(len(lines)) if section_re_expr.match(lines[i]) is not None] + + assert len(section_inds) > 1, "Not enough section headers found in the file" + assert section_inds[0][1].group(1) == "GeneralAttributes", "Expected attributes first" + + section_inds.append((len(lines), None)) #To ease iteration + + header = {} + for line in lines[section_inds[0][0]+1:section_inds[1][0]]: + if not line.startswith(";") and blank_line_expr.match(line) is None: + match = attribute_re_expr.match(line) + assert match is not None, f"Could not parse header line '{line:s}'" + header[match.group(1)] = match.group(2) + + points = vert_normals = vert_groups = None + tris = tri_normals = tri_groups = None + for i in range(1, len(section_inds) - 1): + sec = section_inds[i] + next_sec = section_inds[i+1] + section_name = sec[1].group(1) + + section_lines = lines[sec[0]:next_sec[0]] + + try: + df, header_names = read_section(section_lines[1:]) + assert np.all(df[0] == np.arange(len(df))), f"Non continuous index sequence in section {section_name:s}" + assert np.all(df[1] == "="), f"Equality sign expected in section {section_name:s}" + + df = df[range(2, len(df.columns))] #Actual dataframe + if len(df.columns) > len(header_names): + log.warn(f"Error while reading mesh {fname}, section {section_name}: Mismatch between number of given column headers and data columns." + f"Expected {len(df.columns)}, but got {len(header_names)}. Later headers will be 'N/A'") + header_names += ["N/A"] * (len(df.columns) - len(header_names)) + if section_name != "VerticesAttributesSection": + df.columns = header_names + + if section_name == "VerticesSection": + points, vert_normals, vert_groups = read_vertices(df) + elif section_name == "TrianglesSection": + tris, tri_normals, tri_groups = read_tris(df) + elif section_name == "VerticesAttributesSection": + log.info("Skipping VerticesAttributesSection (not yet implemented)") + except EmptyDataError as err: + print(f"Skipping section {section_name}. Section is empty.") + print("Original error: ", err) + + + assert points is not None, "No vertices found in the file" + + if tris is None: + mesh = pv.PolyData(points) + + else: + mesh = pv.UnstructuredGrid({vtk.VTK_TRIANGLE: tris}, points) + mesh.cell_data["normals"] = tri_normals + mesh.cell_data["group_id"] = tri_groups + + mesh.point_data["normals"] = vert_normals + mesh.point_data["group_id"] = vert_groups + + return mesh, header diff --git a/cartoreader_lite/low_level/study.py b/cartoreader_lite/low_level/study.py new file mode 100644 index 0000000..519570e --- /dev/null +++ b/cartoreader_lite/low_level/study.py @@ -0,0 +1,254 @@ +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, wait +import xml.etree.ElementTree as ET +from xml.etree.ElementTree import Element +import os +import pandas as pd + +from cartoreader_lite.low_level.read_mesh import read_mesh_file +from cartoreader_lite.low_level.visitags import read_visitag_dir +from .utils import camel_to_snake_case, read_point_data, xml_elem_to_dict, xml_to_dataframe +import numpy as np +from itertools import repeat +import tempfile +import zipfile +import pyvista as pv +from typing import Dict + +_parallelize_pool = ProcessPoolExecutor + +class CartoLLMap: + + """Low level CARTO3 map container for all point associated data. + + Parameters + ---------- + xml_h : Element + XML element holding the map data + path_prefix : str + Prefix of the path to load from + """ + + def import_raw_points(self, path_prefix : str): + """Imports all points and its detailed data of the current map + + Parameters + ---------- + path_prefix : str + Prefix of the path to load from + """ + + #read_f = lambda point_id: read_point_data(self.name, point_id, path_prefix) + with _parallelize_pool() as pool: + self.point_raw_data = list(pool.map(read_point_data, repeat(self.name), + self.points_main_data["Id"], repeat(path_prefix), + chunksize=5)) + pass + + def __init__(self, xml_h : Element, path_prefix : str) -> None: + + for k, v in xml_h.items(): + setattr(self, camel_to_snake_case(k), v) + + for elem in xml_h: + if elem.tag == "CartoPoints": + self.points_main_data = xml_to_dataframe(elem) + if "Position3D" in self.points_main_data: + self.points_main_data["Position3D"] = self.points_main_data["Position3D"].apply(lambda x: np.fromstring(x, sep=" ")) + if "CathOrientation" in self.points_main_data: + self.points_main_data["CathOrientation"] = self.points_main_data["CathOrientation"].apply(lambda x: np.fromstring(x, sep=" ")) + + self.virtual_points = [[xml_elem_to_dict(sub_elem) for sub_elem in point if sub_elem.tag == "VirtualPoint"] for point in elem] + self.point_tags = [[{**sub_elem.attrib, **{"value": int(sub_elem.text)}} for sub_elem in point if sub_elem.tag == "Tags"] for point in elem] + elif elem.tag == "TagsTable": + self.tags = xml_to_dataframe(elem) + elif elem.tag == "RefAnnotationConfig": + self.ref_annotation_config = xml_elem_to_dict(elem) + elif elem.tag == "ColoringRangeTable": + self.coloring_range_table = xml_to_dataframe(elem) + + #Import mesh + if "FileNames" in xml_h.keys(): + self.mesh, self.mesh_metadata = read_mesh_file(os.path.join(path_prefix, self.file_names)) + + if "Id" in self.points_main_data: + self.import_raw_points(path_prefix) + +class CartoAuxMesh: + """Class that holds auxiliary meshes of the CARTO system, e.g. generated by `CartoSeg`_. + + Parameters + ---------- + xml_h : Element + XML entry point from where to read the auxiliary meshes + path_prefix : str + Path prefix pointing to the directory to read from + load : bool, optional + If true, the meshes will be read and buffered immediately. + If false, only the names will be loaded and :meth:`load_mesh` will need to be called later. + By default True + """ + + mesh_path : str #: Full path to the mesh file + name : str #: Name of the mesh + mesh_data : pv.UnstructuredGrid #: The loaded mesh + metadata : Dict[str, str] #: Metadata associated with the mesh, given by the XML tags + affine : np.ndarray #: 4x4 affine transformation matrix given by CARTO + + + def __init__(self, xml_h : Element, path_prefix : str, load=True) -> None: + for k, v in xml_h.items(): + setattr(self, camel_to_snake_case(k), v) + + self.mesh_path = os.path.join(path_prefix, xml_h.attrib["FileName"]) + self.name = os.path.splitext(self.file_name)[0] + + if load: + self.load_mesh() + + def load_mesh(self): + """Loads the mesh with the given name into the memory + """ + self.mesh_data, self.metadata = read_mesh_file(self.mesh_path) + if "Matrix" in self.metadata: + self.affine = np.fromstring(self.metadata["Matrix"], sep=" ").reshape([4, 4]) + +class CartoLLStudy: + """Low level CARTO study class that reads all information found in the CARTO3 study and saves it. + + Parameters + ---------- + arg1 : str + A path to either + + * A directory containing the study + * A zip file with the study inside + arg2 : str, optional + The name of the study to load, contained inside the directory or zip file. + Will default to either the zip name or bottom most directory name. + """ + + aux_mesh_reg_mat : np.ndarray = None + + def _parse_meshes(self, xml_h : Element, path_prefix : str): + """Parses and loads the axuiliary meshes given in the study + + Parameters + ---------- + xml_h : Element + The XML element containing the mesh metadata + path_prefix : str + Path prefix pointing to the directory to read from + """ + self.aux_meshes = [] + + with ProcessPoolExecutor() as pool: + for elem in xml_h: + if elem.tag == "RegistrationMatrix": + self.aux_mesh_reg_mat = np.fromstring(elem.text, sep=" ").reshape([4, 4]) #Affine matrix + elif elem.tag == "Mesh": + self.aux_meshes.append(pool.submit(CartoAuxMesh, elem, path_prefix)) #CartoMesh(elem, path_prefix)) + elif elem.tag == "RegistrationData": + self.aux_mesh_reg_data = xml_elem_to_dict(elem) + + self.aux_meshes = [m.result() for m in self.aux_meshes] + + + def _parse_maps(self, maps : Element, path_prefix : str): + """Parses and loads the maps given in the study + + Parameters + ---------- + maps : Element + The XML element containing the map data + path_prefix : str + Path prefix pointing to the directory to read from + """ + self.maps = [] + with _parallelize_pool() as pool: + for elem in maps: + if elem.tag == "Map": + self.maps.append(pool.submit(CartoLLMap, elem, path_prefix)) #self.maps.append(CartoLLMap(elem, path_prefix)) + elif elem.tag == "TagsTable": + self.tags_table = xml_to_dataframe(elem) + elif elem.tag == "ColoringTable": + self.coloring_table = xml_to_dataframe(elem) + + self.maps = [res.result() for res in self.maps] + + + def _read_xml(self, xml_h : ET, path_prefix : str): + """Read the XML data of the study and parses all the data in it + + Parameters + ---------- + xml_h : ET + The XML root element of the study + path_prefix : str + Path prefix pointing to the directory to read from + """ + root = xml_h.getroot() + self.name = root.attrib["name"] + futures = [] + with ThreadPoolExecutor() as pool: + for elem in root: + if elem.tag == "Maps": + futures.append(pool.submit(self._parse_maps, elem, path_prefix)) #self._parse_maps(elem, path_prefix) + elif elem.tag == "Meshes": + futures.append(pool.submit(self._parse_meshes, elem, path_prefix)) + + [res.result() for res in futures] + + def _from_zip(self, zip_fname : str, study_name : str = None): + """Loads the study from a zipped file by extracting it first and then calling :meth:`._from_dir` + + Parameters + ---------- + zip_fname : str + The name of the zip file + study_name : str, optional + The name of the XML study inside the zip file. Can also contain sub-folder paths. + Will default to the name of the zip file. + """ + with tempfile.TemporaryDirectory() as tmp_dir_name: + with zipfile.ZipFile(zip_fname, "r") as zip_f: + zip_f.extractall(tmp_dir_name) + + self._from_dir(tmp_dir_name, study_name) + + def _from_dir(self, dir_name : str, study_name : str = None): + """Loads the study from a directory file + + Parameters + ---------- + dir_name : str + Name of the directory to read from + study_name : str, optional + The name of the XML study inside the zip file. + Will default to the name of the bottom-most directory. + """ + if study_name is None: + study_name = os.path.basename(os.path.normpath(dir_name)) + + if not study_name.endswith(".xml"): + study_name += ".xml" + + #Move any folder directives from the study name to the dir name + dir_name = os.path.join(dir_name, os.path.split(study_name)[0]) + study_name = os.path.split(study_name)[1] + + self.aux_meshes = [] + full_fname = os.path.join(dir_name, study_name) + # Pass the path of the xml document + study_xml = ET.parse(full_fname) + self._read_xml(study_xml, dir_name) + #study_root = study_xml.getroot() + self.visitag_data = read_visitag_dir(os.path.join(dir_name, "VisiTagExport")) + + def __init__(self, arg1 : str, arg2 : str = None) -> None: + assert issubclass(type(arg1), str), "Given arguments not (yet) supported" + if os.path.isdir(arg1): + self._from_dir(arg1, arg2) + elif os.path.isfile(arg1) and arg1.endswith(".zip"): #Possible second argument: study name + self._from_zip(arg1, arg2) + else: + assert False, "Given arguments not (yet) supported" diff --git a/cartoreader_lite/low_level/utils.py b/cartoreader_lite/low_level/utils.py new file mode 100644 index 0000000..1de462c --- /dev/null +++ b/cartoreader_lite/low_level/utils.py @@ -0,0 +1,235 @@ +"""Utility functions to more easily read and write the CARTO3 files on a low level. +""" + +from typing import Iterable, List, Dict, Tuple, IO, Union +import pandas as pd +import xml.etree.ElementTree as ET +from xml.etree.ElementTree import Element +import os +from collections import defaultdict +import re +from os import PathLike +import numpy as np +from scipy.interpolate import interp1d +from scipy.spatial import cKDTree + +multi_whitespace_re = re.compile("\s\s+") + +def read_connectors(xml_elem : Element, path_prefix : str) -> Dict[str, List[pd.DataFrame]]: + """Reads connector data from the main XML element, pointing to multiple files with the attached connector data. + + Parameters + ---------- + xml_elem : Element + The XML Connector element where the data will be found + path_prefix : str + The path prefix where to search for the connector files + + Returns + ------- + Dict[str, List[pd.DataFrame]] + A dictionary mapping from the connector names to the associated pandas DataFrames that will contain the connector data. + """ + connectors = defaultdict(lambda: []) + for connector in xml_elem: + assert connector.tag == f"Connector", "Non connector XML element found ({connector.tag})" + assert len(connector.attrib.keys()) == 1, f"More than one attribute found for connector: {connector.attrib:s}" + k, fname = list(connector.items())[0] + full_fname = os.path.join(path_prefix, fname) + with open(full_fname, "r") as f: + metadata = (os.path.splitext(fname)[0], f.readline().strip()) + connectors[k].append((metadata, pd.read_csv(f, sep="\s+"))) #skiprows=1))) #Not necessary to skiprows if we don't seek to the beginning + + return dict(connectors) + +def read_contact_force(fname : str) -> pd.DataFrame: + with open(fname, "r") as f: + metadata = [f.readline().strip() for i in range(7)] + data = pd.read_csv(f, sep="\s+") + + return metadata, data + +def read_point_data(map_name : str, point_id : int, path_prefix : str = None) -> Tuple[Dict, Dict]: + """Reads all the available point data for given map and point ID, along with its metadata. + + Parameters + ---------- + map_name : str + Name of the map + point_id : int + Point ID to read + path_prefix : str, optional + Path prefix used while looking for files. + Will default to the current directory + + Returns + ------- + Tuple[Dict, Dict] + A tuple containing both a dictionary of metadata and the actual data + """ + + #e.g. 1-1-ReLA_P1380_Point_Export.xml + #print(f"Reading point {point_id}") + xml_fname = f"{map_name:s}_P{point_id:d}_Point_Export.xml" + if path_prefix is not None: + xml_fname = os.path.join(path_prefix, xml_fname) + + point_xml = ET.parse(xml_fname) + xml_root = point_xml.getroot() + + metadata = {} + data = {} + for elem in xml_root: + if elem.tag == "Positions": + data["connector_data"] = read_connectors(elem, path_prefix) + elif elem.tag == "ECG": + #ecg_metadata = + ecg_fname = os.path.join(path_prefix, elem.attrib["FileName"]) + with open(ecg_fname, "r") as ecg_f: + ecg_metadata = [ecg_f.readline().strip() for i in range(3)] + ecg_header = ecg_f.readline() + ecg_header = [elem for elem in re.split(multi_whitespace_re, ecg_header) if len(elem) > 0] #Two or more whitespaces as delimiters + #ecg_f.seek(0) + ecg_data = pd.read_csv(ecg_f, #skiprows=4, #Not necessary if we don't seek to the beginning + header=None, sep="\s+", names=ecg_header, dtype=np.int16) + data["ecg"] = (ecg_metadata, ecg_data) + elif elem.tag == "ContactForce": + data["contact_force_data"] = read_contact_force(os.path.join(path_prefix, elem.attrib["FileName"])) + else: + metadata[elem.tag] = xml_elem_to_dict(elem) + + return metadata, data + +def convert_df_dtypes(df : pd.DataFrame, inplace=True) -> pd.DataFrame: + if not inplace: + df = df.copy() + + #Convert to numerical values wherever possible + for k in df: + df[k] = pd.to_numeric(df[k], errors="ignore") + + return df + +def xml_elem_to_dict(xml_elem : Element) -> dict: + return xml_elem.attrib + + +def xml_to_dataframe(xml_elem : Element, attribs=None) -> pd.DataFrame: + dataframe_dict = defaultdict(lambda: []) + + for single_elem in xml_elem: + for k, v in single_elem.items(): #XML-Attributes + dataframe_dict[k].append(v) + + df = pd.DataFrame.from_dict(dataframe_dict) + return convert_df_dtypes(df) + +_camel_pattern = re.compile(r"(? str: + return _camel_pattern.sub('_', name).lower() + +#https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +def snake_to_camel_case(name : str, capitalize=False) -> str: + ret = ''.join(word.title() for word in name.split('_')) + if not capitalize: + ret = ret[0].lower() + ret[1:] + + return ret + +def convert_fname_to_handle(file : Union[IO, PathLike], mode : str): + is_fname = issubclass(type(file), str) + if is_fname: + #assert file.endswith("pkl.gz"), "Only allowed file type is currently pkl.gz" + file = open(file, mode) + + return file, is_fname + + +def simplify_dataframe_dtypes(df : pd.DataFrame, dtype_dict : dict, inplace=True, double_to_float = False) -> pd.DataFrame: + if not inplace: + df = df.copy() + + for k in df: + if k in dtype_dict: + #TODO: Check for min/max values + df[k] = df[k].astype(dtype_dict[k]) + + elif double_to_float and df[k].dtype == np.float64: #np.issubdtype(df[k].dtype, np.floating) + df[k] = df[k].astype(np.float32) + + return df + +def interp1d_dtype(x : np.ndarray, y : np.ndarray, *args, **kwargs): + + #For complex objects (e.g. strings), just take the closest object + if y.dtype == np.object0: + kdtree = cKDTree(x[:, np.newaxis]) + interp_f = lambda x_query: y[kdtree.query(x_query[:, np.newaxis])[1]] + else: + interp_f = interp1d(x, y, *args, **kwargs) + + if np.issubdtype(y.dtype, np.integer): #Integer values will be rounded back + _interp_f = interp_f + interp_f = lambda x_query: np.round(_interp_f(x_query)).astype(y.dtype) + + return interp_f + +def interpolate_time_data(dfs : Iterable[pd.DataFrame], time_k, time_steps, **interp_kwargs) -> pd.DataFrame: + interp_fs = [] + for df in dfs: + interp_fs.append({}) + for col_i, col_name in enumerate(df): + #series = df.iloc[:, col_i] + + interp_fs[-1][col_name] = interp1d_dtype(df[time_k].to_numpy(), df.iloc[:, col_i].to_numpy(), **interp_kwargs) + #if series.dtype == np.object0: + # kdtree = cKDTree(df[time_k].to_numpy()[:, np.newaxis]) + # interp_fs[-1][col_name] = lambda x: df.iloc[kdtree.query(x[:, np.newaxis])[1], col_i] + #else: + # interp_fs[-1][col_name] = interp1d(df[time_k], df.iloc[:, col_i], **interp_kwargs) + + #Build the unique columns + all_columns = np.concatenate([df.columns for df in dfs]) + #Set the time as the first key + unique_columns = np.concatenate([[time_k], np.setdiff1d(np.unique(all_columns), [time_k])]) + new_df_dict = {time_k: time_steps} + for col_i, col_name in enumerate(unique_columns[1:]): #First column is the timing key + for df, df_interp in zip(dfs, interp_fs): + if col_name in df_interp: + interpolated_val = df_interp[col_name](time_steps) + if col_name in new_df_dict: #Already present -> Multiple dataframes contain the data + assert np.allclose(new_df_dict[col_name], interpolated_val, rtol=1e-1), f"Dataframe values of column {col_name} are not matching" + else: + new_df_dict[col_name] = interpolated_val + + merged_data = pd.DataFrame(new_df_dict) + return merged_data + + + +def unify_time_data(dfs : Iterable[pd.DataFrame], time_k, time_interval, domain="intersection", **interp_kwargs) -> pd.DataFrame: + assert domain in ["intersection"], "Selected domain not available or not yet implemented" + assert all([time_k in df for df in dfs]), f"Selected time key {time_k} not present in all dataframes." + + if domain == "intersection": + time_extent = (max([df[time_k].min() for df in dfs]), min([df[time_k].max() for df in dfs])) + + timespan = time_extent[1] - time_extent[0] + nr_samples = int(np.ceil(timespan / time_interval)) + time_samples = np.arange(time_extent[0], time_extent[1], time_interval) + if time_samples.size != nr_samples + 1: + time_samples = np.concatenate([time_samples, [time_extent[1]]]) + assert time_samples.size == nr_samples + 1 + + return interpolate_time_data(dfs, time_k, time_samples, **interp_kwargs) + +def xyz_to_pos_vec(data : pd.DataFrame, pos_label : str = "pos") -> pd.DataFrame: + xyz_strings = ["X", "Y", "Z"] + assert all([n in data for n in xyz_strings]), f"Can not convert dataframe position from XYZ to a vector, since not all columns were found. Available columns {data.columns}" + + pos_vec = np.stack([data[n] for n in xyz_strings], axis=-1).tolist() + data = data.drop(labels=xyz_strings[1:], axis=1) + data["X"] = pos_vec + return data.rename(columns={"X": pos_label}) diff --git a/cartoreader_lite/low_level/visitags.py b/cartoreader_lite/low_level/visitags.py new file mode 100644 index 0000000..1967e3e --- /dev/null +++ b/cartoreader_lite/low_level/visitags.py @@ -0,0 +1,50 @@ +from concurrent.futures import ThreadPoolExecutor +from pandas.errors import ParserError +from typing import Dict, Iterable, List, Union, IO +import pandas as pd +from os import PathLike +import os +import re + +visitag_misc_data_re_i = re.compile("^\s+(\w+)=\s+(-?\d+)") +visitag_misc_data_re_f = re.compile("^\s+(\w+)=\s+(-?\d+\.\d+)") +visitag_misc_data_re = re.compile("^\s+(\w+)=\s+(\w+)") + +def parse_misc_visitag_data(file_h : Union[IO, PathLike]): + with open(file_h, "r") as f: + lines = f.readlines() + data = {} + for line in lines: + for re_try, dtype in [(visitag_misc_data_re_i, int), (visitag_misc_data_re_f, float), (visitag_misc_data_re, str)]: + match = re_try.match(line) + if match is not None: + data[match.group(1)] = dtype(match.group(2)) + break + + return data + +def parse_visitag_file(file_h : Union[IO, PathLike], *args, **kwargs) -> Union[pd.DataFrame, Dict[str,str]]: + try: + return pd.read_csv(file_h, *args, **kwargs) + except ParserError as err: + return parse_misc_visitag_data(file_h) + +def parse_visitag_files(file_hs : Iterable[Union[IO, PathLike]]) -> List[pd.DataFrame]: + data = [] + with ThreadPoolExecutor() as pool: + for file_h in file_hs: + data.append(pool.submit(parse_visitag_file, file_h, sep="\s+")) + + return [d.result() for d in data] + +def read_visitag_dir(dir_path : str) -> Dict[str, pd.DataFrame]: + #visitag_data = {} + visitag_fnames = [] + for root, dirs, files in os.walk(dir_path): + for file in files: + if file.endswith(".txt"): + visitag_fnames.append(os.path.join(root, file)) + + data = parse_visitag_files(visitag_fnames) + visitag_data = {os.path.splitext(os.path.basename(file))[0]: d for file, d in zip(visitag_fnames, data)} + return visitag_data \ No newline at end of file diff --git a/cartoreader_lite/postprocessing/__init__.py b/cartoreader_lite/postprocessing/__init__.py new file mode 100644 index 0000000..2707249 --- /dev/null +++ b/cartoreader_lite/postprocessing/__init__.py @@ -0,0 +1,2 @@ +"""Module containing the few postprocessing steps implemented in the library +""" \ No newline at end of file diff --git a/cartoreader_lite/postprocessing/geometry.py b/cartoreader_lite/postprocessing/geometry.py new file mode 100644 index 0000000..7f3ea49 --- /dev/null +++ b/cartoreader_lite/postprocessing/geometry.py @@ -0,0 +1,52 @@ +import pyvista as pv +import numpy as np +import vtk +import trimesh +from trimesh.proximity import ProximityQuery +from typing import Tuple, Union + +def create_tri_mesh(mesh : pv.UnstructuredGrid) -> trimesh.Trimesh: + """Creates a Trimesh from an unstructured grid + + Parameters + ---------- + mesh : pv.UnstructuredGrid + mesh to convert. Will be automatically triangulated + + Returns + ------- + trimesh.Trimesh + The converted trimesh + """ + mesh = mesh.triangulate() + assert vtk.VTK_TRIANGLE in mesh.cells_dict and len(mesh.cells_dict) == 1, "Triangulation of the mesh failed" + + verts = mesh.points + faces = mesh.cells_dict[vtk.VTK_TRIANGLE] + return trimesh.Trimesh(verts, faces) + +def project_points(mesh : Union[trimesh.Trimesh, pv.UnstructuredGrid], points : np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Projects a set of points onto a triangulated surface mesh + + Parameters + ---------- + mesh : Union[trimesh.Trimesh, pv.UnstructuredGrid] + A mesh to project on. Will be converted to a trimesh, if it is not one already (see :func:`create_tri_mesh`) + points : np.ndarray + Points to project [Nx3] + + Returns + ------- + Tuple[np.ndarray, np.ndarray, np.ndarray] + The returned triplet will consist of: + + * The projected points [Nx3] + * The projection distance [N] + * The triangle index on which the projection ended up [N] + """ + if type(mesh) == pv.UnstructuredGrid: + mesh = create_tri_mesh(mesh) + + prox = ProximityQuery(mesh) + points_proj, proj_dist, tri_i = prox.on_surface(points) + return points_proj, proj_dist, tri_i diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..083bbdc --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,69 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) +import cartoreader_lite + +# -- Project information ----------------------------------------------------- + +project = 'Cartoreader Lite' +copyright = '2022, Thomas Grandits' +author = 'Thomas Grandits' + +# The full version, including alpha/beta/rc tags +release = cartoreader_lite.__version__ + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.coverage', 'sphinx.ext.napoleon' +] + +#https://stackoverflow.com/questions/56693832/should-sphinx-be-able-to-document-instance-attributes-in-a-class +autodoc_default_options = { + 'members': True, + 'member-order': 'bysource', + #'special-members': '__init__', +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'links.rst'] + +#https://stackoverflow.com/questions/61688403/can-i-keep-all-my-external-links-in-a-separate-file-with-sphinx +#This part makes external links available to all documentation files +rst_epilog ="" +with open("links.rst", "r") as f: + rst_epilog += f.read() + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +#html_theme = 'alabaster' +html_theme = "pydata_sphinx_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/design.rst b/docs/design.rst new file mode 100644 index 0000000..427342d --- /dev/null +++ b/docs/design.rst @@ -0,0 +1,59 @@ +.. _file_format: + +CARTO3 File Format +================== + +This document contains information and a few pointers on the terminology used both inside the CARTO3 file format and also in this library. +There's also a replicating python class associated to each of these parts that will hold the information after reading (listed at the bottom of the section). + +.. warning:: + + The information provided in this document is not an official documentation of the proprietary data format. + As such, the information is merely meant as a guideline to better understand the available data for scientific purposes. + It is in no way a complete and error free documentation that can be used in production, yet alone medical devices. + **No** warranty is provided that the given information is complete or error-free. + +Study +---------- + +The highest abstraction level of the CARTO3 system is a study and the entry point of `cartoreader-lite itself`. +It can contain an arbitrary combination of multiple: + + - `Maps <#map>`_ + - `Visitags <#visitag>`_ + - `Auxiliary Meshes <#auxiliary-mesh>`_ + +Associated class: :class:`.CartoStudy`. + +Map +---------- + +During a study, multiple maps a generated according to the need of the attending staff (e.g. pre- and post ablation). +Each map contains: + + - A mesh with which to associate the recordings + - Multiple `points <#point>`_ with electrical recordings. These are not registered to the mesh + +Associated class: :class:`.CartoMap`. + +Point +----------- + +Each point represents an electrical recording at a single location. +A point contains not just the position, but also :term:`ECG` and :term:`EGM` readings over time. + +Associated class: :class:`.CartoPointDetailData`. + +Visitag +---------- + +Visitag sites store information about the ablation sites. + +Associated class: :class:`.AblationSites` + +Auxiliary Mesh +------------------ + +Additional meshes exported by CARTO3, e.g. by `CartoSeg`_. + +Associated class :class:`.CartoAuxMesh` diff --git a/docs/example.inc b/docs/example.inc new file mode 100644 index 0000000..bff3e3d --- /dev/null +++ b/docs/example.inc @@ -0,0 +1,46 @@ +To test the library, you first need to get CARTO3 data. +None is provided with this repository, but you can download the testing data provided by `OpenEP`_ to quickly try out the library (make sure the libary was installed first): + +.. code-block:: bash + + python tests/generate_test_data.py + + +.. code-block:: python + + from cartoreader_lite import CartoStudy + study_dir = "openep-testingdata/Carto/Export_Study-1-11_25_2021-15-01-32" + study_name = "Study 1 11_25_2021 15-01-32.xml" + study = CartoStudy(study_dir, study_name, + carto_map_kwargs={"discard_invalid_points": False} #All points of the example are outside the WOI, which would be by default discarded + ) + ablation_points = pv.PolyData(np.stack(study.ablation_data.session_avg_data["pos"].to_numpy())) + ablation_points.point_data["RFIndex"] = study.ablation_data.session_avg_data["RFIndex"] + plotter = pv.Plotter() + plotter.add_mesh(ablation_points, cmap="jet") + plotter.add_mesh(study.maps[2].mesh) + plotter.show() + +You should see the recorded map of the `OpenEP`_ example, together with its recorded points like below. + +.. image:: figures/openep-example.png + +`cartoreader_lite` also offers the possibility to directly load the CARTO3 exported zip-files. +For the zipped `OpenEP`_ testing data, this would like the following: + +.. code-block:: python + + study_dir = "openep-testingdata.zip" + study_name = "Carto/Export_Study-1-11_25_2021-15-01-32/Study 1 11_25_2021 15-01-32.xml" + study = CartoStudy(study_dir, study_name) + +Buffering +----------- +Loading the CARTO3 study is heavy on the CPU, which is why it is often useful to backup your studies. +This can be easily achieved through the :meth:`cartoreader_lite.CartoStudy.save` method + +.. code-block:: python + + bak_name = "Study 1 11_25_2021 15-01-32.pkl.gz" + study.save(bak_name) + study_bak = CartoStudy(bak_name) diff --git a/docs/figures/openep-example.png b/docs/figures/openep-example.png new file mode 100644 index 0000000..8a8c4a0 Binary files /dev/null and b/docs/figures/openep-example.png differ diff --git a/docs/generate_doc_figs.py b/docs/generate_doc_figs.py new file mode 100644 index 0000000..375ca9f --- /dev/null +++ b/docs/generate_doc_figs.py @@ -0,0 +1,18 @@ +import pyvista as pv +import numpy as np +from cartoreader_lite import CartoStudy + +if __name__ == "__main__": + study_dir = "openep-testingdata/Carto/Export_Study-1-11_25_2021-15-01-32" + study_name = "Study 1 11_25_2021 15-01-32.xml" + study = CartoStudy(study_dir, study_name, + carto_map_kwargs={"discard_invalid_points": False} #The testing data only contains invalid points (LAT outside WOI) + ) + + ablation_points = pv.PolyData(np.stack(study.ablation_data.session_avg_data["pos"].to_numpy())) + ablation_points.point_data["RFIndex"] = study.ablation_data.session_avg_data["RFIndex"] + plotter = pv.Plotter(off_screen=True) + plotter.add_mesh(ablation_points, cmap="jet") + plotter.add_mesh(study.maps[2].mesh) + plotter.show(screenshot="docs/figures/openep-example.png") + #plotter.show() diff --git a/docs/glossary.rst b/docs/glossary.rst new file mode 100644 index 0000000..5fdfddc --- /dev/null +++ b/docs/glossary.rst @@ -0,0 +1,22 @@ +Glossary +----------- + +The following acronyms will be used throughout the documentation. + +.. glossary:: + + ECG + The **Electrocardiogram** is usually used for electrical recordings of the heart. In this documentation, the term is used for the recordings on the skin. + + EGM + An **Electrogram** is the general term used for electrical recordings over time. In this documentation, this term is mainly reserved for internal recordings. + + LAT + The **Local Activation Time** of a point is defined as the point in time at which it is activated. + + RFIndex + **Radio Frequency Index**, or more commonly ablation index, is a numerical marker associated to each ablation site, used to better quantify the applied ablation force, power and duration according to studies. + For more info, see: ``_. + + WOI + CARTO3 defines the **Window Of Interest** as the time in which recordings are considered valid (e.g. to avoid far field measurements). diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..1bd84ea --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,89 @@ +.. Cartoreader Lite documentation master file, created by + sphinx-quickstart on Wed Jan 19 14:57:28 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Cartoreader Lite's documentation +====================================== + +.. contents:: Quick Start + :depth: 3 + + +Introduction +------------- + +This repository is an inofficial reader to easily process exported `CARTO3 data `_ in Python. +It does not provide the more extensive capabilities to analyze the signals, such as `OpenEP`_, but is rather meant as a simple reader to import CARTO data. +The loaded time data is imported in `pandas `_ and the meshes in `VTK `_ provided through `PyVista `_, allowing for easy access, export and interoperatibility with existing software. + + +CARTO3 System +--------------- +The read file format is the output of the CARTO3 system, which is used as an electrical mapping device for guiding catheter ablation and catheter synchronization therapy. +For more details, please read the official website of Biosense Webster (https://www.jnjmedicaldevices.com/en-US/product/carto-3-system). + + **Carto3 System** + + *CARTO 3 System Version 7 and the CARTO PRIME® Module offers advancement in 3D mapping technology. From signals to diagnosis, across a wide range of electrophysiology procedures, we are reframing the future of electrophysiology.* + + +A short description can also be found in this documentation at :ref:`file_format`. + +Installation +------------- + +.. include:: installation.inc + +Usage +------ + +.. include:: example.inc + +Citation +------------ + +If you use the library in your scientific projects, please cite the associated `Zenodo archive `_. + +.. code-block:: bibtex + + %TODO + @software{thomas_grandits_2021_5594452, + author = {Thomas Grandits}, + title = {A Fast Iterative Method Python package}, + month = oct, + year = 2021, + publisher = {Zenodo}, + version = {1.1}, + doi = {10.5281/zenodo.5594452}, + url = {https://doi.org/10.5281/zenodo.5594452} + } + + +Detailed Content +----------------- +.. toctree:: + :maxdepth: 2 + + interface.rst + design.rst + +Module API +-------------- + +.. autosummary:: + :toctree: _autosummary + :recursive: + :caption: Module API + + cartoreader_lite + +.. include:: + glossary.rst + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.inc b/docs/installation.inc new file mode 100644 index 0000000..9abd343 --- /dev/null +++ b/docs/installation.inc @@ -0,0 +1,7 @@ +To install `cartoreader_lite`, you have to clone the repository and install the libary using `pip`. + +.. code-block:: bash + + git clone https://github.com/thomgrand/cartoreader-lite + cd cartoreader-lite + pip install -e . diff --git a/docs/interface.rst b/docs/interface.rst new file mode 100644 index 0000000..d6773ca --- /dev/null +++ b/docs/interface.rst @@ -0,0 +1,22 @@ +Interface Methods +================= + +Here you will find some information on the data and classes that are available after loading a CARTO3 study. +:class:`cartoreader_lite.CartoStudy` is the high level entry point to read CARTO3 studies. + +.. autoclass:: cartoreader_lite.CartoStudy + +Internal High Level Classes +---------------------------- + +These classes do not need to be manually created, but should be automatically created when generating a :class:`.CartoStudy`. +Their most important members are described below. + +.. autoclass:: cartoreader_lite.CartoMap + +.. autoclass:: cartoreader_lite.CartoPointDetailData + +.. autoclass:: cartoreader_lite.AblationSites + +.. autoclass:: cartoreader_lite.CartoAuxMesh + diff --git a/docs/links.rst b/docs/links.rst new file mode 100644 index 0000000..997d188 --- /dev/null +++ b/docs/links.rst @@ -0,0 +1,7 @@ +.. + Link from the documentation to the CARTOSeg tool +.. _CartoSeg: https://www.jnjmedicaldevices.com/en-US/product/cartoseg-ct-segmentation-module + +.. _OpenEP: https://openep.io/ + +.. _PyPI: https://pypi.org/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1b68d94 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a720e12 --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +from setuptools import setup +from distutils.core import setup +import os + + +with open(os.path.join(os.path.dirname(__file__), 'README.md'), 'r') as readme: + long_description = readme.read() + +setup(name="cartoreader-lite", + version="1.0.0", + description="Cartoreader-lite provides a simplified and easy low-level access to CARTO3 studies.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/thomgrand/cartoreader-lite", + packages=["cartoreader_lite", "cartoreader_lite.high_level", "cartoreader_lite.low_level"], + install_requires=["numpy", "pyvista>=0.33", "vtk", "pandas", "scipy", + "trimesh", "rtree"], #Geometric postprocessing + classifiers=[ + "Programming Language :: Python :: 3" + ], + python_requires='>=3.8', + author="Thomas Grandits", + author_email="tomdev@gmx.net", + license="AGPL", + extras_require = { + "tests": ["pytest", "pytest-cov"], + "docs": ["sphinx", "pydata_sphinx_theme"] + } +) + diff --git a/tests/generate_test_data.py b/tests/generate_test_data.py new file mode 100644 index 0000000..025becc --- /dev/null +++ b/tests/generate_test_data.py @@ -0,0 +1,17 @@ +import os +import subprocess +import zipfile +import shutil + +if __name__ == "__main__": + subprocess.check_call(["git", "clone", "https://github.com/thomgrand/openep-testingdata"]) + + #Unzip the large file + visitag_dir = "openep-testingdata/Carto/Export_Study-1-11_25_2021-15-01-32/VisiTagExport/" + with zipfile.ZipFile(os.path.join(visitag_dir, "AllPositionInGrids.zip"), "r") as zip_f: + zip_f.extractall(visitag_dir) + + shutil.make_archive("openep-testingdata", "zip", "openep-testingdata") + + + diff --git a/tests/test_high_level_read.py b/tests/test_high_level_read.py new file mode 100644 index 0000000..cc75c98 --- /dev/null +++ b/tests/test_high_level_read.py @@ -0,0 +1,95 @@ +import pytest +from cartoreader_lite import CartoStudy +from cartoreader_lite.low_level.study import _parallelize_pool, CartoLLStudy +from concurrent.futures import ThreadPoolExecutor +from io import BytesIO +import pandas as pd +import pickle + +""" +def prepare_lib(): + global _parallelize_pool + print("Setup") + _parallelize_pool = ThreadPoolExecutor + yield None + print("Teardown") +""" + +def compare_studies(study1 : CartoStudy, study2 : CartoStudy): + assert len(study1.maps) == len(study2.maps) + assert all([m1.name == m2.name for m1, m2 in zip(study1.maps, study2.maps)]) + +class TestHighLevelCartoReader(): + + def test_openep_example(self): + #prepare_lib() + study_dir = "openep-testingdata/Carto/Export_Study-1-11_25_2021-15-01-32" + study_name = "Study 1 11_25_2021 15-01-32.xml" + study = CartoStudy(study_dir, study_name, carto_map_kwargs={"discard_invalid_points": False}) + + study.save("test_study.pkl.gz") + study_restored = CartoStudy.load_pickled_study("test_study.pkl.gz") + compare_studies(study, study_restored) + + study_restored = CartoStudy("test_study.pkl.gz") #Test direct constructor + compare_studies(study, study_restored) + + #Default arguments + study.save() + study_restored = CartoStudy.load_pickled_study(f"{study.name}.pkl.gz") + compare_studies(study, study_restored) + + #Save into a stream + with BytesIO() as bytes: + study.save(bytes) + bytes.seek(0) + study_restored = CartoStudy.load_pickled_study(bytes) + + compare_studies(study, study_restored) + + def test_openep_optional_kwargs(self): + #prepare_lib() + study_dir = "openep-testingdata/Carto/Export_Study-1-11_25_2021-15-01-32" + study_name = "Study 1 11_25_2021 15-01-32" #Try without the file ending + ll_study = CartoLLStudy(study_dir, study_name) + ll_study_bak = pickle.dumps(ll_study) + study = CartoStudy(ll_study) + + points = study.maps[2].points + assert type(points) == pd.DataFrame and len(points) == 0 #All points were discarded + + #Don't unify time + ll_study = pickle.loads(ll_study_bak) + study = CartoStudy(ll_study, ablation_sites_kwargs={"resample_unified_time": False}) + assert hasattr(study.ablation_data, "session_rf_data") and type(study.ablation_data.session_rf_data) == list and len(study.ablation_data.session_rf_data) > 0 and type(study.ablation_data.session_rf_data[0][1]) == pd.DataFrame + assert hasattr(study.ablation_data, "session_force_data") and type(study.ablation_data.session_force_data) == list and len(study.ablation_data.session_force_data) > 0 and type(study.ablation_data.session_force_data[0][1]) == pd.DataFrame + assert "pos" in study.ablation_data.session_time_data[0][1] + + #Leave the position vector as X, Y, Z + ll_study = pickle.loads(ll_study_bak) + study = CartoStudy(ll_study, ablation_sites_kwargs={"resample_unified_time": False, "position_to_vec": False}) + assert hasattr(study.ablation_data, "session_time_data") and type(study.ablation_data.session_time_data) == list and len(study.ablation_data.session_time_data) > 0 and type(study.ablation_data.session_time_data[0][1]) == pd.DataFrame + assert all([n in study.ablation_data.session_time_data[0][1] for n in ["X", "Y", "Z"]]) + + #Leave the EGM header numbers in the frame + ll_study = pickle.loads(ll_study_bak) + study = CartoStudy(ll_study, carto_map_kwargs={"remove_egm_header_numbers": False, "discard_invalid_points": False}) + egms = study.maps[2].points.detail[0].egm + ecgs = study.maps[2].points.detail[0].surface_ecg + assert all(["(" in col for col in egms.columns]) + assert all(["(" in col for col in ecgs.columns]) + + ll_study = pickle.loads(ll_study_bak) + study = CartoStudy(ll_study, carto_map_kwargs={"remove_egm_header_numbers": True, "discard_invalid_points": False}) + egms = study.maps[2].points.detail[0].egm + ecgs = study.maps[2].points.detail[0].surface_ecg + assert all(["(" not in col for col in egms.columns]) + assert all(["(" not in col for col in ecgs.columns]) + assert "proj_pos" in points #Check for the projection of the points + assert "proj_dist" in points + + ll_study = pickle.loads(ll_study_bak) + study = CartoStudy(ll_study, carto_map_kwargs={"discard_invalid_points": False, "proj_points": False}) + points = study.maps[2].points + assert "proj_pos" not in points #Check that the projection was not performed + assert "proj_dist" not in points diff --git a/tests/test_low_level_read.py b/tests/test_low_level_read.py new file mode 100644 index 0000000..6781c15 --- /dev/null +++ b/tests/test_low_level_read.py @@ -0,0 +1,46 @@ +from cartoreader_lite.low_level.study import CartoLLStudy, _parallelize_pool +from concurrent.futures import ThreadPoolExecutor +import pyvista as pv +import numpy as np +import pytest + +""" +@pytest.fixture() +def prepare_lib(): + global _parallelize_pool + _parallelize_pool = ThreadPoolExecutor + yield None +""" + +def low_level_sanity_check(study): + assert len(study.maps) == 4 + assert study.maps[0].name == "1-Map" + + map2 = study.maps[2] + assert len(map2.point_raw_data) == 2 + point_data = map2.points_main_data + assert "Position3D" in point_data + assert "CathOrientation" in point_data + assert hasattr(map2, "mesh") and type(map2.mesh) == pv.UnstructuredGrid and map2.mesh.n_cells >= 5e3 and map2.mesh.n_points >= 2e3 + assert hasattr(map2, "mesh_metadata") and "MeshName" in map2.mesh_metadata + +class TestLowLevelCartoReader(): + + def test_from_dir(self): + study_dir = "openep-testingdata/Carto/Export_Study-1-11_25_2021-15-01-32" + study_name = "Study 1 11_25_2021 15-01-32.xml" + study = CartoLLStudy(study_dir, study_name) + low_level_sanity_check(study) + + def test_from_zip(self): + study_dir = "openep-testingdata.zip" + study_name = "Carto/Export_Study-1-11_25_2021-15-01-32/Study 1 11_25_2021 15-01-32.xml" + study = CartoLLStudy(study_dir, study_name) + low_level_sanity_check(study) + + def test_invalid_args(self): + with pytest.raises(Exception): + study = CartoLLStudy([5, 4, 3]) + + with pytest.raises(Exception): + study = CartoLLStudy("N/A File") diff --git a/tests/test_postprocessing.py b/tests/test_postprocessing.py new file mode 100644 index 0000000..53a5c96 --- /dev/null +++ b/tests/test_postprocessing.py @@ -0,0 +1,32 @@ +import numpy as np +import pyvista as pv +import vtk +from cartoreader_lite.postprocessing.geometry import project_points, create_tri_mesh +import trimesh + +class TestPostprocessing(): + + def test_create_tri_mesh(self): + mesh = pv.UnstructuredGrid({vtk.VTK_TRIANGLE: np.array([[0, 1, 2]])}, + np.array([[0, 0, 0], [1.0, 0., 0.], [0, 1, 0]])) + tri_mesh = create_tri_mesh(mesh) + assert type(tri_mesh) == trimesh.Trimesh + assert len(tri_mesh.vertices) == 3 + assert len(tri_mesh.faces) == 1 + + def test_projection(self): + mesh = pv.UnstructuredGrid({vtk.VTK_TRIANGLE: np.array([[0, 1, 2]])}, + np.array([[0, 0, 0], [1.0, 0., 0.], [0, 1, 0]])) + + proj_pos, proj_dist, tri_i = project_points(mesh, np.array([[0.5, 0.5, 1]])) + assert np.allclose(proj_pos, np.array([[0.5, 0.5, 0]])) + assert np.isclose(proj_dist, 1) + assert np.all(tri_i == 0) + + #Test with the already converted mesh + proj_pos, proj_dist, tri_i = project_points(create_tri_mesh(mesh), np.array([[0.5, 0.5, 1]])) + assert np.allclose(proj_pos, np.array([[0.5, 0.5, 0]])) + assert np.isclose(proj_dist, 1) + assert np.all(tri_i == 0) + + diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..406e2d4 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,38 @@ +from cartoreader_lite.low_level.utils import snake_to_camel_case, simplify_dataframe_dtypes, convert_df_dtypes +import pandas as pd +import numpy as np + +def test_snake_to_camel_case(): + test_strings = ["camel_case", "imp_ort_ant_var"] + expected_strings = ["camelCase", "impOrtAntVar"] + assert all([snake_to_camel_case(t) == e for t, e in zip(test_strings, expected_strings)]) + assert all([snake_to_camel_case(t, capitalize=True) == (e[0].upper() + e[1:]) for t, e in zip(test_strings, expected_strings)]) + +def test_simplify_dataframe_dtypes(): + df = pd.DataFrame({"a": np.arange(3, dtype=np.int64), "b": np.array([0, 0.1, 2.], dtype=np.float64), "c": ["a", "b", "c"]}) + + df_conv = simplify_dataframe_dtypes(df, {"a": np.float16}, inplace=False, double_to_float=True) + assert df_conv.a.dtype == np.float16 + assert df_conv.b.dtype == np.float32 + assert df.a.dtype == np.int64 + assert df.b.dtype == np.float64 + + #df_orig = df.copy() + simplify_dataframe_dtypes(df, {"a": np.float16}, inplace=True, double_to_float=True) + assert df.a.dtype == np.float16 + assert df.b.dtype == np.float32 + +def test_convert_df_dtypes(): + df = pd.DataFrame({"a": np.arange(3, dtype=np.int64), "b": np.array([0, 0.1, 2.], dtype=np.float64), "c": ["a", "b", "c"]}) + df.a = df.a.astype(str) + df.b = df.b.astype(str) + + df_conv = convert_df_dtypes(df, inplace=False) + assert np.issubdtype(df_conv.a.dtype, np.integer) + assert np.issubdtype(df_conv.b.dtype, np.floating) + assert np.issubdtype(df.a.dtype, np.object0) + assert np.issubdtype(df.b.dtype, np.object0) + + convert_df_dtypes(df, inplace=True) + assert np.issubdtype(df.a.dtype, np.integer) + assert np.issubdtype(df.b.dtype, np.floating) \ No newline at end of file