diff --git a/docs/index.rst b/docs/index.rst index fb8144f..cbc6e7d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,8 +32,10 @@ Modules and Classes .. toctree:: :maxdepth: 2 - metadata utils + metadata + jobs + diff --git a/docs/jobs/index.rst b/docs/jobs/index.rst new file mode 100644 index 0000000..d6aea97 --- /dev/null +++ b/docs/jobs/index.rst @@ -0,0 +1,10 @@ + +jobs +==== + +.. toctree:: + :maxdepth: 1 + + Pipeline + + diff --git a/docs/jobs/pipeline.rst b/docs/jobs/pipeline.rst new file mode 100644 index 0000000..6290925 --- /dev/null +++ b/docs/jobs/pipeline.rst @@ -0,0 +1,7 @@ + +Pipeline +======== + +.. autoclass:: emtools.jobs.Pipeline + :members: + diff --git a/docs/metadata/starfile.rst b/docs/metadata/starfile.rst index 5c9c4c4..260212c 100644 --- a/docs/metadata/starfile.rst +++ b/docs/metadata/starfile.rst @@ -50,9 +50,11 @@ that parsing rows if not needed. These methods will also create an index of wher data blocks are in the file, so if you need to read a data table, it will jump to that position in the file. +Reading a Table +--------------- -Iterating over the rows ------------------------ +Iterating over the Table rows +----------------------------- In some cases, we just want to iterate over the rows and operate on them one by one. In that case, it is not necessary to fully load the whole table in memory. Iteration diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 6e28590..1dc4e00 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -5,5 +5,7 @@ utils .. toctree:: :maxdepth: 1 - Color, Timer - Process + Color, Pretty, Timer + Process, Path, System + + diff --git a/docs/utils/misc.rst b/docs/utils/misc.rst index ff75ee3..c5933bc 100644 --- a/docs/utils/misc.rst +++ b/docs/utils/misc.rst @@ -1,10 +1,18 @@ -Color, Pretty -============= +Color +===== .. autoclass:: emtools.utils.Color :members: +Pretty +====== + .. autoclass:: emtools.utils.Pretty :members: +Timer +===== + +.. autoclass:: emtools.utils.Timer + :members: diff --git a/docs/utils/process.rst b/docs/utils/process.rst index 40a4e7e..60f2e14 100644 --- a/docs/utils/process.rst +++ b/docs/utils/process.rst @@ -2,13 +2,18 @@ Process ======= +.. autoclass:: emtools.utils.Process + :members: +Path +==== -Reference ---------- - -.. autoclass:: emtools.utils.Process +.. autoclass:: emtools.utils.Path :members: +System +====== +.. autoclass:: emtools.utils.System + :members: diff --git a/emtools/processing/__init__.py b/emtools/jobs/__init__.py similarity index 93% rename from emtools/processing/__init__.py rename to emtools/jobs/__init__.py index 8906b08..fff9a6d 100644 --- a/emtools/processing/__init__.py +++ b/emtools/jobs/__init__.py @@ -14,3 +14,6 @@ # * # ************************************************************************** +from .pipeline import Pipeline + +__all__ = ["Pipeline"] \ No newline at end of file diff --git a/emtools/processing/__main__.py b/emtools/jobs/__main__.py similarity index 100% rename from emtools/processing/__main__.py rename to emtools/jobs/__main__.py diff --git a/emtools/processing/motioncor.py b/emtools/jobs/motioncor.py similarity index 100% rename from emtools/processing/motioncor.py rename to emtools/jobs/motioncor.py diff --git a/emtools/utils/pipeline.py b/emtools/jobs/pipeline.py similarity index 100% rename from emtools/utils/pipeline.py rename to emtools/jobs/pipeline.py diff --git a/emtools/utils/__init__.py b/emtools/utils/__init__.py index 4581338..01f256c 100644 --- a/emtools/utils/__init__.py +++ b/emtools/utils/__init__.py @@ -15,15 +15,16 @@ # ************************************************************************** from .color import Color -from .process import Process from .pretty import Pretty -from .path import Path from .time import Timer -from .pipeline import Pipeline + +from .process import Process +from .path import Path from .system import System + from .server import JsonTCPServer, JsonTCPClient -__all__ = [Color, Process, Pretty, Path, Timer, Pipeline, System, - JsonTCPServer, JsonTCPClient] +__all__ = ["Color", "Pretty", "Timer", "Process", "Path", "System", + "JsonTCPServer", "JsonTCPClient"] diff --git a/emtools/utils/color.py b/emtools/utils/color.py index b06f0e3..627bc28 100644 --- a/emtools/utils/color.py +++ b/emtools/utils/color.py @@ -26,6 +26,9 @@ class Color: + """ Basic helper class to have colored string. + Useful for commands and log messages. """ + @staticmethod def green(msg): return f'{OKGREEN}{msg}{ENDC}' diff --git a/emtools/utils/path.py b/emtools/utils/path.py index 094e568..f783e26 100644 --- a/emtools/utils/path.py +++ b/emtools/utils/path.py @@ -1,15 +1,11 @@ import os import time -from glob import glob from datetime import datetime as dt -from datetime import timedelta from collections import OrderedDict -import numpy as np from .pretty import Pretty from .process import Process -from .color import Color class Path: @@ -149,7 +145,7 @@ def _mkdir(d): @staticmethod def replaceExt(filename, newExt): """ Replace the current path extension(from last .) - with a new one. The new one should not contains the .""" + with a new one. The new one should not contain the .""" return Path.removeExt(filename) + '.' + newExt @staticmethod diff --git a/emtools/utils/pretty.py b/emtools/utils/pretty.py index 6dce00e..626f4c3 100644 --- a/emtools/utils/pretty.py +++ b/emtools/utils/pretty.py @@ -20,6 +20,9 @@ class Pretty: + """ Helper class for "pretty" string formatting from several input types + (e.g. size, dates, timestamps, elapsed, etc.). + """ # Default timestamp DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' DATE_FORMAT = '%Y-%m-%d' diff --git a/emtools/utils/system.py b/emtools/utils/system.py index fc46323..8f54c5d 100644 --- a/emtools/utils/system.py +++ b/emtools/utils/system.py @@ -23,6 +23,7 @@ import socket import platform import psutil + from .process import Process @@ -33,6 +34,7 @@ class System: @staticmethod def gpus(): + """ Return a dictionary with existing GPUs and their properties. """ gpus = [] query = Process(System.NVIDIA_SMI_QUERY[0], *System.NVIDIA_SMI_QUERY[1:], doRaise=False) @@ -69,6 +71,7 @@ def cpus(): @staticmethod def memory(): + """ Return the total virtual memory of the system. (in Gb) """ return psutil.virtual_memory().total // (1024 * 1024 * 1024) # GiB @staticmethod @@ -92,4 +95,5 @@ def specs(): @staticmethod def hostname(): + """ Return the hostname. """ return socket.gethostname()