Skip to content

Commit

Permalink
Remove references to root in PathInfo, PathProvider (#477)
Browse files Browse the repository at this point in the history
* Remove reference of root from Path providers

* Update areaDetector ophyd devices to reflect changes to pathprovider

* Remaining changes to reflect pathprovider changes

* Rename resource_path attr to directory_path

* Update tests to reflect attr change
  • Loading branch information
jwlodek authored Jul 23, 2024
1 parent 52f734a commit 955414a
Show file tree
Hide file tree
Showing 11 changed files with 41 additions and 61 deletions.
6 changes: 1 addition & 5 deletions src/ophyd_async/core/_hdf_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
StreamResource,
)

from ._providers import PathInfo


@dataclass
class HDFDataset:
Expand All @@ -28,14 +26,12 @@ class HDFDataset:

class HDFFile:
"""
:param directory_info: Contains information about how to construct a StreamResource
:param full_file_name: Absolute path to the file to be written
:param datasets: Datasets to write into the file
"""

def __init__(
self,
path_info: PathInfo,
full_file_name: Path,
datasets: List[HDFDataset],
hostname: str = "localhost",
Expand All @@ -53,7 +49,7 @@ def __init__(
(
"file",
self._hostname,
str((path_info.root / full_file_name).absolute()),
str(full_file_name.absolute()),
"",
"",
None,
Expand Down
45 changes: 17 additions & 28 deletions src/ophyd_async/core/_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,13 @@ class PathInfo:
"""
Information about where and how to write a file.
The bluesky event model splits the URI for a resource into two segments to aid in
different applications mounting filesystems at different mount points.
The portion of this path which is relevant only for the writer is the 'root',
while the path from an agreed upon mutual mounting is the resource_path.
The resource_dir is used with the filename to construct the resource_path.
:param root: Path of a root directory, relevant only for the file writer
:param resource_dir: Directory into which files should be written, relative to root
:param directory_path: Directory into which files should be written
:param filename: Base filename to use generated by FilenameProvider, w/o extension
:param create_dir_depth: Optional depth of directories to create if they do not
exist
"""

root: Path
resource_dir: Path
directory_path: Path
filename: str
create_dir_depth: int = 0

Expand Down Expand Up @@ -112,20 +104,17 @@ def __init__(
self,
filename_provider: FilenameProvider,
directory_path: Path,
resource_dir: Path = Path("."),
create_dir_depth: int = 0,
) -> None:
self._filename_provider = filename_provider
self._directory_path = directory_path
self._resource_dir = resource_dir
self._create_dir_depth = create_dir_depth

def __call__(self, device_name: Optional[str] = None) -> PathInfo:
filename = self._filename_provider()

return PathInfo(
root=self._directory_path,
resource_dir=self._resource_dir,
directory_path=self._directory_path,
filename=filename,
create_dir_depth=self._create_dir_depth,
)
Expand All @@ -135,7 +124,7 @@ class AutoIncrementingPathProvider(PathProvider):
def __init__(
self,
filename_provider: FilenameProvider,
directory_path: Path,
base_directory_path: Path,
create_dir_depth: int = 0,
max_digits: int = 5,
starting_value: int = 0,
Expand All @@ -145,7 +134,7 @@ def __init__(
base_name: str = None,
) -> None:
self._filename_provider = filename_provider
self._directory_path = directory_path
self._base_directory_path = base_directory_path
self._create_dir_depth = create_dir_depth
self._base_name = base_name
self._starting_value = starting_value
Expand All @@ -161,20 +150,21 @@ def __call__(self, device_name: Optional[str] = None) -> PathInfo:

padded_counter = f"{self._current_value:0{self._max_digits}}"

resource_dir = str(padded_counter)
auto_inc_dir_name = str(padded_counter)
if self._base_name is not None:
resource_dir = f"{self._base_name}{self._inc_delimeter}{padded_counter}"
auto_inc_dir_name = (
f"{self._base_name}{self._inc_delimeter}{padded_counter}"
)
elif device_name is not None:
resource_dir = f"{device_name}{self._inc_delimeter}{padded_counter}"
auto_inc_dir_name = f"{device_name}{self._inc_delimeter}{padded_counter}"

self._inc_counter += 1
if self._inc_counter == self._num_calls_per_inc:
self._inc_counter = 0
self._current_value += self._increment

return PathInfo(
root=self._directory_path,
resource_dir=resource_dir,
directory_path=self._base_directory_path / auto_inc_dir_name,
filename=filename,
create_dir_depth=self._create_dir_depth,
)
Expand All @@ -184,35 +174,34 @@ class YMDPathProvider(PathProvider):
def __init__(
self,
filename_provider: FilenameProvider,
directory_path: Path,
base_directory_path: Path,
create_dir_depth: int = -3, # Default to -3 to create YMD dirs
device_name_as_base_dir: bool = False,
) -> None:
self._filename_provider = filename_provider
self._directory_path = Path(directory_path)
self._base_directory_path = Path(base_directory_path)
self._create_dir_depth = create_dir_depth
self._device_name_as_base_dir = device_name_as_base_dir

def __call__(self, device_name: Optional[str] = None) -> PathInfo:
sep = os.path.sep
current_date = date.today().strftime(f"%Y{sep}%m{sep}%d")
if device_name is None:
resource_dir = current_date
ymd_dir_path = current_date
elif self._device_name_as_base_dir:
resource_dir = os.path.join(
ymd_dir_path = os.path.join(
current_date,
device_name,
)
else:
resource_dir = os.path.join(
ymd_dir_path = os.path.join(
device_name,
current_date,
)

filename = self._filename_provider()
return PathInfo(
root=self._directory_path,
resource_dir=resource_dir,
directory_path=self._base_directory_path / ymd_dir_path,
filename=filename,
create_dir_depth=self._create_dir_depth,
)
Expand Down
3 changes: 1 addition & 2 deletions src/ophyd_async/epics/adcore/_hdf_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __init__(
async def open(self, multiplier: int = 1) -> Dict[str, DataKey]:
self._file = None
info = self._path_provider(device_name=self.hdf.name)
file_path = str(info.root / info.resource_dir)
file_path = info.directory_path
await asyncio.gather(
self.hdf.num_extra_dims.set(0),
self.hdf.lazy_open.set(True),
Expand Down Expand Up @@ -133,7 +133,6 @@ async def collect_stream_docs(
if not self._file:
path = Path(await self.hdf.full_file_name.get_value())
self._file = HDFFile(
self._path_provider(),
# See https://github.com/bluesky/ophyd-async/issues/122
path,
self._datasets,
Expand Down
5 changes: 1 addition & 4 deletions src/ophyd_async/fastcs/panda/_hdf_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,7 @@ async def open(self, multiplier: int = 1) -> Dict[str, DataKey]:
info = self._path_provider(device_name=self.panda_device.name)
# Set the initial values
await asyncio.gather(
self.panda_device.data.hdf_directory.set(
str(info.root / info.resource_dir)
),
self.panda_device.data.hdf_directory.set(info.directory_path),
self.panda_device.data.hdf_file_name.set(
f"{info.filename}.h5",
),
Expand Down Expand Up @@ -130,7 +128,6 @@ async def collect_stream_docs(
if indices_written:
if not self._file:
self._file = HDFFile(
self._path_provider(),
Path(await self.panda_device.data.hdf_directory.get_value())
/ Path(await self.panda_device.data.hdf_file_name.get_value()),
self._datasets,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,7 @@ async def open_file(

def _get_new_path(self, path_provider: PathProvider) -> Path:
info = path_provider(device_name="pattern")
filename = info.filename
new_path: Path = info.root / info.resource_dir / filename
new_path: Path = info.directory_path / info.filename
return new_path

async def collect_stream_docs(
Expand All @@ -188,7 +187,6 @@ async def collect_stream_docs(
if not self._hdf_stream_provider:
assert self.target_path, "open file has not been called"
self._hdf_stream_provider = HDFFile(
self._path_provider(),
self.target_path,
self._datasets,
)
Expand Down
8 changes: 4 additions & 4 deletions tests/core/test_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ def test_auto_increment_path_provider(static_filename_provider, tmp_path):

for i in range(3):
info = auto_inc_path_provider()
assert os.path.basename(info.resource_dir) == "00000"
assert os.path.basename(info.directory_path) == "00000"
info = auto_inc_path_provider()
assert os.path.basename(info.resource_dir) == "00002"
assert os.path.basename(info.directory_path) == "00002"


def test_ymd_path_provider(static_filename_provider, tmp_path):
Expand All @@ -79,7 +79,7 @@ def test_ymd_path_provider(static_filename_provider, tmp_path):
)

info_a = ymd_path_provider()
assert info_a.resource_dir == date_path
assert info_a.directory_path == tmp_path / date_path

info_b = ymd_path_provider(device_name="test_device")
assert info_b.resource_dir == os.path.join("test_device", date_path)
assert info_b.directory_path == tmp_path / "test_device" / date_path
2 changes: 1 addition & 1 deletion tests/epics/adaravis/test_aravis.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ async def test_can_collect(
test_adaravis: adaravis.AravisDetector, static_path_provider: PathProvider
):
path_info = static_path_provider()
full_file_name = path_info.root / path_info.resource_dir / "foo.h5"
full_file_name = path_info.directory_path / "foo.h5"
set_mock_value(test_adaravis.hdf.full_file_name, str(full_file_name))
set_mock_value(test_adaravis._writer.hdf.file_path_exists, True)
set_mock_value(test_adaravis._writer.hdf.capture, True)
Expand Down
2 changes: 1 addition & 1 deletion tests/epics/adkinetix/test_kinetix.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ async def test_can_collect(
test_adkinetix: adkinetix.KinetixDetector, static_path_provider: StaticPathProvider
):
path_info = static_path_provider()
full_file_name = path_info.root / path_info.resource_dir / "foo.h5"
full_file_name = path_info.directory_path / "foo.h5"
set_mock_value(test_adkinetix.hdf.full_file_name, str(full_file_name))
set_mock_value(test_adkinetix._writer.hdf.file_path_exists, True)
set_mock_value(test_adkinetix._writer.hdf.capture, True)
Expand Down
23 changes: 12 additions & 11 deletions tests/epics/adsimdetector/test_sim.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,15 +176,11 @@ async def test_two_detectors_step(
info_a = writer_a._path_provider(device_name=writer_a.hdf.name)
info_b = writer_b._path_provider(device_name=writer_b.hdf.name)

assert await writer_a.hdf.file_path.get_value() == str(
info_a.root / info_a.resource_dir
)
assert await writer_a.hdf.file_path.get_value() == info_a.directory_path
file_name_a = await writer_a.hdf.file_name.get_value()
assert file_name_a == info_a.filename

assert await writer_b.hdf.file_path.get_value() == str(
info_b.root / info_b.resource_dir
)
assert await writer_b.hdf.file_path.get_value() == info_b.directory_path
file_name_b = await writer_b.hdf.file_name.get_value()
assert file_name_b == info_b.filename

Expand All @@ -195,8 +191,12 @@ async def test_two_detectors_step(
assert descriptor["data_keys"]["testb"]["shape"] == (769, 1025)
assert sda["stream_resource"] == sra["uid"]
assert sdb["stream_resource"] == srb["uid"]
assert srb["uri"] == str("file://localhost") + str(info_b.root / file_name_b)
assert sra["uri"] == str("file://localhost") + str(info_a.root / file_name_a)
assert srb["uri"] == str("file://localhost") + str(
info_b.directory_path / file_name_b
)
assert sra["uri"] == str("file://localhost") + str(
info_a.directory_path / file_name_a
)

assert event["data"] == {}

Expand All @@ -214,9 +214,10 @@ async def test_detector_writes_to_file(

RE(count_sim([single_detector], times=3))

assert await cast(
adcore.ADHDFWriter, single_detector.writer
).hdf.file_path.get_value() == str(tmp_path)
assert (
await cast(adcore.ADHDFWriter, single_detector.writer).hdf.file_path.get_value()
== tmp_path
)

descriptor_index = names.index("descriptor")

Expand Down
2 changes: 1 addition & 1 deletion tests/epics/advimba/test_vimba.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ async def test_can_collect(
test_advimba: advimba.VimbaDetector, static_path_provider: PathProvider
):
path_info = static_path_provider()
full_file_name = path_info.root / path_info.resource_dir / "foo.h5"
full_file_name = path_info.directory_path / "foo.h5"
set_mock_value(test_advimba.hdf.full_file_name, str(full_file_name))
set_mock_value(test_advimba._writer.hdf.file_path_exists, True)
set_mock_value(test_advimba._writer.hdf.capture, True)
Expand Down
2 changes: 1 addition & 1 deletion tests/fastcs/panda/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ async def test_open_close_sets_capture(mock_writer: PandaHDFWriter):
async def test_open_sets_file_path_and_name(mock_writer: PandaHDFWriter, tmp_path):
await mock_writer.open()
path = await mock_writer.panda_device.data.hdf_directory.get_value()
assert path.startswith(str(tmp_path))
assert path == tmp_path / mock_writer.panda_device.name
name = await mock_writer.panda_device.data.hdf_file_name.get_value()
assert name == "data.h5"

Expand Down

0 comments on commit 955414a

Please sign in to comment.