Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove references to root in PathInfo, PathProvider #477

Merged
merged 6 commits into from
Jul 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions src/ophyd_async/core/_hdf_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
StreamResource,
)

from ._providers import PathInfo


@dataclass
class HDFDataset:
Expand All @@ -28,14 +26,12 @@ class HDFDataset:

class HDFFile:
"""
:param directory_info: Contains information about how to construct a StreamResource
:param full_file_name: Absolute path to the file to be written
:param datasets: Datasets to write into the file
"""

def __init__(
self,
path_info: PathInfo,
full_file_name: Path,
datasets: List[HDFDataset],
hostname: str = "localhost",
Expand All @@ -53,7 +49,7 @@ def __init__(
(
"file",
self._hostname,
str((path_info.root / full_file_name).absolute()),
str(full_file_name.absolute()),
"",
"",
None,
Expand Down
45 changes: 17 additions & 28 deletions src/ophyd_async/core/_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,13 @@ class PathInfo:
"""
Information about where and how to write a file.

The bluesky event model splits the URI for a resource into two segments to aid in
different applications mounting filesystems at different mount points.
The portion of this path which is relevant only for the writer is the 'root',
while the path from an agreed upon mutual mounting is the resource_path.
The resource_dir is used with the filename to construct the resource_path.

:param root: Path of a root directory, relevant only for the file writer
:param resource_dir: Directory into which files should be written, relative to root
:param directory_path: Directory into which files should be written
:param filename: Base filename to use generated by FilenameProvider, w/o extension
:param create_dir_depth: Optional depth of directories to create if they do not
exist
"""

root: Path
resource_dir: Path
directory_path: Path
filename: str
create_dir_depth: int = 0

Expand Down Expand Up @@ -112,20 +104,17 @@ def __init__(
self,
filename_provider: FilenameProvider,
directory_path: Path,
resource_dir: Path = Path("."),
create_dir_depth: int = 0,
) -> None:
self._filename_provider = filename_provider
self._directory_path = directory_path
self._resource_dir = resource_dir
self._create_dir_depth = create_dir_depth

def __call__(self, device_name: Optional[str] = None) -> PathInfo:
filename = self._filename_provider()

return PathInfo(
root=self._directory_path,
resource_dir=self._resource_dir,
directory_path=self._directory_path,
filename=filename,
create_dir_depth=self._create_dir_depth,
)
Expand All @@ -135,7 +124,7 @@ class AutoIncrementingPathProvider(PathProvider):
def __init__(
self,
filename_provider: FilenameProvider,
directory_path: Path,
base_directory_path: Path,
create_dir_depth: int = 0,
max_digits: int = 5,
starting_value: int = 0,
Expand All @@ -145,7 +134,7 @@ def __init__(
base_name: str = None,
) -> None:
self._filename_provider = filename_provider
self._directory_path = directory_path
self._base_directory_path = base_directory_path
self._create_dir_depth = create_dir_depth
self._base_name = base_name
self._starting_value = starting_value
Expand All @@ -161,20 +150,21 @@ def __call__(self, device_name: Optional[str] = None) -> PathInfo:

padded_counter = f"{self._current_value:0{self._max_digits}}"

resource_dir = str(padded_counter)
auto_inc_dir_name = str(padded_counter)
if self._base_name is not None:
resource_dir = f"{self._base_name}{self._inc_delimeter}{padded_counter}"
auto_inc_dir_name = (
f"{self._base_name}{self._inc_delimeter}{padded_counter}"
)
elif device_name is not None:
resource_dir = f"{device_name}{self._inc_delimeter}{padded_counter}"
auto_inc_dir_name = f"{device_name}{self._inc_delimeter}{padded_counter}"

self._inc_counter += 1
if self._inc_counter == self._num_calls_per_inc:
self._inc_counter = 0
self._current_value += self._increment

return PathInfo(
root=self._directory_path,
resource_dir=resource_dir,
directory_path=self._base_directory_path / auto_inc_dir_name,
filename=filename,
create_dir_depth=self._create_dir_depth,
)
Expand All @@ -184,35 +174,34 @@ class YMDPathProvider(PathProvider):
def __init__(
self,
filename_provider: FilenameProvider,
directory_path: Path,
base_directory_path: Path,
create_dir_depth: int = -3, # Default to -3 to create YMD dirs
device_name_as_base_dir: bool = False,
) -> None:
self._filename_provider = filename_provider
self._directory_path = Path(directory_path)
self._base_directory_path = Path(base_directory_path)
self._create_dir_depth = create_dir_depth
self._device_name_as_base_dir = device_name_as_base_dir

def __call__(self, device_name: Optional[str] = None) -> PathInfo:
sep = os.path.sep
current_date = date.today().strftime(f"%Y{sep}%m{sep}%d")
if device_name is None:
resource_dir = current_date
ymd_dir_path = current_date
elif self._device_name_as_base_dir:
resource_dir = os.path.join(
ymd_dir_path = os.path.join(
current_date,
device_name,
)
else:
resource_dir = os.path.join(
ymd_dir_path = os.path.join(
device_name,
current_date,
)

filename = self._filename_provider()
return PathInfo(
root=self._directory_path,
resource_dir=resource_dir,
directory_path=self._base_directory_path / ymd_dir_path,
filename=filename,
create_dir_depth=self._create_dir_depth,
)
Expand Down
3 changes: 1 addition & 2 deletions src/ophyd_async/epics/adcore/_hdf_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __init__(
async def open(self, multiplier: int = 1) -> Dict[str, DataKey]:
self._file = None
info = self._path_provider(device_name=self.hdf.name)
file_path = str(info.root / info.resource_dir)
file_path = info.directory_path
await asyncio.gather(
self.hdf.num_extra_dims.set(0),
self.hdf.lazy_open.set(True),
Expand Down Expand Up @@ -133,7 +133,6 @@ async def collect_stream_docs(
if not self._file:
path = Path(await self.hdf.full_file_name.get_value())
self._file = HDFFile(
self._path_provider(),
# See https://github.com/bluesky/ophyd-async/issues/122
path,
self._datasets,
Expand Down
5 changes: 1 addition & 4 deletions src/ophyd_async/fastcs/panda/_hdf_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,7 @@ async def open(self, multiplier: int = 1) -> Dict[str, DataKey]:
info = self._path_provider(device_name=self.panda_device.name)
# Set the initial values
await asyncio.gather(
self.panda_device.data.hdf_directory.set(
str(info.root / info.resource_dir)
),
self.panda_device.data.hdf_directory.set(info.directory_path),
self.panda_device.data.hdf_file_name.set(
f"{info.filename}.h5",
),
Expand Down Expand Up @@ -130,7 +128,6 @@ async def collect_stream_docs(
if indices_written:
if not self._file:
self._file = HDFFile(
self._path_provider(),
Path(await self.panda_device.data.hdf_directory.get_value())
/ Path(await self.panda_device.data.hdf_file_name.get_value()),
self._datasets,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,7 @@ async def open_file(

def _get_new_path(self, path_provider: PathProvider) -> Path:
info = path_provider(device_name="pattern")
filename = info.filename
new_path: Path = info.root / info.resource_dir / filename
new_path: Path = info.directory_path / info.filename
return new_path

async def collect_stream_docs(
Expand All @@ -188,7 +187,6 @@ async def collect_stream_docs(
if not self._hdf_stream_provider:
assert self.target_path, "open file has not been called"
self._hdf_stream_provider = HDFFile(
self._path_provider(),
self.target_path,
self._datasets,
)
Expand Down
8 changes: 4 additions & 4 deletions tests/core/test_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ def test_auto_increment_path_provider(static_filename_provider, tmp_path):

for i in range(3):
info = auto_inc_path_provider()
assert os.path.basename(info.resource_dir) == "00000"
assert os.path.basename(info.directory_path) == "00000"
info = auto_inc_path_provider()
assert os.path.basename(info.resource_dir) == "00002"
assert os.path.basename(info.directory_path) == "00002"


def test_ymd_path_provider(static_filename_provider, tmp_path):
Expand All @@ -79,7 +79,7 @@ def test_ymd_path_provider(static_filename_provider, tmp_path):
)

info_a = ymd_path_provider()
assert info_a.resource_dir == date_path
assert info_a.directory_path == tmp_path / date_path

info_b = ymd_path_provider(device_name="test_device")
assert info_b.resource_dir == os.path.join("test_device", date_path)
assert info_b.directory_path == tmp_path / "test_device" / date_path
2 changes: 1 addition & 1 deletion tests/epics/adaravis/test_aravis.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ async def test_can_collect(
test_adaravis: adaravis.AravisDetector, static_path_provider: PathProvider
):
path_info = static_path_provider()
full_file_name = path_info.root / path_info.resource_dir / "foo.h5"
full_file_name = path_info.directory_path / "foo.h5"
set_mock_value(test_adaravis.hdf.full_file_name, str(full_file_name))
set_mock_value(test_adaravis._writer.hdf.file_path_exists, True)
set_mock_value(test_adaravis._writer.hdf.capture, True)
Expand Down
2 changes: 1 addition & 1 deletion tests/epics/adkinetix/test_kinetix.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ async def test_can_collect(
test_adkinetix: adkinetix.KinetixDetector, static_path_provider: StaticPathProvider
):
path_info = static_path_provider()
full_file_name = path_info.root / path_info.resource_dir / "foo.h5"
full_file_name = path_info.directory_path / "foo.h5"
set_mock_value(test_adkinetix.hdf.full_file_name, str(full_file_name))
set_mock_value(test_adkinetix._writer.hdf.file_path_exists, True)
set_mock_value(test_adkinetix._writer.hdf.capture, True)
Expand Down
23 changes: 12 additions & 11 deletions tests/epics/adsimdetector/test_sim.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,15 +176,11 @@ async def test_two_detectors_step(
info_a = writer_a._path_provider(device_name=writer_a.hdf.name)
info_b = writer_b._path_provider(device_name=writer_b.hdf.name)

assert await writer_a.hdf.file_path.get_value() == str(
info_a.root / info_a.resource_dir
)
assert await writer_a.hdf.file_path.get_value() == info_a.directory_path
file_name_a = await writer_a.hdf.file_name.get_value()
assert file_name_a == info_a.filename

assert await writer_b.hdf.file_path.get_value() == str(
info_b.root / info_b.resource_dir
)
assert await writer_b.hdf.file_path.get_value() == info_b.directory_path
file_name_b = await writer_b.hdf.file_name.get_value()
assert file_name_b == info_b.filename

Expand All @@ -195,8 +191,12 @@ async def test_two_detectors_step(
assert descriptor["data_keys"]["testb"]["shape"] == (769, 1025)
assert sda["stream_resource"] == sra["uid"]
assert sdb["stream_resource"] == srb["uid"]
assert srb["uri"] == str("file://localhost") + str(info_b.root / file_name_b)
assert sra["uri"] == str("file://localhost") + str(info_a.root / file_name_a)
assert srb["uri"] == str("file://localhost") + str(
info_b.directory_path / file_name_b
)
assert sra["uri"] == str("file://localhost") + str(
info_a.directory_path / file_name_a
)

assert event["data"] == {}

Expand All @@ -214,9 +214,10 @@ async def test_detector_writes_to_file(

RE(count_sim([single_detector], times=3))

assert await cast(
adcore.ADHDFWriter, single_detector.writer
).hdf.file_path.get_value() == str(tmp_path)
assert (
await cast(adcore.ADHDFWriter, single_detector.writer).hdf.file_path.get_value()
== tmp_path
)

descriptor_index = names.index("descriptor")

Expand Down
2 changes: 1 addition & 1 deletion tests/epics/advimba/test_vimba.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ async def test_can_collect(
test_advimba: advimba.VimbaDetector, static_path_provider: PathProvider
):
path_info = static_path_provider()
full_file_name = path_info.root / path_info.resource_dir / "foo.h5"
full_file_name = path_info.directory_path / "foo.h5"
set_mock_value(test_advimba.hdf.full_file_name, str(full_file_name))
set_mock_value(test_advimba._writer.hdf.file_path_exists, True)
set_mock_value(test_advimba._writer.hdf.capture, True)
Expand Down
2 changes: 1 addition & 1 deletion tests/fastcs/panda/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ async def test_open_close_sets_capture(mock_writer: PandaHDFWriter):
async def test_open_sets_file_path_and_name(mock_writer: PandaHDFWriter, tmp_path):
await mock_writer.open()
path = await mock_writer.panda_device.data.hdf_directory.get_value()
assert path.startswith(str(tmp_path))
assert path == tmp_path / mock_writer.panda_device.name
name = await mock_writer.panda_device.data.hdf_file_name.get_value()
assert name == "data.h5"

Expand Down