diff --git a/tests/test_0001-source-class.py b/tests/test_0001-source-class.py index 04d7897c8..dbb801317 100644 --- a/tests/test_0001-source-class.py +++ b/tests/test_0001-source-class.py @@ -169,14 +169,14 @@ def test_http_port(): @pytest.mark.network def test_http_size(): with uproot4.source.http.HTTPSource( - "https://scikit-hep.org/uproot/examples/Zmumu.root", + "https://scikit-hep.org/uproot3/examples/Zmumu.root", timeout=10, num_fallback_workers=1, ) as source: size1 = source.num_bytes with uproot4.source.http.MultithreadedHTTPSource( - "https://scikit-hep.org/uproot/examples/Zmumu.root", num_workers=1, timeout=10 + "https://scikit-hep.org/uproot3/examples/Zmumu.root", num_workers=1, timeout=10 ) as source: size2 = source.num_bytes @@ -186,14 +186,14 @@ def test_http_size(): @pytest.mark.network def test_http_size_port(): with uproot4.source.http.HTTPSource( - "https://scikit-hep.org:443/uproot/examples/Zmumu.root", + "https://scikit-hep.org:443/uproot3/examples/Zmumu.root", timeout=10, num_fallback_workers=1, ) as source: size1 = source.num_bytes with uproot4.source.http.MultithreadedHTTPSource( - "https://scikit-hep.org:443/uproot/examples/Zmumu.root", + "https://scikit-hep.org:443/uproot3/examples/Zmumu.root", num_workers=1, timeout=10, ) as source: @@ -217,7 +217,7 @@ def test_http_fail(): def test_no_multipart(): for num_workers in [1, 2]: with uproot4.source.http.MultithreadedHTTPSource( - "https://scikit-hep.org/uproot/examples/Zmumu.root", + "https://scikit-hep.org/uproot3/examples/Zmumu.root", num_workers=num_workers, timeout=10, ) as source: @@ -246,7 +246,7 @@ def test_no_multipart_fail(): def test_fallback(): for num_workers in [1, 2]: with uproot4.source.http.HTTPSource( - "https://scikit-hep.org/uproot/examples/Zmumu.root", + "https://scikit-hep.org/uproot3/examples/Zmumu.root", timeout=10, num_fallback_workers=num_workers, ) as source: diff --git a/tests/test_0006-notify-when-downloaded.py b/tests/test_0006-notify-when-downloaded.py index a2386341a..95131c20c 100644 --- a/tests/test_0006-notify-when-downloaded.py +++ b/tests/test_0006-notify-when-downloaded.py @@ -126,7 +126,7 @@ def test_http_workers(): def test_http_fallback(): notifications = queue.Queue() with uproot4.source.http.HTTPSource( - "https://scikit-hep.org/uproot/examples/Zmumu.root", + "https://scikit-hep.org/uproot3/examples/Zmumu.root", timeout=10, num_fallback_workers=1, ) as source: @@ -143,7 +143,7 @@ def test_http_fallback(): def test_http_fallback_workers(): notifications = queue.Queue() with uproot4.source.http.HTTPSource( - "https://scikit-hep.org/uproot/examples/Zmumu.root", + "https://scikit-hep.org/uproot3/examples/Zmumu.root", timeout=10, num_fallback_workers=5, ) as source: diff --git a/tests/test_0045-lazy-arrays-1.py b/tests/test_0045-lazy-arrays-1.py index 5b481fe56..1e472a35a 100644 --- a/tests/test_0045-lazy-arrays-1.py +++ b/tests/test_0045-lazy-arrays-1.py @@ -151,7 +151,8 @@ def test_awkward_pluralization(): def test_lazy_called_on_nonexistent_file(): + awkward1 = pytest.importorskip("awkward1") filename = "nonexistent_file.root" - with pytest.raises(FileNotFoundError) as excinfo: + with pytest.raises(uproot4._util._FileNotFoundError) as excinfo: uproot4.lazy(filename) assert filename in str(excinfo.value) diff --git a/tests/test_0066-fix-http-fallback-freeze.py b/tests/test_0066-fix-http-fallback-freeze.py index 254de5339..b406d07e6 100644 --- a/tests/test_0066-fix-http-fallback-freeze.py +++ b/tests/test_0066-fix-http-fallback-freeze.py @@ -11,7 +11,7 @@ @pytest.mark.network def test(): with uproot4.open( - {"http://scikit-hep.org/uproot/examples/HZZ.root": "events"} + {"http://scikit-hep.org/uproot3/examples/HZZ.root": "events"} ) as t: - t["MET_px"].array() - t["MET_py"].array() + t["MET_px"].array(library="np") + t["MET_py"].array(library="np") diff --git a/tests/test_0088-read-with-http.py b/tests/test_0088-read-with-http.py index 0039956ce..1e04d7b89 100644 --- a/tests/test_0088-read-with-http.py +++ b/tests/test_0088-read-with-http.py @@ -16,6 +16,16 @@ def test_issue176(): assert len(data) == 100000 +@pytest.mark.network +def test_issue176_again(): + with uproot4.open( + "https://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root" + ) as f: + data = f["tree"].arrays(["Y1", "Y2"], library="np") + assert len(data["Y1"]) == 100000 + assert len(data["Y2"]) == 100000 + + @pytest.mark.network def test_issue121(): with uproot4.open( diff --git a/tests/test_0172-allow-allocators-in-vector-typenames.py b/tests/test_0172-allow-allocators-in-vector-typenames.py index 57f4ed69d..f04db07a4 100644 --- a/tests/test_0172-allow-allocators-in-vector-typenames.py +++ b/tests/test_0172-allow-allocators-in-vector-typenames.py @@ -15,4 +15,4 @@ def test(): assert ( t["rec_part_px_VecOps"].typename == "std::vector" ) # without the allocator - t["rec_part_px_VecOps"].array() + t["rec_part_px_VecOps"].array(library="np") diff --git a/tests/test_0173-empty-and-multiprocessing-bugs.py b/tests/test_0173-empty-and-multiprocessing-bugs.py index 48578129c..08f8d35ae 100644 --- a/tests/test_0173-empty-and-multiprocessing-bugs.py +++ b/tests/test_0173-empty-and-multiprocessing-bugs.py @@ -23,7 +23,7 @@ def readone(filename): f.decompression_executor = uproot4.ThreadPoolExecutor() t = f["events"] b = t["px1"] - b.array() + b.array(library="np") def test_multiprocessing(): diff --git a/tests/test_0182-complain-about-missing-files.py b/tests/test_0182-complain-about-missing-files.py index 183251580..cfddd5157 100644 --- a/tests/test_0182-complain-about-missing-files.py +++ b/tests/test_0182-complain-about-missing-files.py @@ -15,9 +15,12 @@ def test(): bad = one.replace(".root", "-DOES-NOT-EXIST.root") okay = one.replace(".root", "-DOES-NOT-EXIST-*.root") - assert len(list(uproot4.iterate([one, two], step_size="1 TB"))) == 2 + assert len(list(uproot4.iterate([one, two], step_size="1 TB", library="np"))) == 2 with pytest.raises(uproot4._util._FileNotFoundError): - list(uproot4.iterate([one, two, bad])) + list(uproot4.iterate([one, two, bad], library="np")) - assert len(list(uproot4.iterate([one, two, okay], step_size="1 TB"))) == 2 + assert ( + len(list(uproot4.iterate([one, two, okay], step_size="1 TB", library="np"))) + == 2 + ) diff --git a/uproot4/source/http.py b/uproot4/source/http.py index d7ccb5cf4..448c4ad1c 100644 --- a/uproot4/source/http.py +++ b/uproot4/source/http.py @@ -372,8 +372,13 @@ def handle_multipart(self, source, futures, results, response): Helper function for :py:meth:`~uproot4.source.http.HTTPResource.multifuture` to handle the multipart GET response. """ + if hasattr(response, "readline"): + response_buffer = response + else: + response_buffer = _ResponseBuffer(response) + for i in uproot4._util.range(len(futures)): - range_string, size = self.next_header(response) + range_string, size = self.next_header(response_buffer) if range_string is None: raise OSError( """found {0} of {1} expected headers in HTTP multipart @@ -397,7 +402,7 @@ def handle_multipart(self, source, futures, results, response): ) length = stop - start - results[start, stop] = response.read(length) + results[start, stop] = response_buffer.read(length) if len(results[start, stop]) != length: raise OSError( @@ -413,12 +418,12 @@ def handle_multipart(self, source, futures, results, response): future._run(self) - def next_header(self, response): + def next_header(self, response_buffer): """ Helper function for :py:meth:`~uproot4.source.http.HTTPResource.multifuture` - to return the next header from the ``response``. + to return the next header from the ``response_buffer``. """ - line = response.fp.readline() + line = response_buffer.readline() range_string, size = None, None while range_string is None: m = self._content_range_size.match(line) @@ -430,7 +435,7 @@ def next_header(self, response): if m is not None: range_string = m.group(1) size = None - line = response.fp.readline() + line = response_buffer.readline() if len(line.strip()) == 0: break return range_string, size @@ -452,6 +457,39 @@ def task(resource): return uproot4.source.futures.ResourceFuture(task) +class _ResponseBuffer(object): + CHUNK = 1024 + + def __init__(self, stream): + self.already_read = b"" + self.stream = stream + + def read(self, length): + if length < len(self.already_read): + out = self.already_read[:length] + self.already_read = self.already_read[length:] + return out + + elif len(self.already_read) > 0: + out = self.already_read + self.already_read = b"" + return out + self.stream.read(length - len(out)) + + else: + return self.stream.read(length) + + def readline(self): + while True: + try: + index = self.already_read.index(b"\n") + except ValueError: + self.already_read = self.already_read + self.stream.read(self.CHUNK) + else: + out = self.already_read[: index + 1] + self.already_read = self.already_read[index + 1 :] + return out + + class HTTPSource(uproot4.source.chunk.Source): """ Args: