From f3768f6d17ae47e0536b4b9c16fffe4772ede856 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Mon, 28 Oct 2019 12:06:04 -0400 Subject: [PATCH 1/2] Use NumPy's `fromfile` and `tofile` For reading and writing data on disk, start using NumPy's `fromfile` and `tofile`. --- zarr/storage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index 8b387ed1f8..4b867651d0 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -34,6 +34,8 @@ from threading import Lock, RLock import uuid +import numpy + from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray from numcodecs.registry import codec_registry @@ -741,7 +743,7 @@ def __getitem__(self, key): filepath = os.path.join(self.path, key) if os.path.isfile(filepath): with open(filepath, 'rb') as f: - return f.read() + return numpy.fromfile(f, dtype='u1') else: raise KeyError(key) @@ -774,7 +776,7 @@ def __setitem__(self, key, value): temp_path = os.path.join(dir_path, temp_name) try: with open(temp_path, mode='wb') as f: - f.write(value) + value.tofile(f) # move temporary file into place replace(temp_path, file_path) From fbc91dfe647a39f8276c8ed4adb58bf523c1e0f7 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Mon, 28 Oct 2019 12:09:19 -0400 Subject: [PATCH 2/2] Let NumPy handle opening/closing files Instead of opening and closing files ourselves, go ahead and let NumPy handle this when reading from and writing to disk. --- zarr/storage.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index 4b867651d0..670fd62a3e 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -742,8 +742,7 @@ def __init__(self, path): def __getitem__(self, key): filepath = os.path.join(self.path, key) if os.path.isfile(filepath): - with open(filepath, 'rb') as f: - return numpy.fromfile(f, dtype='u1') + return numpy.fromfile(filepath, dtype='u1') else: raise KeyError(key) @@ -775,8 +774,7 @@ def __setitem__(self, key, value): temp_name = file_name + '.' + uuid.uuid4().hex + '.partial' temp_path = os.path.join(dir_path, temp_name) try: - with open(temp_path, mode='wb') as f: - value.tofile(f) + value.tofile(temp_path) # move temporary file into place replace(temp_path, file_path)