From 13cb2be038c37237dc9ebe2ccf071bd25c5c2c44 Mon Sep 17 00:00:00 2001
From: Hotte <hobenhaus@gmail.com>
Date: Fri, 12 May 2023 01:53:06 -0400
Subject: [PATCH 1/2] When retrieving data from S3, account for chunking
---
datajoint/s3.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/datajoint/s3.py b/datajoint/s3.py
index c167c559d..acc722b3d 100644
--- a/datajoint/s3.py
+++ b/datajoint/s3.py
@@ -68,7 +68,9 @@ def fput(self, local_file, name, metadata=None):
def get(self, name):
logger.debug("get: {}:{}".format(self.bucket, name))
try:
- return self.client.get_object(self.bucket, str(name)).data
+ with self.client.get_object(self.bucket, str(name)) as result:
+ data = [d for d in result.stream()]
+ return b"".join(data)
except minio.error.S3Error as e:
if e.code == "NoSuchKey":
raise errors.MissingExternalFile("Missing s3 key %s" % name)
From 2fce9ae243605f63b0275e462ee9d4e5843e7998 Mon Sep 17 00:00:00 2001
From: A-Baji <bidabaji@gmail.com>
Date: Fri, 12 May 2023 21:16:44 +0000
Subject: [PATCH 2/2] update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6a4bf8fcb..76f35ffc1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
- Fixed - `.ipynb` output in tutorials is not visible in dark mode ([#1078](https://github.com/datajoint/datajoint-python/issues/1078)) PR [#1080](https://github.com/datajoint/datajoint-python/pull/1080)
- Changed - Readme to update links and include example pipeline image
- Changed - Docs to add landing page and update navigation
+- Changed - `.data` method to `.stream` in the `get()` method for S3 (external) objects PR [#1085](https://github.com/datajoint/datajoint-python/pull/1085)
### 0.14.0 -- Feb 13, 2023
- Added - `json` data type ([#245](https://github.com/datajoint/datajoint-python/issues/245)) PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051)