aixplain · Apr 15, 2024
diff --git a/‎aixplain/enums/data_type.py
+1-1 b/‎aixplain/enums/data_type.py
+1-1
diff --git a/‎aixplain/processes/data_onboarding/onboard_functions.py
+2-2 b/‎aixplain/processes/data_onboarding/onboard_functions.py
+2-2
diff --git a/‎aixplain/processes/data_onboarding/process_media_files.py
+24 b/‎aixplain/processes/data_onboarding/process_media_files.py
+24
diff --git a/‎aixplain/processes/data_onboarding/process_text_files.py
+1-1 b/‎aixplain/processes/data_onboarding/process_text_files.py
+1-1
diff --git a/‎docs/samples/label_dataset_onboarding/corpus/images/1.jpg
622 KB b/‎docs/samples/label_dataset_onboarding/corpus/images/1.jpg
622 KB
diff --git a/‎docs/samples/label_dataset_onboarding/corpus/images/2.png
6.24 MB b/‎docs/samples/label_dataset_onboarding/corpus/images/2.png
6.24 MB
diff --git a/‎docs/samples/label_dataset_onboarding/corpus/index.csv
+3 b/‎docs/samples/label_dataset_onboarding/corpus/index.csv
+3
diff --git a/‎docs/samples/label_dataset_onboarding/corpus/labels/1.json
+9 b/‎docs/samples/label_dataset_onboarding/corpus/labels/1.json
+9
diff --git a/‎docs/samples/label_dataset_onboarding/corpus/labels/2.json
+9 b/‎docs/samples/label_dataset_onboarding/corpus/labels/2.json
+9
@@ -35,4 +35,4 @@ class DataType(Enum):
     VIDEO = "video"
 
     def __str__(self):
-        return self._value_
+        return self._value_
@@ -97,11 +97,11 @@ def process_data_files(
         -1,
         0,
     )
-    if metadata.dtype in [DataType.AUDIO, DataType.IMAGE] or metadata.dsubtype == DataSubtype.INTERVAL:
+    if metadata.dtype in [DataType.AUDIO, DataType.IMAGE, DataType.LABEL] or metadata.dsubtype == DataSubtype.INTERVAL:
         files, data_column_idx, start_column_idx, end_column_idx, nrows = process_media_files.run(
             metadata=metadata, paths=paths, folder=folder
         )
-    elif metadata.dtype in [DataType.TEXT, DataType.LABEL]:
+    elif metadata.dtype in [DataType.TEXT]:
         files, data_column_idx, nrows = process_text_files.run(metadata=metadata, paths=paths, folder=folder)
     return files, data_column_idx, start_column_idx, end_column_idx, nrows
 
 
@@ -16,6 +16,7 @@
 from pathlib import Path
 from tqdm import tqdm
 from typing import List, Tuple
+from urllib.parse import urlparse
 
 AUDIO_MAX_SIZE = 50000000
 IMAGE_TEXT_MAX_SIZE = 25000000
@@ -45,6 +46,15 @@ def run(metadata: MetaData, paths: List, folder: Path, batch_size: int = 100) ->
     Returns:
         Tuple[List[File], int, int, int]: list of s3 links; data, start and end columns index, and number of rows
     """
+    if metadata.dtype == DataType.LABEL:
+        assert (
+            metadata.storage_type != StorageType.TEXT
+        ), f'Data Asset Onboarding Error: Column "{metadata.name}" of type "{metadata.dtype}" can not be stored in text. Label data should be stored in a JSON file.'
+    else:
+        assert (
+            metadata.storage_type != StorageType.TEXT
+        ), f'Data Asset Onboarding Error: Column "{metadata.name}" of type "{metadata.dtype}" can not be stored in text.'
+
     # if files are stored locally, create a folder to store it
     media_folder = Path(".")
     if metadata.storage_type == StorageType.FILE:
@@ -95,6 +105,14 @@ def run(metadata: MetaData, paths: List, folder: Path, batch_size: int = 100) ->
                     assert (
                         os.path.getsize(media_path) <= AUDIO_MAX_SIZE
                     ), f'Data Asset Onboarding Error: Local audio file "{media_path}" exceeds the size limit of 50 MB.'
+                elif metadata.dtype == DataType.LABEL:
+                    assert (
+                        os.path.getsize(media_path) <= IMAGE_TEXT_MAX_SIZE
+                    ), f'Data Asset Onboarding Error: JSON file with labels "{media_path}" exceeds the size limit of 25 MB.'
+                    _, extension = os.path.splitext(media_path)
+                    assert (
+                        extension == ".json"
+                    ), f'Data Asset Onboarding Error: Label data should be stored in a JSON file and "{media_path}" is not one.'
                 else:
                     assert (
                         os.path.getsize(media_path) <= IMAGE_TEXT_MAX_SIZE
@@ -105,6 +123,12 @@ def run(metadata: MetaData, paths: List, folder: Path, batch_size: int = 100) ->
                     shutil.copy2(media_path, new_path)
                 batch.append(fname)
             else:
+                if metadata.dtype == DataType.LABEL:
+                    path = urlparse(media_path).path
+                    _, extension = os.path.splitext(path)
+                    assert (
+                        extension == ".json"
+                    ), f'Data Asset Onboarding Error: Label data should be stored in a JSON file and "{media_path}" is not one.'
                 batch.append(media_path)
 
             # crop intervals can not be used with interval data types
 
@@ -26,7 +26,7 @@ def process_text(content: str, storage_type: StorageType) -> Text:
         Text: textual content
     """
     if storage_type == StorageType.FILE:
-        # Check the size of file and assert a limit of 50 MB
+        # Check the size of file and assert a limit of 25 MB
         assert (
             os.path.getsize(content) <= 25000000
         ), f'Data Asset Onboarding Error: Local text file "{content}" exceeds the size limit of 25 MB.'
 
@@ -0,0 +1,3 @@
+,images,labels
+0,corpus/images/1.jpg,corpus/labels/1.json
+1,corpus/images/2.png,corpus/labels/2.json
@@ -0,0 +1,9 @@
+{
+    "data": "arcade",
+    "boundingBox": {
+        "top": 0,
+        "bottom": 0,
+        "left": 0,
+        "right": 0
+    }
+}
@@ -0,0 +1,9 @@
+{
+    "data": "building",
+    "boundingBox": {
+        "top": 0,
+        "bottom": 0,
+        "left": 0,
+        "right": 0
+    }
+}
Original file line number	Diff line number	Diff line change
`@@ -97,11 +97,11 @@ def process_data_files(`
`97`	`97`	`-1,`
`98`	`98`	`0,`
`99`	`99`	`)`
`100`		`- if metadata.dtype in [DataType.AUDIO, DataType.IMAGE] or metadata.dsubtype == DataSubtype.INTERVAL:`
	`100`	`+ if metadata.dtype in [DataType.AUDIO, DataType.IMAGE, DataType.LABEL] or metadata.dsubtype == DataSubtype.INTERVAL:`
`101`	`101`	`files, data_column_idx, start_column_idx, end_column_idx, nrows = process_media_files.run(`
`102`	`102`	`metadata=metadata, paths=paths, folder=folder`
`103`	`103`	`)`
`104`		`- elif metadata.dtype in [DataType.TEXT, DataType.LABEL]:`
	`104`	`+ elif metadata.dtype in [DataType.TEXT]:`
`105`	`105`	`files, data_column_idx, nrows = process_text_files.run(metadata=metadata, paths=paths, folder=folder)`
`106`	`106`	`return files, data_column_idx, start_column_idx, end_column_idx, nrows`
`107`	`107`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+,images,labels`
	`2`	`+0,corpus/images/1.jpg,corpus/labels/1.json`
	`3`	`+1,corpus/images/2.png,corpus/labels/2.json`