Update dataloaders to use Aggregation Lists (#264)

This PR updates the TensorFlow and PyTorch data loaders. The data loaders now make use of SmartRedis's aggregation lists and are easier to set up and use. Notice that the data loader API has been adapted and is not backward compatible.

[ committed by @al-rigazzi ]
[ reviewed by @mellis13 @MattToast ]

doc/changelog.rst

@@ -22,6 +22,7 @@ This section details changes made in the development branch that have not yet be
 
 Description
 
+- Update ML data loaders to make use of SmartRedis's aggregation lists
 - Drop support for Ray
 - Allow for models to be launched independently as batch jobs
 - Update to current version of Redis
@@ -31,6 +32,7 @@ Description
 
 Detailed Notes
 
+- The PyTorch and TensorFlow data loaders were update to make use of aggregation lists. This breaks their API, but makes them easier to use. (PR264_)
 - The support for Ray was dropped, as its most recent versions caused problems when deployed through SmartSim.
   We plan to release a separate add-on library to accomplish the same results. If
   you are interested in getting the Ray launch functionality back in your workflow, please get in touch with us! (PR263_)
@@ -49,6 +51,7 @@ Detailed Notes
 - The release of RedisAI 1.2.7 allows us to update support for recent versions of PyTorch, Tensorflow, and ONNX (PR234_)
 - Make installation of correct Torch backend more reliable according to instruction from PyTorch
 
+.. _PR264: https://github.com/CrayLabs/SmartSim/pull/264
 .. _PR263: https://github.com/CrayLabs/SmartSim/pull/263
 .. _PR258: https://github.com/CrayLabs/SmartSim/pull/258
 .. _PR256: https://github.com/CrayLabs/SmartSim/pull/256

smartsim/_core/control/controller.py

@@ -290,7 +290,6 @@ def _launch(self, manifest):
                 raise SmartSimError(msg)
             self._launch_orchestrator(orchestrator)
 
-
         if self.orchestrator_active:
             self._set_dbobjects(manifest)
 

smartsim/_core/control/manifest.py

@@ -50,7 +50,6 @@ def __init__(self, *args):
         self._check_names(self._deployables)
         self._check_entity_lists_nonempty()
 
-
     @property
     def db(self):
         """Return Orchestrator instances in Manifest
@@ -69,7 +68,6 @@ def db(self):
                 _db = deployable
         return _db
 
-
     @property
     def models(self):
         """Return Model instances in Manifest
@@ -83,7 +81,6 @@ def models(self):
                 _models.append(deployable)
         return _models
 
-
     @property
     def ensembles(self):
         """Return Ensemble instances in Manifest
@@ -103,7 +100,6 @@ def ensembles(self):
 
         return _ensembles
 
-
     @property
     def all_entity_lists(self):
         """All entity lists, including ensembles and
@@ -119,7 +115,6 @@ def all_entity_lists(self):
 
         return _all_entity_lists
 
-
     def _check_names(self, deployables):
         used = []
         for deployable in deployables:
@@ -130,7 +125,6 @@ def _check_names(self, deployables):
                 raise SmartSimError("User provided two entities with the same name")
             used.append(name)
 
-
     def _check_types(self, deployables):
         for deployable in deployables:
             if not (
@@ -141,15 +135,13 @@ def _check_types(self, deployables):
                     f"Entity has type {type(deployable)}, not SmartSimEntity or EntityList"
                 )
 
-
     def _check_entity_lists_nonempty(self):
         """Check deployables for sanity before launching"""
 
         for entity_list in self.all_entity_lists:
             if len(entity_list) < 1:
                 raise ValueError(f"{entity_list.name} is empty. Nothing to launch.")
 
-
     def __str__(self):
         s = ""
         e_header = "=== Ensembles ===\n"

smartsim/experiment.py

@@ -774,7 +774,12 @@ def summary(self, format="github"):
                 )
         else:
             return tabulate(
-                values, headers, showindex=True, tablefmt=format, missingval="None", disable_numparse=True
+                values,
+                headers,
+                showindex=True,
+                tablefmt=format,
+                missingval="None",
+                disable_numparse=True,
             )
 
     def _launch_summary(self, manifest):

smartsim/ml/__init__.py

@@ -24,9 +24,4 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from .data import (
-    DynamicDataDownloader,
-    StaticDataDownloader,
-    TrainingDataUploader,
-    form_name,
-)
+from .data import DataDownloader, DataInfo, TrainingDataUploader, form_name