From bc4e9d2af65d01db29550b4181070b52040459f7 Mon Sep 17 00:00:00 2001
From: Doug Ollerenshaw <dougo@alleninstitute.org>
Date: Mon, 8 Jun 2020 21:23:11 -0700
Subject: [PATCH 01/56] replace typing Protocol import

---
 allensdk/core/typing.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/allensdk/core/typing.py b/allensdk/core/typing.py
index df00f7be0..2429575bd 100644
--- a/allensdk/core/typing.py
+++ b/allensdk/core/typing.py
@@ -1,8 +1,12 @@
-from typing import _Protocol
+import sys
+if sys.version_info.minor <= 7:  
+    from typing import _Protocol as Protocol
+else:
+    from typing import Protocol
 from abc import abstractmethod
 
 
-class SupportsStr(_Protocol):
+class SupportsStr(Protocol):
     """Classes that support the __str__ method"""
     @abstractmethod
     def __str__(self) -> str:

From 04c55c303e50b9ca01d8a8e704f8fb20fff3dd4f Mon Sep 17 00:00:00 2001
From: "d.gonzalezmarx" <d.gonzalezmarx@stud.uni-goettingen.de>
Date: Wed, 7 Oct 2020 16:21:12 +0200
Subject: [PATCH 02/56] GH #1739 - Fix ecephys raster_plot plotting units twice

raster_plot now correctly plots each unit once.  A unit
test checks that the number of plotted lines equals the
number of units passed to the raster_plot function.
---
 .../ecephys/visualization/__init__.py              |  6 ++++--
 .../ecephys/test_visualization.py                  | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 allensdk/test/brain_observatory/ecephys/test_visualization.py

diff --git a/allensdk/brain_observatory/ecephys/visualization/__init__.py b/allensdk/brain_observatory/ecephys/visualization/__init__.py
index 6ee6844fb..ecff26ed0 100644
--- a/allensdk/brain_observatory/ecephys/visualization/__init__.py
+++ b/allensdk/brain_observatory/ecephys/visualization/__init__.py
@@ -97,7 +97,9 @@ def raster_plot(spike_times, figsize=(8,8), cmap=plt.cm.tab20, title='spike rast
 
     fig, ax = plt.subplots(figsize=figsize)
     plotter = _VlPlotter(ax, num_objects=len(spike_times['unit_id'].unique()), cmap=cmap, cycle_colors=cycle_colors)
-    spike_times.groupby('unit_id').agg(plotter)
+    # aggregate is called on each column, so pass only one (eg the stimulus_presentation_id)
+    # to plot each unit once
+    spike_times[['stimulus_presentation_id', 'unit_id']].groupby('unit_id').agg(plotter)
     
     ax.set_xlabel('time (s)', fontsize=16)
     ax.set_ylabel('unit', fontsize=16)
@@ -106,4 +108,4 @@ def raster_plot(spike_times, figsize=(8,8), cmap=plt.cm.tab20, title='spike rast
     plt.yticks([])
     plt.axis('tight')
     
-    return fig
\ No newline at end of file
+    return fig
diff --git a/allensdk/test/brain_observatory/ecephys/test_visualization.py b/allensdk/test/brain_observatory/ecephys/test_visualization.py
new file mode 100644
index 000000000..099aa18f6
--- /dev/null
+++ b/allensdk/test/brain_observatory/ecephys/test_visualization.py
@@ -0,0 +1,14 @@
+import allensdk.brain_observatory.ecephys.visualization.__init__ as vis
+import pandas as pd
+
+def test_raster_plot():
+    spike_times = pd.DataFrame({
+        'unit_id': [2, 1, 2],
+        'stimulus_presentation_id': [2, 2, 2, ],
+        'time_since_stimulus_presentation_onset': [0.01, 0.02, 0.03]
+    }, index=pd.Index(name='spike_time', data=[1.01, 1.02, 1.03]))
+    
+    fig = vis.raster_plot(spike_times)
+    ax = fig.get_axes()[0]
+
+    assert len(spike_times['unit_id'].unique()) == len(ax.collections)

From dbba024bf7c309ae0f77a022151318a5fcfb8932 Mon Sep 17 00:00:00 2001
From: Dan <danielk@alleninstitute.org>
Date: Fri, 9 Oct 2020 11:54:04 -0700
Subject: [PATCH 03/56] begins new release candidate branch

---
 allensdk/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/allensdk/__init__.py b/allensdk/__init__.py
index de197af35..b31809cd0 100644
--- a/allensdk/__init__.py
+++ b/allensdk/__init__.py
@@ -37,7 +37,7 @@
 
 
 
-__version__ = '2.3.0'
+__version__ = '2.4.0'
 
 
 try:

From 3db7969312f7babb8345aedbec140557204cdfd3 Mon Sep 17 00:00:00 2001
From: Kat Schelonka <kschelon@gmail.com>
Date: Fri, 9 Oct 2020 14:16:20 -0700
Subject: [PATCH 04/56] GH-1483: Allow null join IDs for some tables in
 BehaviorProjectLimsApi

---
 .../behavior/behavior_project_lims_api.py     | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index bcbb2da86..2a7ea83bf 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -211,13 +211,13 @@ def _get_behavior_summary_table(self,
             FROM behavior_sessions bs
             JOIN donors d on bs.donor_id = d.id
             JOIN genders g on g.id = d.gender_id
-            JOIN (
+            LEFT OUTER JOIN (
                 {self._build_line_from_donor_query("reporter")}
             ) reporter on reporter.donor_id = d.id
-            JOIN (
+            LEFT OUTER JOIN (
                 {self._build_line_from_donor_query("driver")}
             ) driver on driver.donor_id = d.id
-            JOIN equipment ON equipment.id = bs.equipment_id
+            LEFT OUTER JOIN equipment ON equipment.id = bs.equipment_id
             {session_sub_query}
         """
         self.logger.debug(f"get_behavior_session_table query: \n{query}")
@@ -328,19 +328,19 @@ def _get_experiment_table(
                 ON oec.visual_behavior_experiment_container_id = vbc.id
             JOIN ophys_experiments oe ON oe.id = oec.ophys_experiment_id
             JOIN ophys_sessions os ON os.id = oe.ophys_session_id
-            JOIN behavior_sessions bs ON os.id = bs.ophys_session_id
-            JOIN projects pr ON pr.id = os.project_id
+            LEFT OUTER JOIN behavior_sessions bs ON os.id = bs.ophys_session_id
+            LEFT OUTER JOIN projects pr ON pr.id = os.project_id
             JOIN donors d ON d.id = bs.donor_id
             JOIN genders g ON g.id = d.gender_id
-            JOIN (
+            LEFT OUTER JOIN (
                 {self._build_line_from_donor_query(line="reporter")}
             ) reporter on reporter.donor_id = d.id
-            JOIN (
+            LEFT OUTER JOIN (
                 {self._build_line_from_donor_query(line="driver")}
             ) driver on driver.donor_id = d.id
             LEFT JOIN imaging_depths id ON id.id = oe.imaging_depth_id
             JOIN structures st ON st.id = oe.targeted_structure_id
-            JOIN equipment ON equipment.id = os.equipment_id
+            LEFT OUTER JOIN equipment ON equipment.id = os.equipment_id
             {experiment_query};
         """
         self.logger.debug(f"get_experiment_table query: \n{query}")
@@ -384,20 +384,20 @@ def _get_session_table(
                 reporter.reporter_line,
                 driver.driver_line
             FROM ophys_sessions os
-            JOIN behavior_sessions bs ON os.id = bs.ophys_session_id
-            JOIN projects pr ON pr.id = os.project_id
+            LEFT OUTER JOIN behavior_sessions bs ON os.id = bs.ophys_session_id
+            LEFT OUTER JOIN projects pr ON pr.id = os.project_id
             JOIN donors d ON d.id = bs.donor_id
             JOIN genders g ON g.id = d.gender_id
             JOIN (
                 {self._build_experiment_from_session_query()}
             ) exp_ids ON os.id = exp_ids.id
-            JOIN (
+            LEFT OUTER JOIN (
                 {self._build_line_from_donor_query(line="reporter")}
             ) reporter on reporter.donor_id = d.id
-            JOIN (
+            LEFT OUTER JOIN (
                 {self._build_line_from_donor_query(line="driver")}
             ) driver on driver.donor_id = d.id
-            JOIN equipment ON equipment.id = os.equipment_id
+            LEFT OUTER JOIN equipment ON equipment.id = os.equipment_id
             {session_query};
         """
         self.logger.debug(f"get_session_table query: \n{query}")

From 4385d41d3f836ea98d167c57c61a720afbfc759d Mon Sep 17 00:00:00 2001
From: Kat Schelonka <kschelon@gmail.com>
Date: Fri, 9 Oct 2020 14:48:17 -0700
Subject: [PATCH 05/56] Bypass cache for LIMS data by default

---
 .../behavior/behavior_project_cache.py        | 91 +++++++++++--------
 1 file changed, 52 insertions(+), 39 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache.py
index cbfa3b7c3..55af32852 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_cache.py
@@ -92,7 +92,10 @@ def __init__(
         cache : bool
             Whether to write to the cache. Default=True.
         """
-        manifest_ = manifest or "behavior_project_manifest.json"
+        if cache:
+            manifest_ = manifest or "behavior_project_manifest.json"
+        else:
+            manifest_ = None
         version_ = version or self.MANIFEST_VERSION
 
         super().__init__(manifest=manifest_, version=version_, cache=cache)
@@ -103,7 +106,7 @@ def __init__(
     @classmethod
     def from_lims(cls, manifest: Optional[Union[str, Path]] = None,
                   version: Optional[str] = None,
-                  cache: bool = True,
+                  cache: bool = False,
                   fetch_tries: int = 2,
                   lims_credentials: Optional[DbCredentials] = None,
                   mtrain_credentials: Optional[DbCredentials] = None,
@@ -174,20 +177,23 @@ def get_session_table(
         :type by: str
         :rtype: pd.DataFrame
         """
-        write_csv = partial(
-            _write_csv,
-            array_fields=["reporter_line", "driver_line",
-                          "ophys_experiment_id"])
-        read_csv = partial(
-            _read_csv, index_col="ophys_session_id",
-            array_fields=["reporter_line", "driver_line",
-                          "ophys_experiment_id"],
-            array_types=[str, str, int])
-        path = self.get_cache_path(None, self.OPHYS_SESSIONS_KEY)
-        sessions = one_file_call_caching(
-            path,
-            self.fetch_api.get_session_table,
-            write_csv, read_csv)
+        if self.cache:
+            write_csv = partial(
+                _write_csv,
+                array_fields=["reporter_line", "driver_line",
+                              "ophys_experiment_id"])
+            read_csv = partial(
+                _read_csv, index_col="ophys_session_id",
+                array_fields=["reporter_line", "driver_line",
+                              "ophys_experiment_id"],
+                array_types=[str, str, int])
+            path = self.get_cache_path(None, self.OPHYS_SESSIONS_KEY)
+            sessions = one_file_call_caching(
+                path,
+                self.fetch_api.get_session_table,
+                write_csv, read_csv)
+        else:
+            sessions = self.fetch_api.get_session_table()
         if suppress:
             sessions.drop(columns=suppress, inplace=True, errors="ignore")
 
@@ -221,18 +227,21 @@ def get_experiment_table(
         :type suppress: list of str
         :rtype: pd.DataFrame
         """
-        write_csv = partial(
-            _write_csv,
-            array_fields=["reporter_line", "driver_line"])
-        read_csv = partial(
-            _read_csv, index_col="ophys_experiment_id",
-            array_fields=["reporter_line", "driver_line"],
-            array_types=[str, str])
-        path = self.get_cache_path(None, self.OPHYS_EXPERIMENTS_KEY)
-        experiments = one_file_call_caching(
-            path,
-            self.fetch_api.get_experiment_table,
-            write_csv, read_csv)
+        if self.cache:
+            write_csv = partial(
+                _write_csv,
+                array_fields=["reporter_line", "driver_line"])
+            read_csv = partial(
+                _read_csv, index_col="ophys_experiment_id",
+                array_fields=["reporter_line", "driver_line"],
+                array_types=[str, str])
+            path = self.get_cache_path(None, self.OPHYS_EXPERIMENTS_KEY)
+            experiments = one_file_call_caching(
+                path,
+                self.fetch_api.get_experiment_table,
+                write_csv, read_csv)
+        else:
+            experiments = self.fetch_api.get_experiment_table()
         if suppress:
             experiments.drop(columns=suppress, inplace=True, errors="ignore")
         return experiments
@@ -247,17 +256,21 @@ def get_behavior_session_table(
         :type suppress: list of str
         :rtype: pd.DataFrame
         """
-        read_csv = partial(
-            _read_csv, index_col="behavior_session_id",
-            array_fields=["reporter_line", "driver_line"],
-            array_types=[str, str])
-        write_csv = partial(
-            _write_csv, array_fields=["reporter_line", "driver_line"])
-        path = self.get_cache_path(None, self.BEHAVIOR_SESSIONS_KEY)
-        sessions = one_file_call_caching(
-            path,
-            self.fetch_api.get_behavior_only_session_table,
-            write_csv, read_csv)
+
+        if self.cache:
+            read_csv = partial(
+                _read_csv, index_col="behavior_session_id",
+                array_fields=["reporter_line", "driver_line"],
+                array_types=[str, str])
+            write_csv = partial(
+                _write_csv, array_fields=["reporter_line", "driver_line"])
+            path = self.get_cache_path(None, self.BEHAVIOR_SESSIONS_KEY)
+            sessions = one_file_call_caching(
+                path,
+                self.fetch_api.get_behavior_only_session_table,
+                write_csv, read_csv)
+        else:
+            sessions = self.fetch_api.get_behavior_only_session_table()
         sessions = sessions.rename(columns={"genotype": "full_genotype"})
         if suppress:
             sessions.drop(columns=suppress, inplace=True, errors="ignore")

From 317c52abfe232632d7305a3b46d1309ec3ac8f86 Mon Sep 17 00:00:00 2001
From: Kat Schelonka <kschelon@gmail.com>
Date: Tue, 13 Oct 2020 12:21:06 -0700
Subject: [PATCH 06/56] Write to json instead of csv to make serialization a
 little easier

---
 allensdk/api/caching_utilities.py             |  4 ++
 .../behavior/behavior_project_cache.py        | 70 +++++++------------
 .../behavior/test_behavior_project_cache.py   | 23 ++++--
 3 files changed, 47 insertions(+), 50 deletions(-)

diff --git a/allensdk/api/caching_utilities.py b/allensdk/api/caching_utilities.py
index ab5de267e..a45729451 100644
--- a/allensdk/api/caching_utilities.py
+++ b/allensdk/api/caching_utilities.py
@@ -105,6 +105,10 @@ def call_caching(
     except Exception as e:
         if isinstance(e, FileNotFoundError):
             logger.info("No cache file found.")
+        # Pandas throws ValueError rather than FileNotFoundError
+        elif (isinstance(e, ValueError)
+              and str(e) == "Expected object or value"):
+            logger.info("No cache file found.")
         if cleanup is not None and not lazy:
             cleanup()
 
diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache.py
index 55af32852..836caf25d 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_cache.py
@@ -1,4 +1,3 @@
-import numpy as np
 from functools import partial
 from typing import Type, Optional, List, Union
 from pathlib import Path
@@ -178,20 +177,12 @@ def get_session_table(
         :rtype: pd.DataFrame
         """
         if self.cache:
-            write_csv = partial(
-                _write_csv,
-                array_fields=["reporter_line", "driver_line",
-                              "ophys_experiment_id"])
-            read_csv = partial(
-                _read_csv, index_col="ophys_session_id",
-                array_fields=["reporter_line", "driver_line",
-                              "ophys_experiment_id"],
-                array_types=[str, str, int])
             path = self.get_cache_path(None, self.OPHYS_SESSIONS_KEY)
             sessions = one_file_call_caching(
                 path,
                 self.fetch_api.get_session_table,
-                write_csv, read_csv)
+                _write_json, _read_json)
+            sessions.set_index("ophys_session_id")
         else:
             sessions = self.fetch_api.get_session_table()
         if suppress:
@@ -228,18 +219,12 @@ def get_experiment_table(
         :rtype: pd.DataFrame
         """
         if self.cache:
-            write_csv = partial(
-                _write_csv,
-                array_fields=["reporter_line", "driver_line"])
-            read_csv = partial(
-                _read_csv, index_col="ophys_experiment_id",
-                array_fields=["reporter_line", "driver_line"],
-                array_types=[str, str])
             path = self.get_cache_path(None, self.OPHYS_EXPERIMENTS_KEY)
             experiments = one_file_call_caching(
                 path,
                 self.fetch_api.get_experiment_table,
-                write_csv, read_csv)
+                _write_json, _read_json)
+            experiments.set_index("ophys_experiment_id")
         else:
             experiments = self.fetch_api.get_experiment_table()
         if suppress:
@@ -258,17 +243,12 @@ def get_behavior_session_table(
         """
 
         if self.cache:
-            read_csv = partial(
-                _read_csv, index_col="behavior_session_id",
-                array_fields=["reporter_line", "driver_line"],
-                array_types=[str, str])
-            write_csv = partial(
-                _write_csv, array_fields=["reporter_line", "driver_line"])
             path = self.get_cache_path(None, self.BEHAVIOR_SESSIONS_KEY)
             sessions = one_file_call_caching(
                 path,
                 self.fetch_api.get_behavior_only_session_table,
-                write_csv, read_csv)
+                _write_json, _read_json)
+            sessions.set_index("behavior_session_id")
         else:
             sessions = self.fetch_api.get_behavior_only_session_table()
         sessions = sessions.rename(columns={"genotype": "full_genotype"})
@@ -317,24 +297,26 @@ def get_behavior_session_data(self, behavior_session_id: int,
         )
 
 
-def _write_csv(path, df, array_fields=None):
-    """Private writer that encodes array fields into pipe-delimited strings
-    for saving a csv.
-    """
-    df_ = df.copy()
-    for field in array_fields:
-        df_[field] = df_[field].apply(lambda x: "|".join(map(str, x)))
-    df_.to_csv(path)
+def _write_json(path, df):
+    """Wrapper to change the arguments for saving a pandas json
+    dataframe so that it conforms to expectations of the internal
+    cache methods. Can't use partial with the native `to_json` method
+    because the dataframe is not yet created at the time we need to
+    pass in the save method.
+    Saves a dataframe in json format to `path`, in split orientation
+    to save space on disk.
+    Converts dates to seconds from epoch.
+    NOTE: Date serialization is a big pain. Make sure if columns
+    are being added, the _read_json is updated to properly deserialize
+    them back to the expected format by adding them to `convert_dates`.
+    In the future we could schematize this data using marshmallow
+    or something similar."""
+    df.reset_index(inplace=True)
+    df.to_json(path, orient="split", date_unit="s", date_format="epoch")
 
 
-def _read_csv(path, index_col, array_fields=None, array_types=None):
-    """Private reader that can open a csv with pipe-delimited array
-    fields and convert them to array."""
-    df = pd.read_csv(path, index_col=index_col)
-    for field, type_ in zip(array_fields, array_types):
-        if type_ == str:
-            df[field] = df[field].apply(lambda x: x.split("|"))
-        else:
-            df[field] = df[field].apply(
-                lambda x: np.fromstring(x, sep="|", dtype=type_))
+def _read_json(path):
+    """Reads a dataframe file written to the cache by _write_json."""
+    df = pd.read_json(path, date_unit="s", orient="split",
+                      convert_dates=["date_of_acquisition"])
     return df
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_project_cache.py b/allensdk/test/brain_observatory/behavior/test_behavior_project_cache.py
index 1a9123f54..83cf06a5c 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_project_cache.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_project_cache.py
@@ -1,4 +1,5 @@
 import os
+import numpy as np
 import pytest
 import pandas as pd
 import tempfile
@@ -11,6 +12,7 @@
 def session_table():
     return (pd.DataFrame({"ophys_session_id": [1, 2, 3],
                           "ophys_experiment_id": [[4], [5, 6], [7]],
+                          "date_of_acquisition": np.datetime64('2020-02-20'),
                           "reporter_line": [["aa"], ["aa", "bb"], ["cc"]],
                           "driver_line": [["aa"], ["aa", "bb"], ["cc"]]})
             .set_index("ophys_session_id"))
@@ -19,6 +21,7 @@ def session_table():
 @pytest.fixture
 def behavior_table():
     return (pd.DataFrame({"behavior_session_id": [1, 2, 3],
+                          "date_of_acquisition": np.datetime64("NAT"),
                           "reporter_line": [["aa"], ["aa", "bb"], ["cc"]],
                           "driver_line": [["aa"], ["aa", "bb"], ["cc"]]})
             .set_index("behavior_session_id"))
@@ -42,30 +45,36 @@ def get_behavior_only_session_data(self, behavior_session_id):
 
 
 @pytest.fixture
-def TempdirBehaviorCache(mock_api):
+def TempdirBehaviorCache(mock_api, request):
     temp_dir = tempfile.TemporaryDirectory()
     manifest = os.path.join(temp_dir.name, "manifest.json")
     yield BehaviorProjectCache(fetch_api=mock_api(),
+                               cache=request.param,
                                manifest=manifest)
     temp_dir.cleanup()
 
 
+@pytest.mark.parametrize("TempdirBehaviorCache", [True, False], indirect=True)
 def test_get_session_table(TempdirBehaviorCache, session_table):
     cache = TempdirBehaviorCache
     actual = cache.get_session_table()
-    path = cache.manifest.path_info.get("ophys_sessions").get("spec")
-    assert os.path.exists(path)
+    if cache.cache:
+        path = cache.manifest.path_info.get("ophys_sessions").get("spec")
+        assert os.path.exists(path)
     pd.testing.assert_frame_equal(session_table, actual)
 
 
+@pytest.mark.parametrize("TempdirBehaviorCache", [True, False], indirect=True)
 def test_get_behavior_table(TempdirBehaviorCache, behavior_table):
     cache = TempdirBehaviorCache
     actual = cache.get_behavior_session_table()
-    path = cache.manifest.path_info.get("behavior_sessions").get("spec")
-    assert os.path.exists(path)
+    if cache.cache:
+        path = cache.manifest.path_info.get("behavior_sessions").get("spec")
+        assert os.path.exists(path)
     pd.testing.assert_frame_equal(behavior_table, actual)
 
 
+@pytest.mark.parametrize("TempdirBehaviorCache", [True], indirect=True)
 def test_session_table_reads_from_cache(TempdirBehaviorCache, session_table,
                                         caplog):
     caplog.set_level(logging.INFO, logger="call_caching")
@@ -83,6 +92,7 @@ def test_session_table_reads_from_cache(TempdirBehaviorCache, session_table,
     assert [expected_first[0]] == caplog.record_tuples
 
 
+@pytest.mark.parametrize("TempdirBehaviorCache", [True], indirect=True)
 def test_behavior_table_reads_from_cache(TempdirBehaviorCache, behavior_table,
                                          caplog):
     caplog.set_level(logging.INFO, logger="call_caching")
@@ -100,10 +110,11 @@ def test_behavior_table_reads_from_cache(TempdirBehaviorCache, behavior_table,
     assert [expected_first[0]] == caplog.record_tuples
 
 
+@pytest.mark.parametrize("TempdirBehaviorCache", [True, False], indirect=True)
 def test_get_session_table_by_experiment(TempdirBehaviorCache):
     expected = (pd.DataFrame({"ophys_session_id": [1, 2, 2, 3],
                               "ophys_experiment_id": [4, 5, 6, 7]})
                 .set_index("ophys_experiment_id"))
     actual = TempdirBehaviorCache.get_session_table(by="ophys_experiment_id")[
         ["ophys_session_id"]]
-    pd.testing.assert_frame_equal(expected, actual)
\ No newline at end of file
+    pd.testing.assert_frame_equal(expected, actual)

From c8e3a4dc33b24b1b0dcb18c25bfd98b7fdf98fbb Mon Sep 17 00:00:00 2001
From: Kat Schelonka <kschelon@gmail.com>
Date: Tue, 13 Oct 2020 14:36:02 -0700
Subject: [PATCH 07/56] Adds donor id to ophys summary tables

---
 allensdk/brain_observatory/behavior/behavior_project_cache.py   | 2 +-
 .../brain_observatory/behavior/behavior_project_lims_api.py     | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache.py
index 836caf25d..6a90999c3 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_cache.py
@@ -18,7 +18,7 @@
 
 class BehaviorProjectCache(Cache):
 
-    MANIFEST_VERSION = "0.0.1-alpha.2"
+    MANIFEST_VERSION = "0.0.1-alpha.3"
     OPHYS_SESSIONS_KEY = "ophys_sessions"
     BEHAVIOR_SESSIONS_KEY = "behavior_sessions"
     OPHYS_EXPERIMENTS_KEY = "ophys_experiments"
diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index 2a7ea83bf..5fe7f0f96 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -314,6 +314,7 @@ def _get_experiment_table(
                 os.date_of_acquisition,
                 os.isi_experiment_id,
                 os.specimen_id,
+                d.id as donor_id,
                 g.name as sex,
                 DATE_PART('day', os.date_of_acquisition - d.date_of_birth)
                     AS age_in_days,
@@ -377,6 +378,7 @@ def _get_session_table(
                 equipment.name as equipment_name,
                 os.date_of_acquisition,
                 os.specimen_id,
+                d.id as donor_id,
                 g.name as sex,
                 DATE_PART('day', os.date_of_acquisition - d.date_of_birth)
                     AS age_in_days,

From 21197b95f1d996572feb92242c7643bb8542f027 Mon Sep 17 00:00:00 2001
From: Dan <danielk@alleninstitute.org>
Date: Fri, 9 Oct 2020 11:54:04 -0700
Subject: [PATCH 08/56] begins new release candidate branch

---
 allensdk/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/allensdk/__init__.py b/allensdk/__init__.py
index 3a0977cbb..b31809cd0 100644
--- a/allensdk/__init__.py
+++ b/allensdk/__init__.py
@@ -37,7 +37,7 @@
 
 
 
-__version__ = '2.3.1'
+__version__ = '2.4.0'
 
 
 try:

From c747440fa7428a1ae6f6ec82b4f9da5ac5ad36c4 Mon Sep 17 00:00:00 2001
From: Dan <danielk@alleninstitute.org>
Date: Thu, 15 Oct 2020 10:13:14 -0700
Subject: [PATCH 09/56] updates dependency to latest argschema

---
 requirements.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 32ec2a495..3da662766 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,8 +15,7 @@ scikit-image>=0.14.0,<0.17.0
 scikit-build<1.0.0
 statsmodels==0.9.0
 simpleitk<2.0.0
-argschema<2.0.0
-marshmallow==3.0.0rc6
+argschema==2.0.1
 glymur==0.8.19
 xarray<0.16.0
 pynwb>=1.3.2,<2.0.0

From e361c6826fe51e7a5003bb24103f69f2d2c4d377 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 12 Oct 2020 13:32:08 -0700
Subject: [PATCH 10/56] Remove extraneous fields from custom behavior ophys
 subject extension

This commit removes fields that already exist as part of the pyNWB
"Subject" class from the custom behavior ophys subject extension.
---
 .../brain_observatory/behavior/schemas.py     |   3 +
 allensdk/brain_observatory/nwb/metadata.py    |   9 ++
 .../ndx-aibs-behavior-ophys.extension.yaml    | 104 ++++++++----------
 3 files changed, 58 insertions(+), 58 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/schemas.py b/allensdk/brain_observatory/behavior/schemas.py
index a57e92aa7..e1d438ac6 100644
--- a/allensdk/brain_observatory/behavior/schemas.py
+++ b/allensdk/brain_observatory/behavior/schemas.py
@@ -22,6 +22,9 @@ class SubjectMetadataSchema(RaisingSchema):
     neurodata_type = 'BehaviorSubject'
     neurodata_type_inc = 'Subject'
     neurodata_doc = "Metadata for an AIBS behavior or behavior + ophys subject"
+    # Fields to skip converting to extension
+    # In this case they already exist in the 'Subject' builtin pyNWB class
+    neurodata_skip = {"age", "genotype", "sex", "subject_id"}
 
     age = fields.String(
         doc='Age of the specimen donor/subject',
diff --git a/allensdk/brain_observatory/nwb/metadata.py b/allensdk/brain_observatory/nwb/metadata.py
index c646238eb..5bd52c5e4 100644
--- a/allensdk/brain_observatory/nwb/metadata.py
+++ b/allensdk/brain_observatory/nwb/metadata.py
@@ -9,11 +9,20 @@
 
 def extract_from_schema(schema):
 
+    if hasattr(schema, 'neurodata_skip'):
+        fields_to_skip = schema.neurodata_skip
+    else:
+        fields_to_skip = set()
+
     # Extract fields from Schema:
     docval_list = [{'name': 'name', 'type': str, 'doc': 'name'}]
     attributes = []
     nwbfields_list = []
     for name, val in schema().fields.items():
+
+        if name in fields_to_skip:
+            continue
+
         if type(val) == fields.List:
             attributes.append(NWBAttributeSpec(name=name,
                                                dtype=STYPE_DICT[type(val)],
diff --git a/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml b/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml
index 976c2fcd6..a5dd556d3 100644
--- a/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml
+++ b/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml
@@ -3,112 +3,100 @@ groups:
   neurodata_type_inc: LabMetaData
   doc: Metadata for behavior or behavior + ophys task parameters
   attributes:
-  - name: omitted_flash_fraction
-    dtype: float
-    doc: omitted_flash_fraction
+  - name: task
+    dtype: text
+    doc: task
   - name: stimulus_duration_sec
     dtype: float
     doc: duration of each stimulus presentation in seconds
-  - name: reward_volume
-    dtype: float
-    doc: reward_volume
+  - name: stage
+    dtype: text
+    doc: stage
+  - name: n_stimulus_frames
+    dtype: int
+    doc: n_stimulus_frames
   - name: response_window_sec
     dtype: text
     shape:
     - 2
     doc: response_window in seconds
-  - name: blank_duration_sec
-    dtype: text
-    shape:
-    - 2
-    doc: blank duration in seconds
-  - name: stage
-    dtype: text
-    doc: stage
   - name: stimulus_distribution
     dtype: text
     doc: stimulus_distribution
-  - name: n_stimulus_frames
-    dtype: int
-    doc: n_stimulus_frames
-  - name: task
-    dtype: text
-    doc: task
   - name: stimulus
     dtype: text
     doc: stimulus
+  - name: blank_duration_sec
+    dtype: text
+    shape:
+    - 2
+    doc: blank duration in seconds
+  - name: omitted_flash_fraction
+    dtype: float
+    doc: omitted_flash_fraction
+  - name: reward_volume
+    dtype: float
+    doc: reward_volume
 - neurodata_type_def: BehaviorSubject
   neurodata_type_inc: Subject
   doc: Metadata for an AIBS behavior or behavior + ophys subject
   attributes:
-  - name: genotype
-    dtype: text
-    doc: full genotype of subject
   - name: reporter_line
     dtype: text
     shape:
     - null
     doc: Reporter line of subject
-  - name: sex
-    dtype: text
-    doc: Sex of the specimen donor/subject
   - name: driver_line
     dtype: text
     shape:
     - null
     doc: Driver line of subject
-  - name: age
-    dtype: text
-    doc: Age of the specimen donor/subject
-  - name: subject_id
-    dtype: int
-    doc: LabTracks ID of subject
 - neurodata_type_def: OphysBehaviorMetadata
   neurodata_type_inc: LabMetaData
   doc: Metadata for behavior + ophys experiments
   attributes:
-  - name: rig_name
+  - name: session_type
     dtype: text
-    doc: name of two-photon rig
+    doc: Experimental session description
   - name: stimulus_frame_rate
     dtype: float
     doc: Frame rate (frames/second) of the visual_stimulus from the monitor
-  - name: emission_lambda
+  - name: excitation_lambda
     dtype: float
-    doc: emission_lambda
+    doc: excitation_lambda
+  - name: imaging_depth
+    dtype: int
+    doc: Depth (microns) below the cortical surface targeted for two-photon acquisition
+  - name: ophys_frame_rate
+    dtype: float
+    doc: Frame rate (frames/second) of the two-photon microscope
   - name: experiment_container_id
     dtype: int
     doc: Container ID for the container that contains this ophys session
-  - name: field_of_view_height
-    dtype: int
-    doc: field_of_view_height
-  - name: excitation_lambda
-    dtype: float
-    doc: excitation_lambda
-  - name: indicator
-    dtype: text
-    doc: indicator
-  - name: behavior_session_uuid
-    dtype: text
-    doc: MTrain record for session, also called foraging_id
-  - name: session_type
-    dtype: text
-    doc: Experimental session description
   - name: field_of_view_width
     dtype: int
     doc: field_of_view_width
-  - name: targeted_structure
-    dtype: text
-    doc: Anatomical structure targeted for two-photon acquisition
   - name: ophys_experiment_id
     dtype: int
     doc: Id for this ophys session
   - name: experiment_datetime
     dtype: text
     doc: Date of the experiment (UTC, as string)
-  - name: imaging_depth
+  - name: field_of_view_height
     dtype: int
-    doc: Depth (microns) below the cortical surface targeted for two-photon acquisition
-  - name: ophys_frame_rate
+    doc: field_of_view_height
+  - name: emission_lambda
     dtype: float
-    doc: Frame rate (frames/second) of the two-photon microscope
+    doc: emission_lambda
+  - name: targeted_structure
+    dtype: text
+    doc: Anatomical structure targeted for two-photon acquisition
+  - name: behavior_session_uuid
+    dtype: text
+    doc: MTrain record for session, also called foraging_id
+  - name: indicator
+    dtype: text
+    doc: indicator
+  - name: rig_name
+    dtype: text
+    doc: name of two-photon rig

From 59c3711d7e276e5dfc2b664849385670cb76f680 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 12 Oct 2020 16:38:25 -0700
Subject: [PATCH 11/56] Store dff and fluorescence traces as timepoints x ROIs

Previously dff and fluorescence traces were being stored as
ROIs x timepoints Numpy arrays. This commit implements pyNWB
best practice by storing the data with the time dimension
first.
---
 .../behavior_ophys_nwb_api.py                  | 18 +++++++++++++-----
 allensdk/brain_observatory/nwb/__init__.py     | 14 ++++++++++----
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
index a27e909ae..3c6d50a34 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
@@ -207,20 +207,28 @@ def get_cell_specimen_table(self) -> pd.DataFrame:
 
     def get_dff_traces(self) -> pd.DataFrame:
         dff_nwb = self.nwbfile.modules['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces']
-        dff_traces = dff_nwb.data[:]
+        # dff traces stored as timepoints x rois in NWB
+        # We want rois x timepoints, hence the transpose
+        dff_traces = dff_nwb.data[:].T
         number_of_cells, number_of_dff_frames = dff_traces.shape
         num_of_timestamps = len(self.get_ophys_timestamps())
         assert num_of_timestamps == number_of_dff_frames
-        
-        df = pd.DataFrame({'dff': [x for x in dff_traces]}, index=pd.Index(data=dff_nwb.rois.table.id[:], name='cell_roi_id'))
+
+        df = pd.DataFrame({'dff': dff_traces.tolist()},
+                          index=pd.Index(data=dff_nwb.rois.table.id[:],
+                                         name='cell_roi_id'))
         cell_specimen_table = self.get_cell_specimen_table()
         df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id')
         return df
 
     def get_corrected_fluorescence_traces(self) -> pd.DataFrame:
         corrected_fluorescence_nwb = self.nwbfile.modules['two_photon_imaging'].data_interfaces['corrected_fluorescence'].roi_response_series['traces']
-        df = pd.DataFrame({'corrected_fluorescence': [x for x in corrected_fluorescence_nwb.data[:]]},
-                             index=pd.Index(data=corrected_fluorescence_nwb.rois.table.id[:], name='cell_roi_id'))
+        # f traces stored as timepoints x rois in NWB
+        # We want rois x timepoints, hence the transpose
+        f_traces = corrected_fluorescence_nwb.data[:].T
+        df = pd.DataFrame({'corrected_fluorescence': f_traces.tolist()},
+                          index=pd.Index(data=corrected_fluorescence_nwb.rois.table.id[:],
+                                         name='cell_roi_id'))
 
         cell_specimen_table = self.get_cell_specimen_table()
         df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id')
diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py
index 727508100..d3d787509 100644
--- a/allensdk/brain_observatory/nwb/__init__.py
+++ b/allensdk/brain_observatory/nwb/__init__.py
@@ -903,8 +903,9 @@ def add_dff_traces(nwbfile, dff_traces, ophys_timestamps):
     dff_traces = dff_traces.reset_index().set_index('cell_roi_id')[['dff']]
 
     twop_module = nwbfile.modules['two_photon_imaging']
-    data = np.array([dff_traces.loc[cell_roi_id].dff for cell_roi_id in dff_traces.index.values])
-    # assert len(ophys_timestamps.timestamps) == len(data)
+    # trace data in the form of rois x timepoints
+    trace_data = np.array([dff_traces.loc[cell_roi_id].dff
+                           for cell_roi_id in dff_traces.index.values])
 
     cell_specimen_table = nwbfile.modules['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table']
     roi_table_region = cell_specimen_table.create_roi_table_region(
@@ -918,7 +919,7 @@ def add_dff_traces(nwbfile, dff_traces, ophys_timestamps):
 
     dff_interface.create_roi_response_series(
         name='traces',
-        data=data,
+        data=trace_data.T,  # Should be stored as timepoints x rois
         unit='NA',
         rois=roi_table_region,
         timestamps=ophys_timestamps)
@@ -932,13 +933,18 @@ def add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces):
     # Create/Add corrected_fluorescence_traces modules and interfaces:
     assert corrected_fluorescence_traces.index.name == 'cell_roi_id'
     twop_module = nwbfile.modules['two_photon_imaging']
+    # trace data in the form of rois x timepoints
+    f_trace_data = np.array([corrected_fluorescence_traces.loc[cell_roi_id].corrected_fluorescence
+                             for cell_roi_id in corrected_fluorescence_traces.index.values])
+
     roi_table_region = nwbfile.modules['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces'].rois
     ophys_timestamps = twop_module.get_data_interface('dff').roi_response_series['traces'].timestamps
     f_interface = Fluorescence(name='corrected_fluorescence')
     twop_module.add_data_interface(f_interface)
+
     f_interface.create_roi_response_series(
         name='traces',
-        data=np.array([corrected_fluorescence_traces.loc[cell_roi_id].corrected_fluorescence for cell_roi_id in corrected_fluorescence_traces.index.values]),
+        data=f_trace_data.T,  # Should be stored as timepoints x rois
         unit='NA',
         rois=roi_table_region,
         timestamps=ophys_timestamps)

From 1b1ccb2f84a32f9171e37087984a42da3d8edeeb Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Thu, 15 Oct 2020 16:27:51 -0700
Subject: [PATCH 12/56] Address pyNWB best practice recommendations for
 behavior ophys NWB

This is a combination of the following 3 commits:

1) Simplify custom behavior ophys pyNWB extension

   Previously, the OphysBehaviorMetadata extension included a number
   of fields that already exist in base pyNWB classes. This commit
   moves "emission_lambda", "excitation_lambda", "indicator",
   "location", "session_start_time", and "imaging_rate" fields so
   that they are stored in pyNWB classes like ImagingPlane and
   OpticalChannel. This greatly simplifies the OphysBehaviorMetadata
   extension definition.

2) Add more detailed "doc" description to pyNWB extension fields

3) Address pyNWB NWBFile.modules deprecation warning

   Trying to access pyNWB processing modules via the "NWBFile.modules"
   attribute is now deprecated in favor of "NWBFile.processing".
   This commit implements the favored method for accessing
   pyNWB processing modules from an NWBFile object.
---
 .../behavior_ophys_nwb_api.py                 | 104 +++++++++++-------
 .../brain_observatory/behavior/schemas.py     |  91 +++++++++------
 allensdk/brain_observatory/nwb/__init__.py    |  71 ++++++------
 .../ndx-aibs-behavior-ophys.extension.yaml    | 100 +++++++----------
 .../brain_observatory/behavior/conftest.py    |  25 ++++-
 .../behavior/test_write_nwb_behavior_ophys.py |  53 ++++++---
 6 files changed, 264 insertions(+), 180 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
index 3c6d50a34..db4b86e37 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
@@ -1,27 +1,27 @@
 import datetime
-from pynwb import NWBFile, NWBHDF5IO
-import pandas as pd
-import allensdk.brain_observatory.nwb as nwb
-import numpy as np
-import SimpleITK as sitk
-import pytz
-import uuid
-from pandas.util.testing import assert_frame_equal
-import os
 import math
+import uuid
+import warnings
+
 import numpy as np
-import xarray as xr
 import pandas as pd
+import pytz
+import SimpleITK as sitk
+import xarray as xr
 
-
-from allensdk.core.lazy_property import LazyProperty
+import allensdk.brain_observatory.nwb as nwb
+from allensdk.brain_observatory.behavior.behavior_ophys_api import \
+    BehaviorOphysApiBase
+from allensdk.brain_observatory.behavior.schemas import (
+    BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema)
+from allensdk.brain_observatory.behavior.trials_processing import \
+    TRIAL_COLUMN_DESCRIPTION_DICT
+from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension
 from allensdk.brain_observatory.nwb.nwb_api import NwbApi
 from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time
-from allensdk.brain_observatory.behavior.trials_processing import TRIAL_COLUMN_DESCRIPTION_DICT
-from allensdk.brain_observatory.behavior.schemas import OphysBehaviorMetadataSchema, BehaviorTaskParametersSchema
-from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension
-from allensdk.brain_observatory.behavior.behavior_ophys_api import BehaviorOphysApiBase
-
+from allensdk.core.lazy_property import LazyProperty
+from pandas.util.testing import assert_frame_equal
+from pynwb import NWBHDF5IO, NWBFile
 
 load_pynwb_extension(OphysBehaviorMetadataSchema, 'ndx-aibs-behavior-ophys')
 load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys')
@@ -29,12 +29,10 @@
 
 class BehaviorOphysNwbApi(NwbApi, BehaviorOphysApiBase):
 
-
     def __init__(self, *args, **kwargs):
         self.filter_invalid_rois = kwargs.pop("filter_invalid_rois", False)
         super(BehaviorOphysNwbApi, self).__init__(*args, **kwargs)
 
-
     def save(self, session_object):
 
         nwbfile = NWBFile(
@@ -93,7 +91,9 @@ def save(self, session_object):
         nwb.add_task_parameters(nwbfile, session_object.task_parameters)
 
         # Add roi metrics to NWB in-memory object:
-        nwb.add_cell_specimen_table(nwbfile, session_object.cell_specimen_table)
+        nwb.add_cell_specimen_table(nwbfile,
+                                    session_object.cell_specimen_table,
+                                    session_object.metadata)
 
         # Add dff to NWB in-memory object:
         nwb.add_dff_traces(nwbfile, session_object.dff_traces, session_object.ophys_timestamps)
@@ -122,8 +122,8 @@ def get_running_data_df(self, **kwargs):
                 running_data_df[key] = self.nwbfile.get_acquisition(key).data
 
         for key in ['dx']:
-            if ('running' in self.nwbfile.modules) and (key in self.nwbfile.modules['running'].fields['data_interfaces']):
-                running_data_df[key] = self.nwbfile.modules['running'].get_data_interface(key).data
+            if ('running' in self.nwbfile.processing) and (key in self.nwbfile.processing['running'].fields['data_interfaces']):
+                running_data_df[key] = self.nwbfile.processing['running'].get_data_interface(key).data
 
         return running_data_df[['speed', 'dx', 'v_sig', 'v_in']]
 
@@ -131,10 +131,10 @@ def get_stimulus_templates(self, **kwargs):
         return {key: val.data[:] for key, val in self.nwbfile.stimulus_template.items()}
 
     def get_ophys_timestamps(self) -> np.ndarray:
-        return self.nwbfile.modules['two_photon_imaging'].get_data_interface('dff').roi_response_series['traces'].timestamps[:]
+        return self.nwbfile.processing['two_photon_imaging'].get_data_interface('dff').roi_response_series['traces'].timestamps[:]
 
     def get_stimulus_timestamps(self) -> np.ndarray:
-        return self.nwbfile.modules['stimulus'].get_data_interface('timestamps').timestamps[:]
+        return self.nwbfile.processing['stimulus'].get_data_interface('timestamps').timestamps[:]
 
     def get_trials(self) -> pd.DataFrame:
         trials = self.nwbfile.trials.to_dataframe()
@@ -144,16 +144,16 @@ def get_trials(self) -> pd.DataFrame:
         return trials
 
     def get_licks(self) -> np.ndarray:
-        if 'licking' in self.nwbfile.modules:
-            return pd.DataFrame({'time': self.nwbfile.modules['licking'].get_data_interface('licks')['timestamps'].timestamps[:]})
+        if 'licking' in self.nwbfile.processing:
+            return pd.DataFrame({'time': self.nwbfile.processing['licking'].get_data_interface('licks')['timestamps'].timestamps[:]})
         else:
             return pd.DataFrame({'time': []})
 
     def get_rewards(self) -> np.ndarray:
-        if 'rewards' in self.nwbfile.modules:
-            time = self.nwbfile.modules['rewards'].get_data_interface('autorewarded').timestamps[:]
-            autorewarded = self.nwbfile.modules['rewards'].get_data_interface('autorewarded').data[:]
-            volume = self.nwbfile.modules['rewards'].get_data_interface('volume').data[:]
+        if 'rewards' in self.nwbfile.processing:
+            time = self.nwbfile.processing['rewards'].get_data_interface('autorewarded').timestamps[:]
+            autorewarded = self.nwbfile.processing['rewards'].get_data_interface('autorewarded').data[:]
+            volume = self.nwbfile.processing['rewards'].get_data_interface('volume').data[:]
             return pd.DataFrame({'volume': volume, 'timestamps': time, 'autorewarded': autorewarded}).set_index('timestamps')
         else:
             return pd.DataFrame({'volume': [], 'timestamps': [], 'autorewarded': []}).set_index('timestamps')
@@ -170,9 +170,10 @@ def get_segmentation_mask_image(self, image_api=None) -> sitk.Image:
     def get_metadata(self) -> dict:
 
         metadata_nwb_obj = self.nwbfile.lab_meta_data['metadata']
-        data = OphysBehaviorMetadataSchema(exclude=['experiment_datetime']).dump(metadata_nwb_obj)
+        data = OphysBehaviorMetadataSchema(
+            exclude=['experiment_datetime']).dump(metadata_nwb_obj)
 
-        # Add subject related metadata to behavior ophys metadata
+        # Add pyNWB Subject metadata to behavior ophys session metadata
         nwb_subject = self.nwbfile.subject
         data['LabTracks_ID'] = int(nwb_subject.subject_id)
         data['sex'] = nwb_subject.sex
@@ -181,9 +182,30 @@ def get_metadata(self) -> dict:
         data['reporter_line'] = list(nwb_subject.reporter_line)
         data['driver_line'] = list(nwb_subject.driver_line)
 
-        experiment_datetime = metadata_nwb_obj.experiment_datetime
-        data['experiment_datetime'] = OphysBehaviorMetadataSchema().load({'experiment_datetime': experiment_datetime}, partial=True)['experiment_datetime']            
-        data['behavior_session_uuid'] = uuid.UUID(data['behavior_session_uuid'])
+        # Add pyNWB OpticalChannel and ImagingPlane metadata to behavior ophys
+        # session metadata
+        try:
+            two_photon_imaging_module = self.nwbfile.processing['two_photon_imaging']
+            image_seg = two_photon_imaging_module.data_interfaces['image_segmentation']
+            imaging_plane = image_seg.plane_segmentations['cell_specimen_table'].imaging_plane
+            optical_channel = imaging_plane.optical_channel[0]
+
+            data['ophys_frame_rate'] = imaging_plane.imaging_rate
+            data['indicator'] = imaging_plane.indicator
+            data['targeted_structure'] = imaging_plane.location
+            data['excitation_lambda'] = imaging_plane.excitation_lambda
+            data['emission_lambda'] = optical_channel.emission_lambda
+        except KeyError:
+            warnings.warn("Could not locate 'two_photon_imaging' module in "
+                          "NWB file. The following metadata fields will be "
+                          "missing: 'ophys_frame_rate', 'indicator', "
+                          "'targeted_structure', 'excitation_lambda', "
+                          "'emission_lambda'")
+
+        # Add other metadata stored in nwb file to behavior ophys session meta
+        data['experiment_datetime'] = self.nwbfile.session_start_time
+        data['behavior_session_uuid'] = uuid.UUID(
+            data['behavior_session_uuid'])
         return data
 
     def get_task_parameters(self) -> dict:
@@ -193,7 +215,7 @@ def get_task_parameters(self) -> dict:
         return data
 
     def get_cell_specimen_table(self) -> pd.DataFrame:
-        df = self.nwbfile.modules['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe()
+        df = self.nwbfile.processing['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe()
         df.index.rename('cell_roi_id', inplace=True)
         df['cell_specimen_id'] = [None if csid == -1 else csid for csid in df['cell_specimen_id'].values]
         df['image_mask'] = [mask.astype(bool) for mask in df['image_mask'].values]
@@ -206,7 +228,7 @@ def get_cell_specimen_table(self) -> pd.DataFrame:
         return df
 
     def get_dff_traces(self) -> pd.DataFrame:
-        dff_nwb = self.nwbfile.modules['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces']
+        dff_nwb = self.nwbfile.processing['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces']
         # dff traces stored as timepoints x rois in NWB
         # We want rois x timepoints, hence the transpose
         dff_traces = dff_nwb.data[:].T
@@ -222,7 +244,7 @@ def get_dff_traces(self) -> pd.DataFrame:
         return df
 
     def get_corrected_fluorescence_traces(self) -> pd.DataFrame:
-        corrected_fluorescence_nwb = self.nwbfile.modules['two_photon_imaging'].data_interfaces['corrected_fluorescence'].roi_response_series['traces']
+        corrected_fluorescence_nwb = self.nwbfile.processing['two_photon_imaging'].data_interfaces['corrected_fluorescence'].roi_response_series['traces']
         # f traces stored as timepoints x rois in NWB
         # We want rois x timepoints, hence the transpose
         f_traces = corrected_fluorescence_nwb.data[:].T
@@ -237,8 +259,8 @@ def get_corrected_fluorescence_traces(self) -> pd.DataFrame:
     def get_motion_correction(self) -> pd.DataFrame:
 
         motion_correction_data = {}
-        motion_correction_data['x'] = self.nwbfile.modules['motion_correction'].get_data_interface('x').data[:]
-        motion_correction_data['y'] = self.nwbfile.modules['motion_correction'].get_data_interface('y').data[:]
+        motion_correction_data['x'] = self.nwbfile.processing['motion_correction'].get_data_interface('x').data[:]
+        motion_correction_data['y'] = self.nwbfile.processing['motion_correction'].get_data_interface('y').data[:]
 
         return pd.DataFrame(motion_correction_data)
 
@@ -304,4 +326,4 @@ def compare_fields(x1, x2, err_msg=""):
                 assert x1[key] == x2[key], key_err_msg
 
     else:
-        assert x1 == x2, err_msg
\ No newline at end of file
+        assert x1 == x2, err_msg
diff --git a/allensdk/brain_observatory/behavior/schemas.py b/allensdk/brain_observatory/behavior/schemas.py
index e1d438ac6..e9364a8b1 100644
--- a/allensdk/brain_observatory/behavior/schemas.py
+++ b/allensdk/brain_observatory/behavior/schemas.py
@@ -73,17 +73,43 @@ class BehaviorMetadataSchema(RaisingSchema):
     )
 
 
-class OphysMetadataSchema(RaisingSchema):
-    """This schema contains metadata pertaining to optical physiology (ophys).
-    """
+class NwbOphysMetadataSchema(RaisingSchema):
+    """This schema contains fields that will be stored in pyNWB base classes
+    pertaining to optical physiology."""
+    # 'emission_lambda' will be stored in
+    # pyNWB OpticalChannel 'emission_lambda' attr
     emission_lambda = fields.Float(
-        doc='emission_lambda',
+        doc='Emission lambda of fluorescent indicator',
         required=True,
     )
+    # 'excitation_lambda' will be stored in the pyNWB ImagingPlane
+    # 'excitation_lambda' attr
     excitation_lambda = fields.Float(
-        doc='excitation_lambda',
+        doc='Excitation lambda of fluorescent indicator',
+        required=True,
+    )
+    # 'indicator' will be stored in the pyNWB ImagingPlane 'indicator' attr
+    indicator = fields.String(
+        doc='Name of optical physiology fluorescent indicator',
+        required=True,
+    )
+    # 'targeted_structure' will be stored in the pyNWB
+    # ImagingPlane 'location' attr
+    targeted_structure = fields.String(
+        doc='Anatomical structure targeted for two-photon acquisition',
         required=True,
     )
+    # 'ophys_frame_rate' will  be stored in the pyNWB ImagingPlane
+    # 'imaging_rate' attr
+    ophys_frame_rate = fields.Float(
+        doc='Frame rate (frames/second) of the two-photon microscope',
+        required=True,
+    )
+
+
+class OphysMetadataSchema(NwbOphysMetadataSchema):
+    """This schema contains metadata pertaining to optical physiology (ophys).
+    """
     experiment_container_id = fields.Int(
         doc='Container ID for the container that contains this ophys session',
         required=True,
@@ -93,24 +119,20 @@ class OphysMetadataSchema(RaisingSchema):
              'targeted for two-photon acquisition'),
         required=True,
     )
-    indicator = fields.String(
-        doc='indicator',
-        required=True,
-    )
     ophys_experiment_id = fields.Int(
         doc='Id for this ophys session',
         required=True,
     )
-    ophys_frame_rate = fields.Float(
-        doc='Frame rate (frames/second) of the two-photon microscope',
+    rig_name = fields.String(
+        doc='Name of optical physiology experiment rig',
         required=True,
     )
-    rig_name = fields.String(
-        doc='name of two-photon rig',
+    field_of_view_width = fields.Int(
+        doc='Width of optical physiology imaging plane in pixels',
         required=True,
     )
-    targeted_structure = fields.String(
-        doc='Anatomical structure targeted for two-photon acquisition',
+    field_of_view_height = fields.Int(
+        doc='Height of optical physiology imaging plane in pixels',
         required=True,
     )
 
@@ -123,24 +145,24 @@ class OphysBehaviorMetadataSchema(BehaviorMetadataSchema, OphysMetadataSchema):
     neurodata_type = 'OphysBehaviorMetadata'
     neurodata_type_inc = 'LabMetaData'
     neurodata_doc = "Metadata for behavior + ophys experiments"
+    # Fields to skip converting to extension
+    # They already exist as attributes for the following pyNWB classes:
+    # OpticalChannel, ImagingPlane, NWBFile
+    neurodata_skip = {"emission_lambda", "excitation_lambda", "indicator",
+                      "targeted_structure", "experiment_datetime",
+                      "ophys_frame_rate"}
 
     session_type = fields.String(
         doc='Experimental session description',
         allow_none=True,
         required=True,
     )
+    # 'experiment_datetime' will be stored in
+    # pynwb NWBFile 'session_start_time' attr
     experiment_datetime = fields.DateTime(
         doc='Date of the experiment (UTC, as string)',
         required=True,
     )
-    field_of_view_width = fields.Int(
-        doc='field_of_view_width',
-        required=True,
-    )
-    field_of_view_height = fields.Int(
-        doc='field_of_view_height',
-        required=True,
-    )
 
 
 class CompleteOphysBehaviorMetadataSchema(OphysBehaviorMetadataSchema,
@@ -162,46 +184,49 @@ class BehaviorTaskParametersSchema(RaisingSchema):
 
     blank_duration_sec = fields.List(
         fields.Float,
-        doc='blank duration in seconds',
+        doc=('The lower and upper bound (in seconds) for a randomly chosen '
+             'inter-stimulus interval duration for a trial'),
         required=True,
         shape=(2,),
     )
     stimulus_duration_sec = fields.Float(
-        doc='duration of each stimulus presentation in seconds',
+        doc='Duration of each stimulus presentation in seconds',
         required=True,
     )
     omitted_flash_fraction = fields.Float(
-        doc='omitted_flash_fraction',
+        doc='Fraction of flashes/image presentations that were omitted',
         required=True,
         allow_nan=True,
     )
     response_window_sec = fields.List(
         fields.Float,
-        doc='response_window in seconds',
+        doc=('The lower and upper bound (in seconds) for a randomly chosen '
+             'time window where subject response influences trial outcome'),
         required=True,
         shape=(2,),
     )
     reward_volume = fields.Float(
-        doc='reward_volume',
+        doc='Volume of water (in mL) delivered as reward',
         required=True,
     )
     stage = fields.String(
-        doc='stage',
+        doc='Stage of behavioral task',
         required=True,
     )
     stimulus = fields.String(
-        doc='stimulus',
+        doc='Stimulus type',
         required=True,
     )
     stimulus_distribution = fields.String(
-        doc='stimulus_distribution',
+        doc=("Distribution type of drawing change times "
+             "(e.g. 'geometric', 'exponential')"),
         required=True,
     )
     task = fields.String(
-        doc='task',
+        doc='The name of the behavioral task',
         required=True,
     )
     n_stimulus_frames = fields.Int(
-        doc='n_stimulus_frames',
+        doc='Total number of stimuli frames',
         required=True,
     )
diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py
index d3d787509..b67bf9cbc 100644
--- a/allensdk/brain_observatory/nwb/__init__.py
+++ b/allensdk/brain_observatory/nwb/__init__.py
@@ -4,6 +4,7 @@
 from typing import Iterable, Tuple
 
 import h5py
+import marshmallow
 import numpy as np
 import pandas as pd
 import datetime
@@ -23,8 +24,9 @@
 from allensdk.brain_observatory.behavior.image_api import Image
 from allensdk.brain_observatory.behavior.image_api import ImageApi
 from allensdk.brain_observatory.behavior.schemas import (
-    CompleteOphysBehaviorMetadataSchema, OphysBehaviorMetadataSchema,
-    BehaviorTaskParametersSchema, SubjectMetadataSchema
+    CompleteOphysBehaviorMetadataSchema, NwbOphysMetadataSchema,
+    OphysBehaviorMetadataSchema, BehaviorTaskParametersSchema,
+    SubjectMetadataSchema
 )
 from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension
 
@@ -363,7 +365,7 @@ def add_running_data_df_to_nwbfile(nwbfile, running_data_df, unit_dict, index_ke
 
     add_running_speed_to_nwbfile(nwbfile, running_speed, name='speed', unit=unit_dict['speed'])
 
-    running_mod = nwbfile.modules['running']
+    running_mod = nwbfile.processing['running']
     timestamps_ts = running_mod.get_data_interface('speed').timestamps
 
     running_dx_series = TimeSeries(
@@ -473,7 +475,7 @@ def add_stimulus_presentations(nwbfile, stimulus_table, tag='stimulus_time_inter
 
     """
     stimulus_table = stimulus_table.copy()
-    ts = nwbfile.modules['stimulus'].get_data_interface('timestamps')
+    ts = nwbfile.processing['stimulus'].get_data_interface('timestamps')
     possible_names = {'stimulus_name', 'image_name'}
     stimulus_name_column = get_column_name(stimulus_table.columns,
                                            possible_names)
@@ -686,11 +688,11 @@ def add_image(nwbfile, image_data, image_name, module_name, module_description,
 
     assert spacing[0] == spacing[1] and len(spacing) == 2 and unit == 'mm'
 
-    if module_name not in nwbfile.modules:
+    if module_name not in nwbfile.processing:
         ophys_mod = ProcessingModule(module_name, module_description)
         nwbfile.add_processing_module(ophys_mod)
     else:
-        ophys_mod = nwbfile.modules[module_name]
+        ophys_mod = nwbfile.processing[module_name]
 
     image = GrayscaleImage(image_name, data, resolution=spacing[0] / 10, description=description)
 
@@ -756,10 +758,15 @@ def add_metadata(nwbfile, metadata: dict):
         genotype=subject_metadata["genotype"],
         subject_id=str(subject_metadata["subject_id"]),
         reporter_line=subject_metadata["reporter_line"],
-        sex=subject_metadata["sex"])
+        sex=subject_metadata["sex"],
+        species='Mus musculus')
     nwbfile.subject = nwb_subject
 
-    # Rest of metadata can go into our custom extension
+    # Remove metadata that will go into pyNWB base classes
+    for key in OphysBehaviorMetadataSchema.neurodata_skip:
+        metadata_clean.pop(key, None)
+
+    # Remaining metadata can go into our custom extension
     new_metadata_dict = {}
     for key, val in metadata_clean.items():
         if isinstance(val, list):
@@ -795,11 +802,13 @@ def add_task_parameters(nwbfile, task_parameters):
 
 
 def add_cell_specimen_table(nwbfile: NWBFile,
-                            cell_specimen_table: pd.DataFrame):
+                            cell_specimen_table: pd.DataFrame,
+                            session_metadata: dict):
     """
     This function takes the cell specimen table and writes the ROIs
     contained within. It writes these to a new NWB imaging plane
     based off the previously supplied metadata
+
     Parameters
     ----------
     nwbfile: NWBFile
@@ -810,38 +819,40 @@ def add_cell_specimen_table(nwbfile: NWBFile,
         experiment, stored in json file and loaded.
         example: /home/nicholasc/projects/allensdk/allensdk/test/
                  brain_observatory/behavior/cell_specimen_table_789359614.json
+    session_metadata: dict
+        Dictionary containing cell_specimen_table related metadata. Should
+        include at minimum the following fields:
+            "emission_lambda", "excitation_lambda", "indicator",
+            "targeted_structure", and ophys_frame_rate"
 
     Returns
     -------
     nwbfile: NWBFile
         The altered in memory NWBFile object that now has a specimen table
     """
+    cell_specimen_metadata = NwbOphysMetadataSchema().load(
+            session_metadata, unknown=marshmallow.EXCLUDE)
     cell_roi_table = cell_specimen_table.reset_index().set_index('cell_roi_id')
 
     # Device:
     device_name = nwbfile.lab_meta_data['metadata'].rig_name
     nwbfile.create_device(device_name,
-                          "Allen Brain Observatory")
+                          "Allen Brain Observatory - Scientifica 2P Rig")
     device = nwbfile.get_device(device_name)
 
-    # Location:
-    location_description = "Area: {}, Depth: {} um".format(
-        nwbfile.lab_meta_data['metadata'].targeted_structure,
-        nwbfile.lab_meta_data['metadata'].imaging_depth)
-
     # FOV:
     fov_width = nwbfile.lab_meta_data['metadata'].field_of_view_width
     fov_height = nwbfile.lab_meta_data['metadata'].field_of_view_height
     imaging_plane_description = "{} field of view in {} at depth {} um".format(
         (fov_width, fov_height),
-        nwbfile.lab_meta_data['metadata'].targeted_structure,
+        cell_specimen_metadata['targeted_structure'],
         nwbfile.lab_meta_data['metadata'].imaging_depth)
 
     # Optical Channel:
     optical_channel = OpticalChannel(
         name='channel_1',
         description='2P Optical Channel',
-        emission_lambda=nwbfile.lab_meta_data['metadata'].emission_lambda)
+        emission_lambda=cell_specimen_metadata['emission_lambda'])
 
     # Imaging Plane:
     imaging_plane = nwbfile.create_imaging_plane(
@@ -849,23 +860,19 @@ def add_cell_specimen_table(nwbfile: NWBFile,
         optical_channel=optical_channel,
         description=imaging_plane_description,
         device=device,
-        excitation_lambda=nwbfile.lab_meta_data['metadata'].excitation_lambda,
-        imaging_rate=nwbfile.lab_meta_data['metadata'].ophys_frame_rate,
-        indicator=nwbfile.lab_meta_data['metadata'].indicator,
-        location=location_description,
-        manifold=[],  # Should this be passed in for future support?
-        conversion=1.0,
-        unit='unknown',  # Should this be passed in for future support?
-        reference_frame='unknown')  # Should this be passed in for future support?
+        excitation_lambda=cell_specimen_metadata['excitation_lambda'],
+        imaging_rate=cell_specimen_metadata['ophys_frame_rate'],
+        indicator=cell_specimen_metadata['indicator'],
+        location=cell_specimen_metadata['targeted_structure'])
 
     # Image Segmentation:
     image_segmentation = ImageSegmentation(name="image_segmentation")
 
-    if 'two_photon_imaging' not in nwbfile.modules:
+    if 'two_photon_imaging' not in nwbfile.processing:
         two_photon_imaging_module = ProcessingModule('two_photon_imaging', '2P processing module')
         nwbfile.add_processing_module(two_photon_imaging_module)
     else:
-        two_photon_imaging_module = nwbfile.modules['two_photon_imaging']
+        two_photon_imaging_module = nwbfile.processing['two_photon_imaging']
 
     two_photon_imaging_module.add_data_interface(image_segmentation)
 
@@ -902,12 +909,12 @@ def add_cell_specimen_table(nwbfile: NWBFile,
 def add_dff_traces(nwbfile, dff_traces, ophys_timestamps):
     dff_traces = dff_traces.reset_index().set_index('cell_roi_id')[['dff']]
 
-    twop_module = nwbfile.modules['two_photon_imaging']
+    twop_module = nwbfile.processing['two_photon_imaging']
     # trace data in the form of rois x timepoints
     trace_data = np.array([dff_traces.loc[cell_roi_id].dff
                            for cell_roi_id in dff_traces.index.values])
 
-    cell_specimen_table = nwbfile.modules['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table']
+    cell_specimen_table = nwbfile.processing['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table']
     roi_table_region = cell_specimen_table.create_roi_table_region(
         description="segmented cells labeled by cell_specimen_id",
         region=slice(len(dff_traces)))
@@ -932,12 +939,12 @@ def add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces):
 
     # Create/Add corrected_fluorescence_traces modules and interfaces:
     assert corrected_fluorescence_traces.index.name == 'cell_roi_id'
-    twop_module = nwbfile.modules['two_photon_imaging']
+    twop_module = nwbfile.processing['two_photon_imaging']
     # trace data in the form of rois x timepoints
     f_trace_data = np.array([corrected_fluorescence_traces.loc[cell_roi_id].corrected_fluorescence
                              for cell_roi_id in corrected_fluorescence_traces.index.values])
 
-    roi_table_region = nwbfile.modules['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces'].rois
+    roi_table_region = nwbfile.processing['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces'].rois
     ophys_timestamps = twop_module.get_data_interface('dff').roi_response_series['traces'].timestamps
     f_interface = Fluorescence(name='corrected_fluorescence')
     twop_module.add_data_interface(f_interface)
@@ -954,7 +961,7 @@ def add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces):
 
 def add_motion_correction(nwbfile, motion_correction):
 
-    twop_module = nwbfile.modules['two_photon_imaging']
+    twop_module = nwbfile.processing['two_photon_imaging']
     ophys_timestamps = twop_module.get_data_interface('dff').roi_response_series['traces'].timestamps
 
     t1 = TimeSeries(
diff --git a/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml b/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml
index a5dd556d3..bc5ef12dd 100644
--- a/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml
+++ b/allensdk/brain_observatory/nwb/ndx-aibs-behavior-ophys.extension.yaml
@@ -5,98 +5,82 @@ groups:
   attributes:
   - name: task
     dtype: text
-    doc: task
-  - name: stimulus_duration_sec
+    doc: The name of the behavioral task
+  - name: omitted_flash_fraction
     dtype: float
-    doc: duration of each stimulus presentation in seconds
-  - name: stage
+    doc: Fraction of flashes/image presentations that were omitted
+  - name: blank_duration_sec
     dtype: text
-    doc: stage
+    shape:
+    - 2
+    doc: The lower and upper bound (in seconds) for a randomly chosen inter-stimulus
+      interval duration for a trial
   - name: n_stimulus_frames
     dtype: int
-    doc: n_stimulus_frames
+    doc: Total number of stimuli frames
+  - name: stimulus
+    dtype: text
+    doc: Stimulus type
   - name: response_window_sec
     dtype: text
     shape:
     - 2
-    doc: response_window in seconds
+    doc: The lower and upper bound (in seconds) for a randomly chosen time window
+      where subject response influences trial outcome
   - name: stimulus_distribution
     dtype: text
-    doc: stimulus_distribution
-  - name: stimulus
-    dtype: text
-    doc: stimulus
-  - name: blank_duration_sec
-    dtype: text
-    shape:
-    - 2
-    doc: blank duration in seconds
-  - name: omitted_flash_fraction
-    dtype: float
-    doc: omitted_flash_fraction
+    doc: Distribution type of drawing change times (e.g. 'geometric', 'exponential')
   - name: reward_volume
     dtype: float
-    doc: reward_volume
+    doc: Volume of water (in mL) delivered as reward
+  - name: stimulus_duration_sec
+    dtype: float
+    doc: Duration of each stimulus presentation in seconds
+  - name: stage
+    dtype: text
+    doc: Stage of behavioral task
 - neurodata_type_def: BehaviorSubject
   neurodata_type_inc: Subject
   doc: Metadata for an AIBS behavior or behavior + ophys subject
   attributes:
-  - name: reporter_line
+  - name: driver_line
     dtype: text
     shape:
     - null
-    doc: Reporter line of subject
-  - name: driver_line
+    doc: Driver line of subject
+  - name: reporter_line
     dtype: text
     shape:
     - null
-    doc: Driver line of subject
+    doc: Reporter line of subject
 - neurodata_type_def: OphysBehaviorMetadata
   neurodata_type_inc: LabMetaData
   doc: Metadata for behavior + ophys experiments
   attributes:
-  - name: session_type
-    dtype: text
-    doc: Experimental session description
-  - name: stimulus_frame_rate
-    dtype: float
-    doc: Frame rate (frames/second) of the visual_stimulus from the monitor
-  - name: excitation_lambda
-    dtype: float
-    doc: excitation_lambda
   - name: imaging_depth
     dtype: int
     doc: Depth (microns) below the cortical surface targeted for two-photon acquisition
-  - name: ophys_frame_rate
-    dtype: float
-    doc: Frame rate (frames/second) of the two-photon microscope
-  - name: experiment_container_id
-    dtype: int
-    doc: Container ID for the container that contains this ophys session
-  - name: field_of_view_width
-    dtype: int
-    doc: field_of_view_width
+  - name: session_type
+    dtype: text
+    doc: Experimental session description
+  - name: rig_name
+    dtype: text
+    doc: Name of optical physiology experiment rig
   - name: ophys_experiment_id
     dtype: int
     doc: Id for this ophys session
-  - name: experiment_datetime
-    dtype: text
-    doc: Date of the experiment (UTC, as string)
-  - name: field_of_view_height
+  - name: experiment_container_id
     dtype: int
-    doc: field_of_view_height
-  - name: emission_lambda
+    doc: Container ID for the container that contains this ophys session
+  - name: stimulus_frame_rate
     dtype: float
-    doc: emission_lambda
-  - name: targeted_structure
-    dtype: text
-    doc: Anatomical structure targeted for two-photon acquisition
+    doc: Frame rate (frames/second) of the visual_stimulus from the monitor
   - name: behavior_session_uuid
     dtype: text
     doc: MTrain record for session, also called foraging_id
-  - name: indicator
-    dtype: text
-    doc: indicator
-  - name: rig_name
-    dtype: text
-    doc: name of two-photon rig
+  - name: field_of_view_height
+    dtype: int
+    doc: Height of optical physiology imaging plane in pixels
+  - name: field_of_view_width
+    dtype: int
+    doc: Width of optical physiology imaging plane in pixels
diff --git a/allensdk/test/brain_observatory/behavior/conftest.py b/allensdk/test/brain_observatory/behavior/conftest.py
index eafb4646f..135d79e1e 100644
--- a/allensdk/test/brain_observatory/behavior/conftest.py
+++ b/allensdk/test/brain_observatory/behavior/conftest.py
@@ -117,7 +117,7 @@ def stimulus_presentations_behavior(stimulus_templates, stimulus_presentations):
 
 @pytest.fixture
 def metadata():
-
+    """Fixture that passes all possible behavior ophys session metadata"""
     return {"ophys_experiment_id": 1234,
             "experiment_container_id": 5678,
             "ophys_frame_rate": 31.0,
@@ -142,6 +142,29 @@ def metadata():
             }
 
 
+@pytest.fixture
+def partial_metadata():
+    """Fixture that passes only metadata that will be saved in
+    custom pyNWB extension fields"""
+    return {"ophys_experiment_id": 1234,
+            "experiment_container_id": 5678,
+            "stimulus_frame_rate": 60.0,
+            "imaging_depth": 375,
+            "session_type": 'Unknown',
+            "experiment_datetime": pytz.utc.localize(datetime.datetime.now()),
+            "reporter_line": ["Ai93(TITL-GCaMP6f)"],
+            "driver_line": ["Camk2a-tTA", "Slc17a7-IRES2-Cre"],
+            "LabTracks_ID": 416369,
+            "full_genotype": "Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt",
+            "behavior_session_uuid": uuid.uuid4(),
+            "field_of_view_width": 2,
+            "field_of_view_height": 2,
+            "rig_name": 'my_device',
+            "sex": 'M',
+            "age": 'P139',
+            }
+
+
 @pytest.fixture
 def task_parameters():
 
diff --git a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
index e6b8da563..4f1b87bec 100644
--- a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
+++ b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
@@ -1,11 +1,16 @@
-import pytest
-import pandas as pd
-import numpy as np
 import math
+import warnings
+
+import numpy as np
+import pandas as pd
+import pynwb
+import pytest
 
 import allensdk.brain_observatory.nwb as nwb
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi
-from allensdk.brain_observatory.behavior.schemas import OphysBehaviorMetadataSchema, BehaviorTaskParametersSchema
+from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import \
+    BehaviorOphysNwbApi
+from allensdk.brain_observatory.behavior.schemas import (
+    BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema)
 
 
 @pytest.mark.parametrize('roundtrip', [True, False])
@@ -163,20 +168,38 @@ def test_segmentation_mask_image(nwbfile, roundtrip, roundtripper, segmentation_
     assert image_api.deserialize(segmentation_mask_image) == image_api.deserialize(obt.get_segmentation_mask_image())
 
 
+@pytest.mark.parametrize('test_partial_metadata', [True, False])
 @pytest.mark.parametrize('roundtrip', [True, False])
-def test_add_metadata(nwbfile, roundtrip, roundtripper, metadata):
-
-    nwb.add_metadata(nwbfile, metadata)
+def test_add_partial_metadata(test_partial_metadata, roundtrip, roundtripper,
+                              cell_specimen_table, metadata, partial_metadata):
+
+    meta = partial_metadata if test_partial_metadata else metadata
+    nwbfile = pynwb.NWBFile(
+        session_description='asession',
+        identifier='afile',
+        session_start_time=meta['experiment_datetime']
+    )
+    nwb.add_metadata(nwbfile, meta)
+    if not test_partial_metadata:
+        nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, meta)
 
     if roundtrip:
         obt = roundtripper(nwbfile, BehaviorOphysNwbApi)
     else:
         obt = BehaviorOphysNwbApi.from_nwbfile(nwbfile)
 
-    metadata_obt = obt.get_metadata()
+    if not test_partial_metadata:
+        metadata_obt = obt.get_metadata()
+    else:
+        with warnings.catch_warnings(record=True) as record:
+            metadata_obt = obt.get_metadata()
+        exp_warn_msg = "Could not locate 'two_photon_imaging' module in NWB"
+        print(record)
+
+        assert record[0].message.args[0].startswith(exp_warn_msg)
 
-    assert len(metadata_obt) == len(metadata)
-    for key, val in metadata.items():
+    assert len(metadata_obt) == len(meta)
+    for key, val in meta.items():
         assert val == metadata_obt[key]
 
 
@@ -208,7 +231,7 @@ def test_add_task_parameters(nwbfile, roundtrip, roundtripper, task_parameters):
 def test_get_cell_specimen_table(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper, cell_specimen_table, metadata, ophys_timestamps):
 
     nwb.add_metadata(nwbfile, metadata)
-    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table)
+    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata)
 
     if roundtrip:
         obt = roundtripper(nwbfile, BehaviorOphysNwbApi, filter_invalid_rois=filter_invalid_rois)
@@ -226,7 +249,7 @@ def test_get_cell_specimen_table(nwbfile, roundtrip, filter_invalid_rois, valid_
 def test_get_dff_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper, dff_traces, cell_specimen_table, metadata, ophys_timestamps):
 
     nwb.add_metadata(nwbfile, metadata)
-    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table)
+    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata)
     nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps)
 
     if roundtrip:
@@ -245,7 +268,7 @@ def test_get_dff_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids,
 def test_get_corrected_fluorescence_traces(nwbfile, roundtrip, filter_invalid_rois, valid_roi_ids, roundtripper, dff_traces, corrected_fluorescence_traces, cell_specimen_table, metadata, ophys_timestamps):
 
     nwb.add_metadata(nwbfile, metadata)
-    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table)
+    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata)
     nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps)
     nwb.add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces)
 
@@ -264,7 +287,7 @@ def test_get_corrected_fluorescence_traces(nwbfile, roundtrip, filter_invalid_ro
 def test_get_motion_correction(nwbfile, roundtrip, roundtripper, motion_correction, ophys_timestamps, metadata, cell_specimen_table, dff_traces):
 
     nwb.add_metadata(nwbfile, metadata)
-    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table)
+    nwb.add_cell_specimen_table(nwbfile, cell_specimen_table, metadata)
     nwb.add_dff_traces(nwbfile, dff_traces, ophys_timestamps)
     nwb.add_motion_correction(nwbfile, motion_correction)
 

From f333400bd31ff7f839b5926823804d125558cd1d Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 16 Oct 2020 08:45:51 -0700
Subject: [PATCH 13/56] Add SI unit to lick autorewarded pyNWB field

---
 allensdk/brain_observatory/nwb/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py
index b67bf9cbc..8c738cfae 100644
--- a/allensdk/brain_observatory/nwb/__init__.py
+++ b/allensdk/brain_observatory/nwb/__init__.py
@@ -652,14 +652,14 @@ def add_rewards(nwbfile, rewards_df):
         name='volume',
         data=rewards_df.volume.values,
         timestamps=rewards_df.index.values,
-        unit='ml'
+        unit='mL'
     )
 
     autorewarded_ts = TimeSeries(
         name='autorewarded',
         data=rewards_df.autorewarded.values,
         timestamps=reward_volume_ts.timestamps,
-        unit=None
+        unit='mL'
     )
 
     rewards_mod = ProcessingModule('rewards', 'Licking behavior processing module')

From e8fc5e0408710c2b880341dbf88e1cb9c7019257 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 16 Oct 2020 15:49:32 -0700
Subject: [PATCH 14/56] Fix behavior ophys NWB bug where ROIs are stored
 incorrectly

Previously, ROIs were added to the behavior ophys NWB file
incorrectly. ROIs all ended up starting at the 0,0 corner of the
whole imaging space.

This was caused by feeding the "LIMS formatted ROI" (List[List[bool]])
through the allensdk.brain_observatory.roi_masks.create_roi_mask
function twice instead of only once. This happened once in
allensdk.internal.api.ophys_lims_api.OphysLimsApi
get_cell_specimen_table() method and yet again in the
allensdk.brain_observatory.nwb.__init__ add_cell_specimen_table()
function.

Now, only the OphysLimsApi get_cell_specimen_table() is the only
place where LIMS formatted ROIs are deserialized to the
allensdk.brain_observatory.roi_masks.RoiMask class.

A CellSpecimenTable schema discrepancy was also fixed.
'x' and 'y' fields should be Int not Float based on what the
behavior_ophys_write_nwb_strategy from LIMS is passing in.
---
 .../behavior_ophys_nwb_api.py                 | 11 ++++--
 .../behavior/write_nwb/_schemas.py            |  4 +-
 allensdk/brain_observatory/nwb/__init__.py    | 24 +++++++-----
 allensdk/internal/api/ophys_lims_api.py       | 14 ++++++-
 .../brain_observatory/behavior/conftest.py    | 39 +++++++++++--------
 5 files changed, 57 insertions(+), 35 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
index db4b86e37..633879883 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
@@ -9,6 +9,10 @@
 import SimpleITK as sitk
 import xarray as xr
 
+from pandas.util.testing import assert_frame_equal
+from pynwb import NWBHDF5IO, NWBFile
+
+import allensdk.brain_observatory.roi_masks as roi
 import allensdk.brain_observatory.nwb as nwb
 from allensdk.brain_observatory.behavior.behavior_ophys_api import \
     BehaviorOphysApiBase
@@ -20,8 +24,6 @@
 from allensdk.brain_observatory.nwb.nwb_api import NwbApi
 from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time
 from allensdk.core.lazy_property import LazyProperty
-from pandas.util.testing import assert_frame_equal
-from pynwb import NWBHDF5IO, NWBFile
 
 load_pynwb_extension(OphysBehaviorMetadataSchema, 'ndx-aibs-behavior-ophys')
 load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys')
@@ -215,10 +217,11 @@ def get_task_parameters(self) -> dict:
         return data
 
     def get_cell_specimen_table(self) -> pd.DataFrame:
-        df = self.nwbfile.processing['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe()
+        # NOTE: ROI masks are stored in full frame width and height arrays
+        df = self.nwbfile.processing['ophys'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table'].to_dataframe()
         df.index.rename('cell_roi_id', inplace=True)
         df['cell_specimen_id'] = [None if csid == -1 else csid for csid in df['cell_specimen_id'].values]
-        df['image_mask'] = [mask.astype(bool) for mask in df['image_mask'].values]
+
         df.reset_index(inplace=True)
         df.set_index('cell_specimen_id', inplace=True)
 
diff --git a/allensdk/brain_observatory/behavior/write_nwb/_schemas.py b/allensdk/brain_observatory/behavior/write_nwb/_schemas.py
index 26a1259ad..38f6df620 100644
--- a/allensdk/brain_observatory/behavior/write_nwb/_schemas.py
+++ b/allensdk/brain_observatory/behavior/write_nwb/_schemas.py
@@ -8,8 +8,8 @@
 class CellSpecimenTable(RaisingSchema):
     cell_roi_id = Dict(String, Int, required=True)
     cell_specimen_id = Dict(String, Int(allow_none=True), required=True)
-    x = Dict(String, Float, required=True)
-    y = Dict(String, Float, required=True)
+    x = Dict(String, Int, required=True)
+    y = Dict(String, Int, required=True)
     max_correction_up = Dict(String, Float, required=True)
     max_correction_right = Dict(String, Float, required=True)
     max_correction_down = Dict(String, Float, required=True)
diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py
index 8c738cfae..28efd3dfa 100644
--- a/allensdk/brain_observatory/nwb/__init__.py
+++ b/allensdk/brain_observatory/nwb/__init__.py
@@ -1,7 +1,7 @@
 import logging
 import warnings
 from pathlib import Path
-from typing import Iterable, Tuple
+from typing import Iterable
 
 import h5py
 import marshmallow
@@ -893,15 +893,19 @@ def add_cell_specimen_table(nwbfile: NWBFile,
                                                                              "No Description Available"))
 
     # go through each roi and add it to the plan segmentation object
-    for cell_roi_id, row in cell_roi_table.iterrows():
-        sub_mask = np.array(row.pop('image_mask'))
-        curr_roi = roi.create_roi_mask(fov_width, fov_height, [(fov_width - 1), 0, (fov_height - 1), 0],
-                                       roi_mask=sub_mask)
-        mask = curr_roi.get_mask_plane()
-        csid = row.pop('cell_specimen_id')
-        row['cell_specimen_id'] = -1 if csid is None else csid
-        row['id'] = cell_roi_id
-        plane_segmentation.add_roi(image_mask=mask, **row.to_dict())
+    for cell_roi_id, table_row in cell_roi_table.iterrows():
+
+        # NOTE: The 'image_mask' in this cell_roi_table has already been
+        # processing by the allensdk.internal.api.ophys_lims_api
+        # get_cell_specimen_table() method. As a result, the ROI is stored in
+        # an array that is the same shape as the FULL field of view of the
+        # experiment (e.g. 512 x 512).
+        mask = table_row.pop('image_mask')
+
+        csid = table_row.pop('cell_specimen_id')
+        table_row['cell_specimen_id'] = -1 if csid is None else csid
+        table_row['id'] = cell_roi_id
+        plane_segmentation.add_roi(image_mask=mask, **table_row.to_dict())
 
     return nwbfile
 
diff --git a/allensdk/internal/api/ophys_lims_api.py b/allensdk/internal/api/ophys_lims_api.py
index e8a156a8e..e6683ced5 100644
--- a/allensdk/internal/api/ophys_lims_api.py
+++ b/allensdk/internal/api/ophys_lims_api.py
@@ -418,10 +418,20 @@ def get_raw_cell_specimen_table_dict(self):
     def get_cell_specimen_table(self):
         cell_specimen_table = pd.DataFrame.from_dict(self.get_raw_cell_specimen_table_dict()).set_index('cell_roi_id').sort_index()
         fov_width, fov_height = self.get_field_of_view_shape()['width'], self.get_field_of_view_shape()['height']
+
+        # Convert cropped ROI masks to uncropped versions
         image_mask_list = []
-        for sub_mask in cell_specimen_table['image_mask'].values:
-            curr_roi = roi.create_roi_mask(fov_width, fov_height, [(fov_width - 1), 0, (fov_height - 1), 0], roi_mask=np.array(sub_mask, dtype=np.bool))
+        for cell_roi_id, table_row in cell_specimen_table.iterrows():
+            # Deserialize roi data into AllenSDK RoiMask object
+            curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height,
+                                   label=None, mask_group=-1)
+            curr_roi.x = table_row['x']
+            curr_roi.y = table_row['y']
+            curr_roi.width = table_row['width']
+            curr_roi.height = table_row['height']
+            curr_roi.mask = np.array(table_row['image_mask'])
             image_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
+
         cell_specimen_table['image_mask'] = image_mask_list
         cell_specimen_table = cell_specimen_table[sorted(cell_specimen_table.columns)]
 
diff --git a/allensdk/test/brain_observatory/behavior/conftest.py b/allensdk/test/brain_observatory/behavior/conftest.py
index 135d79e1e..d91e79f75 100644
--- a/allensdk/test/brain_observatory/behavior/conftest.py
+++ b/allensdk/test/brain_observatory/behavior/conftest.py
@@ -134,8 +134,8 @@ def metadata():
             "emission_lambda": 1.0,
             "excitation_lambda": 1.0,
             "indicator": 'HW',
-            "field_of_view_width": 2,
-            "field_of_view_height": 2,
+            "field_of_view_width": 4,
+            "field_of_view_height": 4,
             "rig_name": 'my_device',
             "sex": 'M',
             "age": 'P139',
@@ -157,8 +157,8 @@ def partial_metadata():
             "LabTracks_ID": 416369,
             "full_genotype": "Slc17a7-IRES2-Cre/wt;Camk2a-tTA/wt;Ai93(TITL-GCaMP6f)/wt",
             "behavior_session_uuid": uuid.uuid4(),
-            "field_of_view_width": 2,
-            "field_of_view_height": 2,
+            "field_of_view_width": 4,
+            "field_of_view_height": 4,
             "rig_name": 'my_device',
             "sex": 'M',
             "age": 'P139',
@@ -183,20 +183,25 @@ def task_parameters():
 
 @pytest.fixture
 def cell_specimen_table():
+
     return pd.DataFrame({'cell_roi_id': [123, 321],
-                         'x': [1, 1],
-                         'y': [1, 1],
-                         'width': [1, 1],
-                         'height': [1, 1],
-                         'valid_roi':[True, False],
-                         'max_correction_up':[1., 1.],
-                         'max_correction_down':[1., 1.],
-                         'max_correction_left':[1., 1.],
-                         'max_correction_right':[1., 1.],
-                         'mask_image_plane':[1, 1],
-                         'ophys_cell_segmentation_run_id':[1, 1],
-                         'image_mask': [np.array([[True, True], [False, False]]), np.array([[True, True], [False, False]])]},
-                          index=pd.Index([None, None], dtype=int, name='cell_specimen_id'))
+                         'x': [0, 2],
+                         'y': [0, 2],
+                         'width': [2, 2],
+                         'height': [2, 2],
+                         'valid_roi': [True, False],
+                         'max_correction_up': [1., 1.],
+                         'max_correction_down': [1., 1.],
+                         'max_correction_left': [1., 1.],
+                         'max_correction_right': [1., 1.],
+                         'mask_image_plane': [1, 1],
+                         'ophys_cell_segmentation_run_id': [1, 1],
+                         'image_mask': [np.array([[True, True],
+                                                  [True, False]]),
+                                        np.array([[True, True],
+                                                  [False, True]])]},
+                        index=pd.Index([None, None], dtype=int,
+                        name='cell_specimen_id'))
 
 
 @pytest.fixture

From e5ef8a3b464db2473cef0243cb416886cb29f65d Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 19 Oct 2020 11:45:17 -0700
Subject: [PATCH 15/56] Rename 'two_photon_imaging' pyNWB processing module to
 'ophys'

To better align with best practice guidelines:
https://www.nwb.org/best-practices/
---
 .../behavior_ophys_nwb_api.py                 | 26 ++++++-----
 allensdk/brain_observatory/nwb/__init__.py    | 44 +++++++++----------
 .../behavior/test_write_nwb_behavior_ophys.py |  2 +-
 3 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
index 633879883..ce9aa0551 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
@@ -41,7 +41,8 @@ def save(self, session_object):
             session_description=str(session_object.metadata['session_type']),
             identifier=str(session_object.ophys_experiment_id),
             session_start_time=session_object.metadata['experiment_datetime'],
-            file_create_date=pytz.utc.localize(datetime.datetime.now())
+            file_create_date=pytz.utc.localize(datetime.datetime.now()),
+            institution="Allen Institute for Brain Science"
         )
 
         # Add stimulus_timestamps to NWB in-memory object:
@@ -133,7 +134,7 @@ def get_stimulus_templates(self, **kwargs):
         return {key: val.data[:] for key, val in self.nwbfile.stimulus_template.items()}
 
     def get_ophys_timestamps(self) -> np.ndarray:
-        return self.nwbfile.processing['two_photon_imaging'].get_data_interface('dff').roi_response_series['traces'].timestamps[:]
+        return self.nwbfile.processing['ophys'].get_data_interface('dff').roi_response_series['traces'].timestamps[:]
 
     def get_stimulus_timestamps(self) -> np.ndarray:
         return self.nwbfile.processing['stimulus'].get_data_interface('timestamps').timestamps[:]
@@ -161,13 +162,13 @@ def get_rewards(self) -> np.ndarray:
             return pd.DataFrame({'volume': [], 'timestamps': [], 'autorewarded': []}).set_index('timestamps')
 
     def get_max_projection(self, image_api=None) -> sitk.Image:
-        return self.get_image('max_projection', 'two_photon_imaging', image_api=image_api)
+        return self.get_image('max_projection', 'ophys', image_api=image_api)
 
     def get_average_projection(self, image_api=None) -> sitk.Image:
-        return self.get_image('average_image', 'two_photon_imaging', image_api=image_api)
+        return self.get_image('average_image', 'ophys', image_api=image_api)
 
     def get_segmentation_mask_image(self, image_api=None) -> sitk.Image:
-        return self.get_image('segmentation_mask_image', 'two_photon_imaging', image_api=image_api)
+        return self.get_image('segmentation_mask_image', 'ophys', image_api=image_api)
 
     def get_metadata(self) -> dict:
 
@@ -187,8 +188,8 @@ def get_metadata(self) -> dict:
         # Add pyNWB OpticalChannel and ImagingPlane metadata to behavior ophys
         # session metadata
         try:
-            two_photon_imaging_module = self.nwbfile.processing['two_photon_imaging']
-            image_seg = two_photon_imaging_module.data_interfaces['image_segmentation']
+            ophys_module = self.nwbfile.processing['ophys']
+            image_seg = ophys_module.data_interfaces['image_segmentation']
             imaging_plane = image_seg.plane_segmentations['cell_specimen_table'].imaging_plane
             optical_channel = imaging_plane.optical_channel[0]
 
@@ -198,7 +199,7 @@ def get_metadata(self) -> dict:
             data['excitation_lambda'] = imaging_plane.excitation_lambda
             data['emission_lambda'] = optical_channel.emission_lambda
         except KeyError:
-            warnings.warn("Could not locate 'two_photon_imaging' module in "
+            warnings.warn("Could not locate 'ophys' module in "
                           "NWB file. The following metadata fields will be "
                           "missing: 'ophys_frame_rate', 'indicator', "
                           "'targeted_structure', 'excitation_lambda', "
@@ -231,7 +232,7 @@ def get_cell_specimen_table(self) -> pd.DataFrame:
         return df
 
     def get_dff_traces(self) -> pd.DataFrame:
-        dff_nwb = self.nwbfile.processing['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces']
+        dff_nwb = self.nwbfile.processing['ophys'].data_interfaces['dff'].roi_response_series['traces']
         # dff traces stored as timepoints x rois in NWB
         # We want rois x timepoints, hence the transpose
         dff_traces = dff_nwb.data[:].T
@@ -247,7 +248,7 @@ def get_dff_traces(self) -> pd.DataFrame:
         return df
 
     def get_corrected_fluorescence_traces(self) -> pd.DataFrame:
-        corrected_fluorescence_nwb = self.nwbfile.processing['two_photon_imaging'].data_interfaces['corrected_fluorescence'].roi_response_series['traces']
+        corrected_fluorescence_nwb = self.nwbfile.processing['ophys'].data_interfaces['corrected_fluorescence'].roi_response_series['traces']
         # f traces stored as timepoints x rois in NWB
         # We want rois x timepoints, hence the transpose
         f_traces = corrected_fluorescence_nwb.data[:].T
@@ -260,10 +261,11 @@ def get_corrected_fluorescence_traces(self) -> pd.DataFrame:
         return df
 
     def get_motion_correction(self) -> pd.DataFrame:
+        ophys_module = self.nwbfile.processing['ophys']
 
         motion_correction_data = {}
-        motion_correction_data['x'] = self.nwbfile.processing['motion_correction'].get_data_interface('x').data[:]
-        motion_correction_data['y'] = self.nwbfile.processing['motion_correction'].get_data_interface('y').data[:]
+        motion_correction_data['x'] = ophys_module.get_data_interface('ophys_motion_correction_x').data[:]
+        motion_correction_data['y'] = ophys_module.get_data_interface('ophys_motion_correction_y').data[:]
 
         return pd.DataFrame(motion_correction_data)
 
diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py
index 28efd3dfa..20e5976c3 100644
--- a/allensdk/brain_observatory/nwb/__init__.py
+++ b/allensdk/brain_observatory/nwb/__init__.py
@@ -708,17 +708,17 @@ def add_image(nwbfile, image_data, image_name, module_name, module_description,
 
 def add_max_projection(nwbfile, max_projection, image_api=None):
 
-    add_image(nwbfile, max_projection, 'max_projection', 'two_photon_imaging', 'Ophys timestamps processing module', image_api=image_api)
+    add_image(nwbfile, max_projection, 'max_projection', 'ophys', 'Ophys processing module', image_api=image_api)
 
 
 def add_average_image(nwbfile, average_image, image_api=None):
 
-    add_image(nwbfile, average_image, 'average_image', 'two_photon_imaging', 'Ophys timestamps processing module', image_api=image_api)
+    add_image(nwbfile, average_image, 'average_image', 'ophys', 'Ophys processing module', image_api=image_api)
 
 
 def add_segmentation_mask_image(nwbfile, segmentation_mask_image, image_api=None):
 
-    add_image(nwbfile, segmentation_mask_image, 'segmentation_mask_image', 'two_photon_imaging', 'Ophys timestamps processing module', image_api=image_api)
+    add_image(nwbfile, segmentation_mask_image, 'segmentation_mask_image', 'ophys', 'Ophys processing module', image_api=image_api)
 
 
 def add_stimulus_index(nwbfile, stimulus_index, nwb_template):
@@ -868,13 +868,13 @@ def add_cell_specimen_table(nwbfile: NWBFile,
     # Image Segmentation:
     image_segmentation = ImageSegmentation(name="image_segmentation")
 
-    if 'two_photon_imaging' not in nwbfile.processing:
-        two_photon_imaging_module = ProcessingModule('two_photon_imaging', '2P processing module')
-        nwbfile.add_processing_module(two_photon_imaging_module)
+    if 'ophys' not in nwbfile.processing:
+        ophys_module = ProcessingModule('ophys', 'Ophys processing module')
+        nwbfile.add_processing_module(ophys_module)
     else:
-        two_photon_imaging_module = nwbfile.processing['two_photon_imaging']
+        ophys_module = nwbfile.processing['ophys']
 
-    two_photon_imaging_module.add_data_interface(image_segmentation)
+    ophys_module.add_data_interface(image_segmentation)
 
     # Plane Segmentation:
     plane_segmentation = image_segmentation.create_plane_segmentation(
@@ -913,12 +913,12 @@ def add_cell_specimen_table(nwbfile: NWBFile,
 def add_dff_traces(nwbfile, dff_traces, ophys_timestamps):
     dff_traces = dff_traces.reset_index().set_index('cell_roi_id')[['dff']]
 
-    twop_module = nwbfile.processing['two_photon_imaging']
+    ophys_module = nwbfile.processing['ophys']
     # trace data in the form of rois x timepoints
     trace_data = np.array([dff_traces.loc[cell_roi_id].dff
                            for cell_roi_id in dff_traces.index.values])
 
-    cell_specimen_table = nwbfile.processing['two_photon_imaging'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table']
+    cell_specimen_table = nwbfile.processing['ophys'].data_interfaces['image_segmentation'].plane_segmentations['cell_specimen_table']
     roi_table_region = cell_specimen_table.create_roi_table_region(
         description="segmented cells labeled by cell_specimen_id",
         region=slice(len(dff_traces)))
@@ -926,7 +926,7 @@ def add_dff_traces(nwbfile, dff_traces, ophys_timestamps):
     # Create/Add dff modules and interfaces:
     assert dff_traces.index.name == 'cell_roi_id'
     dff_interface = DfOverF(name='dff')
-    twop_module.add_data_interface(dff_interface)
+    ophys_module.add_data_interface(dff_interface)
 
     dff_interface.create_roi_response_series(
         name='traces',
@@ -943,15 +943,15 @@ def add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces):
 
     # Create/Add corrected_fluorescence_traces modules and interfaces:
     assert corrected_fluorescence_traces.index.name == 'cell_roi_id'
-    twop_module = nwbfile.processing['two_photon_imaging']
+    ophys_module = nwbfile.processing['ophys']
     # trace data in the form of rois x timepoints
     f_trace_data = np.array([corrected_fluorescence_traces.loc[cell_roi_id].corrected_fluorescence
                              for cell_roi_id in corrected_fluorescence_traces.index.values])
 
-    roi_table_region = nwbfile.processing['two_photon_imaging'].data_interfaces['dff'].roi_response_series['traces'].rois
-    ophys_timestamps = twop_module.get_data_interface('dff').roi_response_series['traces'].timestamps
+    roi_table_region = nwbfile.processing['ophys'].data_interfaces['dff'].roi_response_series['traces'].rois
+    ophys_timestamps = ophys_module.get_data_interface('dff').roi_response_series['traces'].timestamps
     f_interface = Fluorescence(name='corrected_fluorescence')
-    twop_module.add_data_interface(f_interface)
+    ophys_module.add_data_interface(f_interface)
 
     f_interface.create_roi_response_series(
         name='traces',
@@ -965,24 +965,22 @@ def add_corrected_fluorescence_traces(nwbfile, corrected_fluorescence_traces):
 
 def add_motion_correction(nwbfile, motion_correction):
 
-    twop_module = nwbfile.processing['two_photon_imaging']
-    ophys_timestamps = twop_module.get_data_interface('dff').roi_response_series['traces'].timestamps
+    ophys_module = nwbfile.processing['ophys']
+    ophys_timestamps = ophys_module.get_data_interface('dff').roi_response_series['traces'].timestamps
 
     t1 = TimeSeries(
-        name='x',
+        name='ophys_motion_correction_x',
         data=motion_correction['x'].values,
         timestamps=ophys_timestamps,
         unit='pixels'
     )
 
     t2 = TimeSeries(
-        name='y',
+        name='ophys_motion_correction_y',
         data=motion_correction['y'].values,
         timestamps=ophys_timestamps,
         unit='pixels'
     )
 
-    motion_module = ProcessingModule('motion_correction', 'Motion Correction processing module')
-    motion_module.add_data_interface(t1)
-    motion_module.add_data_interface(t2)
-    nwbfile.add_processing_module(motion_module)
+    ophys_module.add_data_interface(t1)
+    ophys_module.add_data_interface(t2)
diff --git a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
index 4f1b87bec..5337c8050 100644
--- a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
+++ b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
@@ -193,7 +193,7 @@ def test_add_partial_metadata(test_partial_metadata, roundtrip, roundtripper,
     else:
         with warnings.catch_warnings(record=True) as record:
             metadata_obt = obt.get_metadata()
-        exp_warn_msg = "Could not locate 'two_photon_imaging' module in NWB"
+        exp_warn_msg = "Could not locate 'ophys' module in NWB"
         print(record)
 
         assert record[0].message.args[0].startswith(exp_warn_msg)

From ac7fde761f92dd5ebf435b4471ed14866beba72c Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 19 Oct 2020 15:22:17 -0700
Subject: [PATCH 16/56] Add experiment descriptions to behavior ophys NWB files

This commit adds experiment descriptions to be havior ophys NWB files.
These descriptions can be determined based on the behavior ophys
"session type" string. Example:

"OPHYS_1_images_A" is the first stage of the visual behavior experiments
"OPHYS_6_images_A" is the last stage of the visual behavior experiments
---
 .../behavior_ophys_nwb_api.py                 | 76 ++++++++++++++++++-
 .../behavior/test_behavior_ophys_nwb_api.py   | 57 ++++++++++++++
 2 files changed, 131 insertions(+), 2 deletions(-)
 create mode 100644 allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
index ce9aa0551..aaf5de52a 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
@@ -29,6 +29,73 @@
 load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys')
 
 
+def _get_expt_description(session_type: str) -> str:
+    """Determine a behavior ophys session's experiment description based on
+    session type.
+
+    Parameters
+    ----------
+    session_type : str
+        A session description string (e.g. OPHYS_1_images_B )
+
+    Returns
+    -------
+    str
+        A description of the experiment based on the session_type.
+
+    Raises
+    ------
+    RuntimeError
+        Behavior ophys sessions should only have 6 different session types.
+        Unknown session types (or malformed session_type strings) will raise
+        an error.
+    """
+    # Experiment descriptions for different session types:
+    # OPHYS_1 -> OPHYS_6
+    ophys_1_3 = dict.fromkeys(
+        ["OPHYS_1", "OPHYS_3"],
+        ("2-photon calcium imaging in the visual cortex of the mouse "
+         "brain as the mouse performs a visual change detection task "
+         "with a set of natural scenes upon which it has previously been "
+         "trained.")
+    )
+    ophys_4_6 = dict.fromkeys(
+        ["OPHYS_4", "OPHYS_6"],
+        ("2-photon calcium imaging in the visual cortex of the mouse "
+         "brain as the mouse performs a visual change detection task "
+         "with a set of natural scenes that are unique from those on "
+         "which it was previously trained.")
+    )
+    ophys_2_5 = {
+        "OPHYS_2": ("2-photon calcium imaging in the visual cortex of the "
+                    "mouse brain as the mouse is shown images from a "
+                    "change detection task with a set of natural scenes "
+                    "upon which it has previously been trained, but with "
+                    "the lick-response sensor withdrawn (passive/open "
+                    "loop mode)."),
+        "OPHYS_5": ("2-photon calcium imaging in the visual cortex of the "
+                    "mouse brain as the mouse is shown images from a "
+                    "change detection task with a set of natural scenes "
+                    "that are unique from those on which it was "
+                    "previously trained, but with the lick-response "
+                    "sensor withdrawn (passive/open loop mode)."),
+    }
+    expt_description_dict = {**ophys_1_3, **ophys_2_5, **ophys_4_6}
+
+    # Session type string will look something like: OPHYS_4_images_A
+    truncated_session_type = "_".join(session_type.split("_")[:2])
+
+    try:
+        return expt_description_dict[truncated_session_type]
+    except KeyError as e:
+        e_msg = (
+            f"Encountered an unknown session type "
+            f"({truncated_session_type}) when trying to determine "
+            f"experiment descriptions. Valid session types are: "
+            f"{expt_description_dict.keys()}")
+        raise RuntimeError(e_msg) from e
+
+
 class BehaviorOphysNwbApi(NwbApi, BehaviorOphysApiBase):
 
     def __init__(self, *args, **kwargs):
@@ -37,12 +104,17 @@ def __init__(self, *args, **kwargs):
 
     def save(self, session_object):
 
+        session_type = str(session_object.metadata['session_type'])
+
         nwbfile = NWBFile(
-            session_description=str(session_object.metadata['session_type']),
+            session_description=session_type,
             identifier=str(session_object.ophys_experiment_id),
             session_start_time=session_object.metadata['experiment_datetime'],
             file_create_date=pytz.utc.localize(datetime.datetime.now()),
-            institution="Allen Institute for Brain Science"
+            institution="Allen Institute for Brain Science",
+            keywords=["2-photon", "calcium imaging", "visual cortex",
+                      "behavior", "task"],
+            experiment_description=_get_expt_description(session_type)
         )
 
         # Add stimulus_timestamps to NWB in-memory object:
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py
new file mode 100644
index 000000000..e84b690be
--- /dev/null
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py
@@ -0,0 +1,57 @@
+import pytest
+
+from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import (
+    _get_expt_description
+)
+
+OPHYS_1_3_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the mouse "
+    "brain as the mouse performs a visual change detection task "
+    "with a set of natural scenes upon which it has previously been "
+    "trained."
+)
+OPHYS_2_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the "
+    "mouse brain as the mouse is shown images from a "
+    "change detection task with a set of natural scenes "
+    "upon which it has previously been trained, but with "
+    "the lick-response sensor withdrawn (passive/open "
+    "loop mode)."
+)
+OPHYS_4_6_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the mouse "
+    "brain as the mouse performs a visual change detection task "
+    "with a set of natural scenes that are unique from those on "
+    "which it was previously trained."
+)
+OPHYS_5_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the "
+    "mouse brain as the mouse is shown images from a "
+    "change detection task with a set of natural scenes "
+    "that are unique from those on which it was "
+    "previously trained, but with the lick-response "
+    "sensor withdrawn (passive/open loop mode)."
+)
+
+@pytest.mark.parametrize("session_type, expected_description", [
+    ("OPHYS_1_images_A", OPHYS_1_3_DESCRIPTION),
+    ("OPHYS_2_images_B", OPHYS_2_DESCRIPTION),
+    ("OPHYS_3_images_C", OPHYS_1_3_DESCRIPTION),
+    ("OPHYS_4_images_D", OPHYS_4_6_DESCRIPTION),
+    ("OPHYS_5_images_E", OPHYS_5_DESCRIPTION),
+    ("OPHYS_6_images_F", OPHYS_4_6_DESCRIPTION)
+])
+def test_get_expt_description_with_valid_session_type(session_type,
+                                                      expected_description):
+    obt = _get_expt_description(session_type)
+    assert obt == expected_description
+
+@pytest.mark.parametrize("session_type", [
+    ("bogus_session_type"),
+    ("stuff"),
+    ("OPHYS_7")
+])
+def test_get_expt_description_raises_with_invalid_session_type(session_type):
+    error_msg_match_phrase = r"Encountered an unknown session type*"
+    with pytest.raises(RuntimeError, match=error_msg_match_phrase):
+        _ = _get_expt_description(session_type)

From fea3acc8dcf01e447d1da0b1d4894bb2677332b9 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Wed, 21 Oct 2020 13:59:40 -0700
Subject: [PATCH 17/56] Revert Argschema version bump

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 3da662766..32ec2a495 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,8 @@ scikit-image>=0.14.0,<0.17.0
 scikit-build<1.0.0
 statsmodels==0.9.0
 simpleitk<2.0.0
-argschema==2.0.1
+argschema<2.0.0
+marshmallow==3.0.0rc6
 glymur==0.8.19
 xarray<0.16.0
 pynwb>=1.3.2,<2.0.0

From 485d9d0f967edddcf890c72ea614d1f34c9f4719 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Wed, 21 Oct 2020 14:27:47 -0700
Subject: [PATCH 18/56] Move get_expt_description and descriptions to more
 relevant location

---
 .../behavior_ophys_nwb_api.py                 | 79 ++-----------------
 .../behavior/metadata_processing.py           | 76 +++++++++++++++++-
 .../behavior/test_behavior_ophys_nwb_api.py   | 57 -------------
 .../behavior/test_metadata_processing.py      | 29 ++++++-
 4 files changed, 111 insertions(+), 130 deletions(-)
 delete mode 100644 allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
index aaf5de52a..130cf64f8 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
@@ -12,14 +12,18 @@
 from pandas.util.testing import assert_frame_equal
 from pynwb import NWBHDF5IO, NWBFile
 
-import allensdk.brain_observatory.roi_masks as roi
 import allensdk.brain_observatory.nwb as nwb
-from allensdk.brain_observatory.behavior.behavior_ophys_api import \
+from allensdk.brain_observatory.behavior.metadata_processing import (
+    get_expt_description
+)
+from allensdk.brain_observatory.behavior.behavior_ophys_api import (
     BehaviorOphysApiBase
+)
 from allensdk.brain_observatory.behavior.schemas import (
     BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema)
-from allensdk.brain_observatory.behavior.trials_processing import \
+from allensdk.brain_observatory.behavior.trials_processing import (
     TRIAL_COLUMN_DESCRIPTION_DICT
+)
 from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension
 from allensdk.brain_observatory.nwb.nwb_api import NwbApi
 from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time
@@ -29,73 +33,6 @@
 load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys')
 
 
-def _get_expt_description(session_type: str) -> str:
-    """Determine a behavior ophys session's experiment description based on
-    session type.
-
-    Parameters
-    ----------
-    session_type : str
-        A session description string (e.g. OPHYS_1_images_B )
-
-    Returns
-    -------
-    str
-        A description of the experiment based on the session_type.
-
-    Raises
-    ------
-    RuntimeError
-        Behavior ophys sessions should only have 6 different session types.
-        Unknown session types (or malformed session_type strings) will raise
-        an error.
-    """
-    # Experiment descriptions for different session types:
-    # OPHYS_1 -> OPHYS_6
-    ophys_1_3 = dict.fromkeys(
-        ["OPHYS_1", "OPHYS_3"],
-        ("2-photon calcium imaging in the visual cortex of the mouse "
-         "brain as the mouse performs a visual change detection task "
-         "with a set of natural scenes upon which it has previously been "
-         "trained.")
-    )
-    ophys_4_6 = dict.fromkeys(
-        ["OPHYS_4", "OPHYS_6"],
-        ("2-photon calcium imaging in the visual cortex of the mouse "
-         "brain as the mouse performs a visual change detection task "
-         "with a set of natural scenes that are unique from those on "
-         "which it was previously trained.")
-    )
-    ophys_2_5 = {
-        "OPHYS_2": ("2-photon calcium imaging in the visual cortex of the "
-                    "mouse brain as the mouse is shown images from a "
-                    "change detection task with a set of natural scenes "
-                    "upon which it has previously been trained, but with "
-                    "the lick-response sensor withdrawn (passive/open "
-                    "loop mode)."),
-        "OPHYS_5": ("2-photon calcium imaging in the visual cortex of the "
-                    "mouse brain as the mouse is shown images from a "
-                    "change detection task with a set of natural scenes "
-                    "that are unique from those on which it was "
-                    "previously trained, but with the lick-response "
-                    "sensor withdrawn (passive/open loop mode)."),
-    }
-    expt_description_dict = {**ophys_1_3, **ophys_2_5, **ophys_4_6}
-
-    # Session type string will look something like: OPHYS_4_images_A
-    truncated_session_type = "_".join(session_type.split("_")[:2])
-
-    try:
-        return expt_description_dict[truncated_session_type]
-    except KeyError as e:
-        e_msg = (
-            f"Encountered an unknown session type "
-            f"({truncated_session_type}) when trying to determine "
-            f"experiment descriptions. Valid session types are: "
-            f"{expt_description_dict.keys()}")
-        raise RuntimeError(e_msg) from e
-
-
 class BehaviorOphysNwbApi(NwbApi, BehaviorOphysApiBase):
 
     def __init__(self, *args, **kwargs):
@@ -114,7 +51,7 @@ def save(self, session_object):
             institution="Allen Institute for Brain Science",
             keywords=["2-photon", "calcium imaging", "visual cortex",
                       "behavior", "task"],
-            experiment_description=_get_expt_description(session_type)
+            experiment_description=get_expt_description(session_type)
         )
 
         # Add stimulus_timestamps to NWB in-memory object:
diff --git a/allensdk/brain_observatory/behavior/metadata_processing.py b/allensdk/brain_observatory/behavior/metadata_processing.py
index 838554085..dd873666a 100644
--- a/allensdk/brain_observatory/behavior/metadata_processing.py
+++ b/allensdk/brain_observatory/behavior/metadata_processing.py
@@ -1,3 +1,77 @@
+OPHYS_1_3_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the mouse "
+    "brain as the mouse performs a visual change detection task "
+    "with a set of natural scenes upon which it has previously been "
+    "trained."
+)
+OPHYS_2_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the "
+    "mouse brain as the mouse is shown images from a "
+    "change detection task with a set of natural scenes "
+    "upon which it has previously been trained, but with "
+    "the lick-response sensor withdrawn (passive/open "
+    "loop mode)."
+)
+OPHYS_4_6_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the mouse "
+    "brain as the mouse performs a visual change detection task "
+    "with a set of natural scenes that are unique from those on "
+    "which it was previously trained."
+)
+OPHYS_5_DESCRIPTION = (
+    "2-photon calcium imaging in the visual cortex of the "
+    "mouse brain as the mouse is shown images from a "
+    "change detection task with a set of natural scenes "
+    "that are unique from those on which it was "
+    "previously trained, but with the lick-response "
+    "sensor withdrawn (passive/open loop mode)."
+)
+
+
+def get_expt_description(session_type: str) -> str:
+    """Determine a behavior ophys session's experiment description based on
+    session type.
+
+    Parameters
+    ----------
+    session_type : str
+        A session description string (e.g. OPHYS_1_images_B )
+
+    Returns
+    -------
+    str
+        A description of the experiment based on the session_type.
+
+    Raises
+    ------
+    RuntimeError
+        Behavior ophys sessions should only have 6 different session types.
+        Unknown session types (or malformed session_type strings) will raise
+        an error.
+    """
+    # Experiment descriptions for different session types:
+    # OPHYS_1 -> OPHYS_6
+    ophys_1_3 = dict.fromkeys(["OPHYS_1", "OPHYS_3"], OPHYS_1_3_DESCRIPTION)
+    ophys_4_6 = dict.fromkeys(["OPHYS_4", "OPHYS_6"], OPHYS_4_6_DESCRIPTION)
+    ophys_2_5 = {"OPHYS_2": OPHYS_2_DESCRIPTION,
+                 "OPHYS_5": OPHYS_5_DESCRIPTION}
+
+    expt_description_dict = {**ophys_1_3, **ophys_2_5, **ophys_4_6}
+
+    # Session type string will look something like: OPHYS_4_images_A
+    truncated_session_type = "_".join(session_type.split("_")[:2])
+
+    try:
+        return expt_description_dict[truncated_session_type]
+    except KeyError as e:
+        e_msg = (
+            f"Encountered an unknown session type "
+            f"({truncated_session_type}) when trying to determine "
+            f"experiment descriptions. Valid session types are: "
+            f"{expt_description_dict.keys()}")
+        raise RuntimeError(e_msg) from e
+
+
 def get_task_parameters(data):
 
     task_parameters = {}
@@ -15,4 +89,4 @@ def get_task_parameters(data):
         n_stimulus_frames += sum(stim_table.get("draw_log", []))
     task_parameters['n_stimulus_frames'] = n_stimulus_frames
 
-    return task_parameters
\ No newline at end of file
+    return task_parameters
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py
deleted file mode 100644
index e84b690be..000000000
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_nwb_api.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import pytest
-
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import (
-    _get_expt_description
-)
-
-OPHYS_1_3_DESCRIPTION = (
-    "2-photon calcium imaging in the visual cortex of the mouse "
-    "brain as the mouse performs a visual change detection task "
-    "with a set of natural scenes upon which it has previously been "
-    "trained."
-)
-OPHYS_2_DESCRIPTION = (
-    "2-photon calcium imaging in the visual cortex of the "
-    "mouse brain as the mouse is shown images from a "
-    "change detection task with a set of natural scenes "
-    "upon which it has previously been trained, but with "
-    "the lick-response sensor withdrawn (passive/open "
-    "loop mode)."
-)
-OPHYS_4_6_DESCRIPTION = (
-    "2-photon calcium imaging in the visual cortex of the mouse "
-    "brain as the mouse performs a visual change detection task "
-    "with a set of natural scenes that are unique from those on "
-    "which it was previously trained."
-)
-OPHYS_5_DESCRIPTION = (
-    "2-photon calcium imaging in the visual cortex of the "
-    "mouse brain as the mouse is shown images from a "
-    "change detection task with a set of natural scenes "
-    "that are unique from those on which it was "
-    "previously trained, but with the lick-response "
-    "sensor withdrawn (passive/open loop mode)."
-)
-
-@pytest.mark.parametrize("session_type, expected_description", [
-    ("OPHYS_1_images_A", OPHYS_1_3_DESCRIPTION),
-    ("OPHYS_2_images_B", OPHYS_2_DESCRIPTION),
-    ("OPHYS_3_images_C", OPHYS_1_3_DESCRIPTION),
-    ("OPHYS_4_images_D", OPHYS_4_6_DESCRIPTION),
-    ("OPHYS_5_images_E", OPHYS_5_DESCRIPTION),
-    ("OPHYS_6_images_F", OPHYS_4_6_DESCRIPTION)
-])
-def test_get_expt_description_with_valid_session_type(session_type,
-                                                      expected_description):
-    obt = _get_expt_description(session_type)
-    assert obt == expected_description
-
-@pytest.mark.parametrize("session_type", [
-    ("bogus_session_type"),
-    ("stuff"),
-    ("OPHYS_7")
-])
-def test_get_expt_description_raises_with_invalid_session_type(session_type):
-    error_msg_match_phrase = r"Encountered an unknown session type*"
-    with pytest.raises(RuntimeError, match=error_msg_match_phrase):
-        _ = _get_expt_description(session_type)
diff --git a/allensdk/test/brain_observatory/behavior/test_metadata_processing.py b/allensdk/test/brain_observatory/behavior/test_metadata_processing.py
index 96e01384a..dee5eda3c 100644
--- a/allensdk/test/brain_observatory/behavior/test_metadata_processing.py
+++ b/allensdk/test/brain_observatory/behavior/test_metadata_processing.py
@@ -1,7 +1,9 @@
+import pytest
 import numpy as np
 
 from allensdk.brain_observatory.behavior.metadata_processing import (
-    get_task_parameters)
+    OPHYS_1_3_DESCRIPTION, OPHYS_2_DESCRIPTION, OPHYS_4_6_DESCRIPTION,
+    OPHYS_5_DESCRIPTION, get_task_parameters, get_expt_description)
 
 
 def test_get_task_parameters():
@@ -54,3 +56,28 @@ def test_get_task_parameters():
         except (TypeError, ValueError):
             assert expected[k] == v
     assert list(actual.keys()) == list(expected.keys())
+
+
+@pytest.mark.parametrize("session_type, expected_description", [
+    ("OPHYS_1_images_A", OPHYS_1_3_DESCRIPTION),
+    ("OPHYS_2_images_B", OPHYS_2_DESCRIPTION),
+    ("OPHYS_3_images_C", OPHYS_1_3_DESCRIPTION),
+    ("OPHYS_4_images_D", OPHYS_4_6_DESCRIPTION),
+    ("OPHYS_5_images_E", OPHYS_5_DESCRIPTION),
+    ("OPHYS_6_images_F", OPHYS_4_6_DESCRIPTION)
+])
+def test_get_expt_description_with_valid_session_type(session_type,
+                                                      expected_description):
+    obt = get_expt_description(session_type)
+    assert obt == expected_description
+
+
+@pytest.mark.parametrize("session_type", [
+    ("bogus_session_type"),
+    ("stuff"),
+    ("OPHYS_7")
+])
+def test_get_expt_description_raises_with_invalid_session_type(session_type):
+    error_msg_match_phrase = r"Encountered an unknown session type*"
+    with pytest.raises(RuntimeError, match=error_msg_match_phrase):
+        _ = get_expt_description(session_type)

From 5fd764bcbc55dcc45274dd0d71d6b2231bfc4562 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 23 Oct 2020 14:53:10 -0700
Subject: [PATCH 19/56] Use try-except-else to detect if behavior ophys NWB has
 ophys module

---
 .../behavior_ophys_api/behavior_ophys_nwb_api.py    | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
index 130cf64f8..d8218f793 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
@@ -198,6 +198,13 @@ def get_metadata(self) -> dict:
         # session metadata
         try:
             ophys_module = self.nwbfile.processing['ophys']
+        except KeyError:
+            warnings.warn("Could not locate 'ophys' module in "
+                          "NWB file. The following metadata fields will be "
+                          "missing: 'ophys_frame_rate', 'indicator', "
+                          "'targeted_structure', 'excitation_lambda', "
+                          "'emission_lambda'")
+        else:
             image_seg = ophys_module.data_interfaces['image_segmentation']
             imaging_plane = image_seg.plane_segmentations['cell_specimen_table'].imaging_plane
             optical_channel = imaging_plane.optical_channel[0]
@@ -207,12 +214,6 @@ def get_metadata(self) -> dict:
             data['targeted_structure'] = imaging_plane.location
             data['excitation_lambda'] = imaging_plane.excitation_lambda
             data['emission_lambda'] = optical_channel.emission_lambda
-        except KeyError:
-            warnings.warn("Could not locate 'ophys' module in "
-                          "NWB file. The following metadata fields will be "
-                          "missing: 'ophys_frame_rate', 'indicator', "
-                          "'targeted_structure', 'excitation_lambda', "
-                          "'emission_lambda'")
 
         # Add other metadata stored in nwb file to behavior ophys session meta
         data['experiment_datetime'] = self.nwbfile.session_start_time

From 2673792397f1cbc30dfaa7ee3cd9736c4d152af6 Mon Sep 17 00:00:00 2001
From: Wesley Jones <wes@upgradeya.com>
Date: Mon, 9 Nov 2020 22:38:03 -0500
Subject: [PATCH 20/56] GH #1481 - Add timeout kwarg to 'from_warehouse'

- Adds the ability to pass kwargs to the RmaEngine
including the timeout kwarg
- Adds pytests to verify the kwargs are passed into
the RmaEngine
- Updates the default timeout for the HttpEngine from
10 to 50 minutes
---
 .../ecephys_project_api/http_engine.py        |  2 +-
 .../ecephys/ecephys_project_cache.py          |  6 +++++-
 .../ecephys/test_ecephys_project_cache.py     | 19 ++++++++++++++++++-
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py b/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py
index 844ecfb02..c07b6024e 100644
--- a/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py
+++ b/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py
@@ -11,7 +11,7 @@
 import nest_asyncio
 from tqdm.auto import tqdm
 
-DEFAULT_TIMEOUT = 10 * 60  # seconds
+DEFAULT_TIMEOUT = 50 * 60  # set timeout to 50 minutes
 DEFAULT_CHUNKSIZE = 1024 * 10  # bytes
 
 
diff --git a/allensdk/brain_observatory/ecephys/ecephys_project_cache.py b/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
index ab68b3503..a8aa84dbd 100644
--- a/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
+++ b/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
@@ -656,7 +656,8 @@ def from_warehouse(cls,
                        manifest: Optional[Union[str, Path]] = None,
                        version: Optional[str] = None,
                        cache: bool = True,
-                       fetch_tries: int = 2):
+                       fetch_tries: int = 2,
+                       **kwargs):
         """
         Create an instance of EcephysProjectCache with an
         EcephysProjectWarehouseApi. Retrieves released data stored in
@@ -684,12 +685,15 @@ def from_warehouse(cls,
         fetch_tries : int
             Maximum number of times to attempt a download before giving up and
             raising an exception. Note that this is total tries, not retries
+        **kwargs
+            Arguments that can be passed to the RmaEngine constructor
         """
         if scheme and host:
             app_kwargs = {"scheme": scheme, "host": host,
                           "asynchronous": asynchronous}
         else:
             app_kwargs = {"asynchronous": asynchronous}
+        app_kwargs.update(kwargs)
         return cls._from_http_source_default(
             EcephysProjectWarehouseApi, app_kwargs, manifest=manifest,
             version=version, cache=cache, fetch_tries=fetch_tries
diff --git a/allensdk/test/brain_observatory/ecephys/test_ecephys_project_cache.py b/allensdk/test/brain_observatory/ecephys/test_ecephys_project_cache.py
index 0941878b7..def086af8 100644
--- a/allensdk/test/brain_observatory/ecephys/test_ecephys_project_cache.py
+++ b/allensdk/test/brain_observatory/ecephys/test_ecephys_project_cache.py
@@ -12,7 +12,8 @@
 from allensdk.core.authentication import DbCredentials
 import allensdk.brain_observatory.ecephys.write_nwb.__main__ as write_nwb
 from allensdk.brain_observatory.ecephys.ecephys_project_api.http_engine import (
-    write_from_stream, write_bytes_from_coroutine, AsyncHttpEngine, HttpEngine
+    write_from_stream, write_bytes_from_coroutine, AsyncHttpEngine, HttpEngine,
+    DEFAULT_TIMEOUT as HTTP_ENGINE_DEFAULT_TIMEOUT
 )
 
 mock_lims_credentials = DbCredentials(dbname='mock_lims', user='mock_user',
@@ -493,3 +494,19 @@ def test_stream_writer_method_default_correct(tmpdir_factory):
     manifest = os.path.join(tmpdir, "manifest.json")
     cache = epc.EcephysProjectCache(stream_writer=None, manifest=manifest)
     assert cache.stream_writer == cache.fetch_api.rma_engine.write_bytes
+
+def test_default_timeout_from_warehouse(tmpdir_factory):
+    tmpdir = str(tmpdir_factory.mktemp("test_from_warehouse_default"))
+    cache = epc.EcephysProjectCache.from_warehouse(
+        manifest=os.path.join(tmpdir, "manifest.json")
+    )
+    assert cache.fetch_api.rma_engine.timeout == HTTP_ENGINE_DEFAULT_TIMEOUT
+
+def test_user_provided_timeout_from_warehouse(tmpdir_factory):
+    user_provided_timeout = 3
+    tmpdir = str(tmpdir_factory.mktemp("test_from_warehouse_default"))
+    cache = epc.EcephysProjectCache.from_warehouse(
+        manifest=os.path.join(tmpdir, "manifest.json"),
+        timeout = user_provided_timeout
+    )
+    assert cache.fetch_api.rma_engine.timeout == user_provided_timeout

From 38a3619390727836b595c2cc905e3c20060c6196 Mon Sep 17 00:00:00 2001
From: Wesley Jones <wes@upgradeya.com>
Date: Tue, 10 Nov 2020 21:52:24 -0500
Subject: [PATCH 21/56] GH #1481 - Providing the timeout parameter as an
 explicit kwarg

---
 .../brain_observatory/ecephys/ecephys_project_cache.py    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/allensdk/brain_observatory/ecephys/ecephys_project_cache.py b/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
index a8aa84dbd..ea07b6cd7 100644
--- a/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
+++ b/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
@@ -657,7 +657,7 @@ def from_warehouse(cls,
                        version: Optional[str] = None,
                        cache: bool = True,
                        fetch_tries: int = 2,
-                       **kwargs):
+                       timeout = 3000):
         """
         Create an instance of EcephysProjectCache with an
         EcephysProjectWarehouseApi. Retrieves released data stored in
@@ -685,15 +685,15 @@ def from_warehouse(cls,
         fetch_tries : int
             Maximum number of times to attempt a download before giving up and
             raising an exception. Note that this is total tries, not retries
-        **kwargs
-            Arguments that can be passed to the RmaEngine constructor
+        timeout
+            Timeout for the RmaEngine constructor. Defaults to 50 minutes.
         """
         if scheme and host:
             app_kwargs = {"scheme": scheme, "host": host,
                           "asynchronous": asynchronous}
         else:
             app_kwargs = {"asynchronous": asynchronous}
-        app_kwargs.update(kwargs)
+        app_kwargs['timeout'] = timeout
         return cls._from_http_source_default(
             EcephysProjectWarehouseApi, app_kwargs, manifest=manifest,
             version=version, cache=cache, fetch_tries=fetch_tries

From abc7cc9f7d8e47b49cefb4fd3da4dd24c5fdc786 Mon Sep 17 00:00:00 2001
From: Wesley Jones <wes@upgradeya.com>
Date: Thu, 12 Nov 2020 06:57:03 -0500
Subject: [PATCH 22/56] GH #1481

- Reduces the default RMAEngine timeout from 50 to 20 minutes
- Adds the type info for the timeout kwarg
- Adding more specific documentation for the timeout kwarg
---
 .../ecephys/ecephys_project_api/http_engine.py            | 2 +-
 .../brain_observatory/ecephys/ecephys_project_cache.py    | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py b/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py
index c07b6024e..d2fe54d6a 100644
--- a/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py
+++ b/allensdk/brain_observatory/ecephys/ecephys_project_api/http_engine.py
@@ -11,7 +11,7 @@
 import nest_asyncio
 from tqdm.auto import tqdm
 
-DEFAULT_TIMEOUT = 50 * 60  # set timeout to 50 minutes
+DEFAULT_TIMEOUT = 20 * 60  # seconds
 DEFAULT_CHUNKSIZE = 1024 * 10  # bytes
 
 
diff --git a/allensdk/brain_observatory/ecephys/ecephys_project_cache.py b/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
index ea07b6cd7..7cfb90167 100644
--- a/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
+++ b/allensdk/brain_observatory/ecephys/ecephys_project_cache.py
@@ -657,7 +657,7 @@ def from_warehouse(cls,
                        version: Optional[str] = None,
                        cache: bool = True,
                        fetch_tries: int = 2,
-                       timeout = 3000):
+                       timeout: int = 1200):
         """
         Create an instance of EcephysProjectCache with an
         EcephysProjectWarehouseApi. Retrieves released data stored in
@@ -685,8 +685,10 @@ def from_warehouse(cls,
         fetch_tries : int
             Maximum number of times to attempt a download before giving up and
             raising an exception. Note that this is total tries, not retries
-        timeout
-            Timeout for the RmaEngine constructor. Defaults to 50 minutes.
+        timeout : int
+            Amount of time (in seconds) to wait on an HTTP request before raising
+            an error. Increase this duration if you find that warehouse servers
+            are not responding quickly. Defaults to 1200 seconds (20 minutes).
         """
         if scheme and host:
             app_kwargs = {"scheme": scheme, "host": host,

From 951d4ef72ec6bc87bfc8d53076b986b3f690f6a8 Mon Sep 17 00:00:00 2001
From: matyasz <tjmatyasz@gmail.com>
Date: Wed, 18 Nov 2020 09:37:27 -0800
Subject: [PATCH 23/56] GH-1327 - Queries now return external_donor_id as
 mouse_id

---
 .../behavior/behavior_project_lims_api.py                  | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index bcbb2da86..0e0159933 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -202,6 +202,7 @@ def _get_behavior_summary_table(self,
                 bs.date_of_acquisition,
                 d.id as donor_id,
                 d.full_genotype,
+                d.external_donor_name AS mouse_id,
                 reporter.reporter_line,
                 driver.driver_line,
                 g.name AS sex,
@@ -288,7 +289,7 @@ def _get_experiment_table(
                         ophys_experiment_id, project_code, session_name,
                         session_type, equipment_name, date_of_acquisition,
                         specimen_id, full_genotype, sex, age_in_days,
-                        reporter_line, driver_line
+                        reporter_line, driver_line, mouse_id
 
         :param ophys_experiment_ids: optional list of ophys_experiment_ids
             to include
@@ -318,6 +319,7 @@ def _get_experiment_table(
                 DATE_PART('day', os.date_of_acquisition - d.date_of_birth)
                     AS age_in_days,
                 d.full_genotype,
+                d.external_donor_name AS mouse_id,
                 reporter.reporter_line,
                 driver.driver_line,
                 id.depth as imaging_depth,
@@ -356,7 +358,7 @@ def _get_session_table(
                         ophys_experiment_id, project_code, session_name,
                         session_type, equipment_name, date_of_acquisition,
                         specimen_id, full_genotype, sex, age_in_days,
-                        reporter_line, driver_line
+                        reporter_line, driver_line, mouse_id
 
         :param ophys_session_ids: optional list of ophys_session_ids to include
         :rtype: pd.DataFrame
@@ -381,6 +383,7 @@ def _get_session_table(
                 DATE_PART('day', os.date_of_acquisition - d.date_of_birth)
                     AS age_in_days,
                 d.full_genotype,
+                d.external_donor_name AS mouse_id,
                 reporter.reporter_line,
                 driver.driver_line
             FROM ophys_sessions os

From cbafe5159016ee42d0762ccfba242f11df6e1804 Mon Sep 17 00:00:00 2001
From: matyasz <tjmatyasz@gmail.com>
Date: Thu, 19 Nov 2020 01:53:31 -0800
Subject: [PATCH 24/56] Fixed bug in BehaviorProjectLimsApi, now uses proper
 mtrain credentials

---
 .../brain_observatory/behavior/behavior_project_lims_api.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index 5fe7f0f96..b175a9b01 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -108,9 +108,9 @@ def default(
 
         if mtrain_credentials:
             mtrain_engine = PostgresQueryMixin(
-                dbname=lims_credentials.dbname, user=lims_credentials.user,
-                host=lims_credentials.host, password=lims_credentials.password,
-                port=lims_credentials.port)
+                dbname=mtrain_credentials.dbname, user=mtrain_credentials.user,
+                host=mtrain_credentials.host, password=mtrain_credentials.password,
+                port=mtrain_credentials.port)
         else:
             # Currying is equivalent to decorator syntactic sugar
             mtrain_engine = (

From ba2e9520301d7153962094bc8d034c9b8f5c40ca Mon Sep 17 00:00:00 2001
From: matyasz <tjmatyasz@gmail.com>
Date: Thu, 19 Nov 2020 01:55:27 -0800
Subject: [PATCH 25/56] Fixed bug in BehaviorProjectLimsApi, now uses proper
 mtrain credentials

---
 .../brain_observatory/behavior/behavior_project_lims_api.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index 0e0159933..abb205fe1 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -108,9 +108,9 @@ def default(
 
         if mtrain_credentials:
             mtrain_engine = PostgresQueryMixin(
-                dbname=lims_credentials.dbname, user=lims_credentials.user,
-                host=lims_credentials.host, password=lims_credentials.password,
-                port=lims_credentials.port)
+                dbname=mtrain_credentials.dbname, user=mtrain_credentials.user,
+                host=mtrain_credentials.host, password=mtrain_credentials.password,
+                port=mtrain_credentials.port)
         else:
             # Currying is equivalent to decorator syntactic sugar
             mtrain_engine = (

From 34c76caedff63e7caf6100953806c27fba73b3bb Mon Sep 17 00:00:00 2001
From: Kat Schelonka <kschelon@gmail.com>
Date: Thu, 19 Nov 2020 10:55:28 -0800
Subject: [PATCH 26/56] Revert "Fixed bug in BehaviorProjectLimsApi, now uses
 proper mtrain credentials"

This reverts commit cbafe5159016ee42d0762ccfba242f11df6e1804.
---
 .../brain_observatory/behavior/behavior_project_lims_api.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index b175a9b01..5fe7f0f96 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -108,9 +108,9 @@ def default(
 
         if mtrain_credentials:
             mtrain_engine = PostgresQueryMixin(
-                dbname=mtrain_credentials.dbname, user=mtrain_credentials.user,
-                host=mtrain_credentials.host, password=mtrain_credentials.password,
-                port=mtrain_credentials.port)
+                dbname=lims_credentials.dbname, user=lims_credentials.user,
+                host=lims_credentials.host, password=lims_credentials.password,
+                port=lims_credentials.port)
         else:
             # Currying is equivalent to decorator syntactic sugar
             mtrain_engine = (

From 5312bff92e81427bfa1b8552b8c9927e042ab22a Mon Sep 17 00:00:00 2001
From: matyasz <tjmatyasz@gmail.com>
Date: Wed, 18 Nov 2020 09:37:27 -0800
Subject: [PATCH 27/56] GH-1327 - Queries now return external_donor_id as
 mouse_id

---
 .../behavior/behavior_project_lims_api.py                  | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index 5fe7f0f96..2d28a8a92 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -202,6 +202,7 @@ def _get_behavior_summary_table(self,
                 bs.date_of_acquisition,
                 d.id as donor_id,
                 d.full_genotype,
+                d.external_donor_name AS mouse_id,
                 reporter.reporter_line,
                 driver.driver_line,
                 g.name AS sex,
@@ -288,7 +289,7 @@ def _get_experiment_table(
                         ophys_experiment_id, project_code, session_name,
                         session_type, equipment_name, date_of_acquisition,
                         specimen_id, full_genotype, sex, age_in_days,
-                        reporter_line, driver_line
+                        reporter_line, driver_line, mouse_id
 
         :param ophys_experiment_ids: optional list of ophys_experiment_ids
             to include
@@ -319,6 +320,7 @@ def _get_experiment_table(
                 DATE_PART('day', os.date_of_acquisition - d.date_of_birth)
                     AS age_in_days,
                 d.full_genotype,
+                d.external_donor_name AS mouse_id,
                 reporter.reporter_line,
                 driver.driver_line,
                 id.depth as imaging_depth,
@@ -357,7 +359,7 @@ def _get_session_table(
                         ophys_experiment_id, project_code, session_name,
                         session_type, equipment_name, date_of_acquisition,
                         specimen_id, full_genotype, sex, age_in_days,
-                        reporter_line, driver_line
+                        reporter_line, driver_line, mouse_id
 
         :param ophys_session_ids: optional list of ophys_session_ids to include
         :rtype: pd.DataFrame
@@ -383,6 +385,7 @@ def _get_session_table(
                 DATE_PART('day', os.date_of_acquisition - d.date_of_birth)
                     AS age_in_days,
                 d.full_genotype,
+                d.external_donor_name AS mouse_id,
                 reporter.reporter_line,
                 driver.driver_line
             FROM ophys_sessions os

From 2e4a6127bb35c9bda07766838ad9b83b3b942e60 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Thu, 12 Nov 2020 11:57:28 -0800
Subject: [PATCH 28/56] Move behavior internal base classes

Previously Abstract Base Classes for allensdk.brain_observatory.behavior
like "BehaviorBase", "BehaviorOphysBase", and "BehaviorProjectBase"
were stored under allensdk.brain_observatory.behavior.

This commit reorganizes the base classes such that:
- "BehaviorBase" and "BehaviorOphysBase" are now imported from:
  `allensdk.brain_observatory.behavior.session_apis.abcs`
- "BehaviorProjectBase" is now imported from:
  `allensdk.brain_observatory.behavior.project_apis.abcs`

Rather than making an "internal/external" api distinction, it is
much more useful to make a distinction between `session` and
`project` apis as well as to make distinctions between
abstract base classes as well as their data fetching instantiations.
---
 allensdk/brain_observatory/behavior/behavior_data_session.py  | 2 +-
 allensdk/brain_observatory/behavior/behavior_project_cache.py | 4 ++--
 .../brain_observatory/behavior/behavior_project_lims_api.py   | 4 ++--
 allensdk/brain_observatory/behavior/internal/__init__.py      | 2 --
 .../brain_observatory/behavior/project_apis/abcs/__init__.py  | 1 +
 .../{internal => project_apis/abcs}/behavior_project_base.py  | 0
 .../brain_observatory/behavior/session_apis/abcs/__init__.py  | 2 ++
 .../behavior/{internal => session_apis/abcs}/behavior_base.py | 2 +-
 .../{internal => session_apis/abcs}/behavior_ophys_base.py    | 2 +-
 allensdk/internal/api/behavior_data_lims_api.py               | 2 +-
 10 files changed, 11 insertions(+), 10 deletions(-)
 delete mode 100644 allensdk/brain_observatory/behavior/internal/__init__.py
 create mode 100644 allensdk/brain_observatory/behavior/project_apis/abcs/__init__.py
 rename allensdk/brain_observatory/behavior/{internal => project_apis/abcs}/behavior_project_base.py (100%)
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/abcs/__init__.py
 rename allensdk/brain_observatory/behavior/{internal => session_apis/abcs}/behavior_base.py (99%)
 rename allensdk/brain_observatory/behavior/{internal => session_apis/abcs}/behavior_ophys_base.py (98%)

diff --git a/allensdk/brain_observatory/behavior/behavior_data_session.py b/allensdk/brain_observatory/behavior/behavior_data_session.py
index 1dc17ba4b..e5203bbc1 100644
--- a/allensdk/brain_observatory/behavior/behavior_data_session.py
+++ b/allensdk/brain_observatory/behavior/behavior_data_session.py
@@ -5,7 +5,7 @@
 import inspect
 
 from allensdk.internal.api.behavior_data_lims_api import BehaviorDataLimsApi
-from allensdk.brain_observatory.behavior.internal import BehaviorBase
+from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorBase
 from allensdk.brain_observatory.running_speed import RunningSpeed
 
 BehaviorDataApi = Type[BehaviorBase]
diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache.py
index 6a90999c3..54a14a9c6 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_cache.py
@@ -8,8 +8,8 @@
 
 from allensdk.brain_observatory.behavior.behavior_project_lims_api import (
     BehaviorProjectLimsApi)
-from allensdk.brain_observatory.behavior.internal.behavior_project_base\
-    import BehaviorProjectBase
+from allensdk.brain_observatory.behavior.project_apis.abcs import (
+    BehaviorProjectBase)
 from allensdk.api.caching_utilities import one_file_call_caching, call_caching
 from allensdk.core.authentication import DbCredentials
 
diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
index 8a30e70b6..f2aea0020 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_lims_api.py
@@ -2,8 +2,8 @@
 from typing import Optional, List, Dict, Any, Iterable
 import logging
 
-from allensdk.brain_observatory.behavior.internal.behavior_project_base\
-    import BehaviorProjectBase
+from allensdk.brain_observatory.behavior.project_apis.abcs import (
+    BehaviorProjectBase)
 from allensdk.brain_observatory.behavior.behavior_data_session import (
     BehaviorDataSession)
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
diff --git a/allensdk/brain_observatory/behavior/internal/__init__.py b/allensdk/brain_observatory/behavior/internal/__init__.py
deleted file mode 100644
index 3364c5707..000000000
--- a/allensdk/brain_observatory/behavior/internal/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from allensdk.brain_observatory.behavior.internal.behavior_base import BehaviorBase  # noqa: F401
-from allensdk.brain_observatory.behavior.internal.behavior_ophys_base import BehaviorOphysBase  # noqa: F401
diff --git a/allensdk/brain_observatory/behavior/project_apis/abcs/__init__.py b/allensdk/brain_observatory/behavior/project_apis/abcs/__init__.py
new file mode 100644
index 000000000..0e9ff87e6
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/project_apis/abcs/__init__.py
@@ -0,0 +1 @@
+from allensdk.brain_observatory.behavior.project_apis.abcs.behavior_project_base import BehaviorProjectBase  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/internal/behavior_project_base.py b/allensdk/brain_observatory/behavior/project_apis/abcs/behavior_project_base.py
similarity index 100%
rename from allensdk/brain_observatory/behavior/internal/behavior_project_base.py
rename to allensdk/brain_observatory/behavior/project_apis/abcs/behavior_project_base.py
diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/__init__.py b/allensdk/brain_observatory/behavior/session_apis/abcs/__init__.py
new file mode 100644
index 000000000..824db012f
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/session_apis/abcs/__init__.py
@@ -0,0 +1,2 @@
+from allensdk.brain_observatory.behavior.session_apis.abcs.behavior_base import BehaviorBase  # noqa: F401, E501
+from allensdk.brain_observatory.behavior.session_apis.abcs.behavior_ophys_base import BehaviorOphysBase  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/internal/behavior_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py
similarity index 99%
rename from allensdk/brain_observatory/behavior/internal/behavior_base.py
rename to allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py
index 930d9992f..a76648fa2 100644
--- a/allensdk/brain_observatory/behavior/internal/behavior_base.py
+++ b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_base.py
@@ -1,6 +1,6 @@
 import abc
 
-from typing import Dict, NamedTuple
+from typing import Dict
 
 import numpy as np
 import pandas as pd
diff --git a/allensdk/brain_observatory/behavior/internal/behavior_ophys_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
similarity index 98%
rename from allensdk/brain_observatory/behavior/internal/behavior_ophys_base.py
rename to allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
index 2159a231f..1f510554e 100644
--- a/allensdk/brain_observatory/behavior/internal/behavior_ophys_base.py
+++ b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
@@ -4,7 +4,7 @@
 import numpy as np
 import pandas as pd
 
-from allensdk.brain_observatory.behavior.internal.behavior_base import BehaviorBase
+from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorBase
 from allensdk.brain_observatory.behavior.image_api import Image
 
 
diff --git a/allensdk/internal/api/behavior_data_lims_api.py b/allensdk/internal/api/behavior_data_lims_api.py
index a4750e976..939cdce01 100644
--- a/allensdk/internal/api/behavior_data_lims_api.py
+++ b/allensdk/internal/api/behavior_data_lims_api.py
@@ -7,7 +7,7 @@
 from typing import Dict, Optional, Union, List, Any
 
 from allensdk.core.exceptions import DataFrameIndexError
-from allensdk.brain_observatory.behavior.internal.behavior_base import (
+from allensdk.brain_observatory.behavior.session_apis.abcs import (
     BehaviorBase)
 from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
 from allensdk.brain_observatory.behavior.running_processing import (

From 93296e475c01de62339381f92f36d8536a1f5482 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Thu, 12 Nov 2020 13:13:50 -0800
Subject: [PATCH 29/56] Move brain_observatory.behavior
 behavior_project_lims_api.py

This commit moves the "BehaviorProjectLimsApi" so that it is
organized under:

allensdk.brain_observatory.behavior.project_apis.data_fetchers

As the class is a LIMS data fetcher for the visual behavior
project cache.
---
 allensdk/brain_observatory/behavior/behavior_project_cache.py   | 2 +-
 .../behavior/project_apis/data_fetchers/__init__.py             | 1 +
 .../data_fetchers}/behavior_project_lims_api.py                 | 0
 .../behavior/test_behavior_project_lims_api.py                  | 2 +-
 4 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py
 rename allensdk/brain_observatory/behavior/{ => project_apis/data_fetchers}/behavior_project_lims_api.py (100%)

diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache.py
index 54a14a9c6..f8520d16e 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_cache.py
@@ -6,7 +6,7 @@
 
 from allensdk.api.cache import Cache
 
-from allensdk.brain_observatory.behavior.behavior_project_lims_api import (
+from allensdk.brain_observatory.behavior.project_apis.data_fetchers import (
     BehaviorProjectLimsApi)
 from allensdk.brain_observatory.behavior.project_apis.abcs import (
     BehaviorProjectBase)
diff --git a/allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py b/allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py
new file mode 100644
index 000000000..d1cc23be3
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py
@@ -0,0 +1 @@
+from allensdk.brain_observatory.behavior.project_apis.data_fetchers.behavior_project_lims_api import BehaviorProjectLimsApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/project_apis/data_fetchers/behavior_project_lims_api.py
similarity index 100%
rename from allensdk/brain_observatory/behavior/behavior_project_lims_api.py
rename to allensdk/brain_observatory/behavior/project_apis/data_fetchers/behavior_project_lims_api.py
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py
index 61c1b381c..e4c58a741 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py
@@ -1,6 +1,6 @@
 import pytest
 
-from allensdk.brain_observatory.behavior.behavior_project_lims_api import (
+from allensdk.brain_observatory.behavior.project_apis.data_fetchers import (
     BehaviorProjectLimsApi)
 from allensdk.test_utilities.custom_comparators import (
     WhitespaceStrippedString)

From f8006e133aa79c7f37ad5d8b06964fdba2d2e969 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 13 Nov 2020 10:35:47 -0800
Subject: [PATCH 30/56] Move behavior_ophys_nwb_api.py to a new location

The BehaviorOphysNwbApi is a behavior + ophys session
data fetching class which contains methods necessary for
filling a BehaviorOphysSession object from an NWB file.

Previously the BehaviorOphysNwbApi was imported from:
allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophy_nwb_api

It has now been moved to and can be imported from:
allensdk.brain_observatory.behavior.session_apis.data_fetchers
---
 .../behavior/behavior_ophys_api/__init__.py   | 62 -------------------
 .../behavior/behavior_ophys_session.py        |  4 +-
 .../session_apis/data_fetchers/__init__.py    |  1 +
 .../data_fetchers}/behavior_ophys_nwb_api.py  |  6 +-
 .../behavior/swdb/behavior_project_cache.py   |  3 +-
 ...save_extended_stimulus_presentations_df.py |  5 +-
 .../behavior/swdb/save_flash_response_df.py   |  3 +-
 .../behavior/swdb/save_trial_response_df.py   |  3 +-
 .../behavior/write_nwb/__main__.py            |  3 +-
 .../behavior/test_behavior_ophys_session.py   |  3 +-
 .../behavior/test_write_nwb_behavior_ophys.py |  4 +-
 .../examples/nb/behavior_ophys_session.ipynb  |  2 +-
 12 files changed, 21 insertions(+), 78 deletions(-)
 delete mode 100644 allensdk/brain_observatory/behavior/behavior_ophys_api/__init__.py
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py
 rename allensdk/brain_observatory/behavior/{behavior_ophys_api => session_apis/data_fetchers}/behavior_ophys_nwb_api.py (99%)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/__init__.py b/allensdk/brain_observatory/behavior/behavior_ophys_api/__init__.py
deleted file mode 100644
index 8a3b905bc..000000000
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/__init__.py
+++ /dev/null
@@ -1,62 +0,0 @@
-
-class BehaviorOphysApiBase:
-
-    def get_ophys_experiment_id(self) -> int:
-        return self.get_metadata()['ophys_experiment_id']
-
-    def get_max_projection(self):
-        raise NotImplementedError
-
-    def get_stimulus_timestamps(self):
-        raise NotImplementedError
-
-    def get_ophys_timestamps(self):
-        raise NotImplementedError
-
-    def get_metadata(self):
-        raise NotImplementedError
-
-    def get_dff_traces(self):
-        raise NotImplementedError
-
-    def get_cell_specimen_table(self):
-        raise NotImplementedError
-
-    def get_running_speed(self):
-        raise NotImplementedError
-
-    def get_running_data_df(self):
-        raise NotImplementedError
-
-    def get_stimulus_presentations(self):
-        raise NotImplementedError
-
-    def get_stimulus_templates(self):
-        raise NotImplementedError
-
-    def get_licks(self):
-        raise NotImplementedError
-
-    def get_rewards(self):
-        raise NotImplementedError
-
-    def get_task_parameters(self):
-        raise NotImplementedError
-
-    def get_trials(self):
-        raise NotImplementedError
-
-    def get_corrected_fluorescence_traces(self):
-        raise NotImplementedError
-
-    def get_motion_correction(self):
-        raise NotImplementedError
-
-    def get_average_projection(self):
-        raise NotImplementedError
-
-    def get_segmentation_mask_image(self):
-        raise NotImplementedError
-
-    def get_eye_tracking_data(self):
-        raise NotImplementedError
diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_session.py b/allensdk/brain_observatory/behavior/behavior_ophys_session.py
index a59eb5112..76cfb1069 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_session.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_session.py
@@ -7,8 +7,8 @@
 
 from allensdk.brain_observatory.session_api_utils import ParamsMixin
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
-from allensdk.brain_observatory.behavior.behavior_ophys_api\
-    .behavior_ophys_nwb_api import BehaviorOphysNwbApi
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi)
 from allensdk.deprecated import legacy
 from allensdk.brain_observatory.behavior.trials_processing import (
     calculate_reward_rate)
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py
new file mode 100644
index 000000000..500e8424c
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py
@@ -0,0 +1 @@
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers.behavior_ophys_nwb_api import BehaviorOphysNwbApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
similarity index 99%
rename from allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
rename to allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
index d8218f793..0067f9f76 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
@@ -16,8 +16,8 @@
 from allensdk.brain_observatory.behavior.metadata_processing import (
     get_expt_description
 )
-from allensdk.brain_observatory.behavior.behavior_ophys_api import (
-    BehaviorOphysApiBase
+from allensdk.brain_observatory.behavior.session_apis.abcs import (
+    BehaviorOphysBase
 )
 from allensdk.brain_observatory.behavior.schemas import (
     BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema)
@@ -33,7 +33,7 @@
 load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys')
 
 
-class BehaviorOphysNwbApi(NwbApi, BehaviorOphysApiBase):
+class BehaviorOphysNwbApi(NwbApi, BehaviorOphysBase):
 
     def __init__(self, *args, **kwargs):
         self.filter_invalid_rois = kwargs.pop("filter_invalid_rois", False)
diff --git a/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py b/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py
index 37fdda4ab..fc2aeb2df 100644
--- a/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py
@@ -5,7 +5,8 @@
 import re
 
 from allensdk import one
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi)
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.core.lazy_property import LazyProperty
 from allensdk.brain_observatory.behavior.trials_processing import calculate_reward_rate
diff --git a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
index 6269de4ea..541e791b9 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
@@ -6,9 +6,8 @@
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession,
 )
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import (
-    BehaviorOphysNwbApi,
-)
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 
 import behavior_project_cache as bpc
diff --git a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
index d40c130f9..f85af9b71 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
@@ -5,7 +5,8 @@
 import itertools
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc
 from allensdk.brain_observatory.behavior.swdb.analysis_tools import get_nearest_frame, get_trace_around_timepoint, get_mean_in_window
diff --git a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
index a1296e2ac..598d8b03b 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
@@ -6,7 +6,8 @@
 import itertools
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 
 from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc
diff --git a/allensdk/brain_observatory/behavior/write_nwb/__main__.py b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
index b39448b08..4bb37353f 100644
--- a/allensdk/brain_observatory/behavior/write_nwb/__main__.py
+++ b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
@@ -10,7 +10,8 @@
 
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi, equals
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi, equals)
 from allensdk.brain_observatory.behavior.write_nwb._schemas import InputSchema, OutputSchema
 from allensdk.brain_observatory.argschema_utilities import write_or_print_outputs
 
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
index 939b10d05..bf721e3b7 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
@@ -11,7 +11,8 @@
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.brain_observatory.behavior.write_nwb.__main__ import BehaviorOphysJsonApi
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi, equals, compare_fields
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi, equals, compare_fields)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.behavior_ophys_api import BehaviorOphysApiBase
 from allensdk.brain_observatory.behavior.image_api import ImageApi
diff --git a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
index 5337c8050..9457b2d2e 100644
--- a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
+++ b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
@@ -7,8 +7,8 @@
 import pytest
 
 import allensdk.brain_observatory.nwb as nwb
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import \
-    BehaviorOphysNwbApi
+from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+    BehaviorOphysNwbApi)
 from allensdk.brain_observatory.behavior.schemas import (
     BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema)
 
diff --git a/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb b/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb
index 649c17ba9..02722bcdc 100644
--- a/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb
+++ b/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb
@@ -13,7 +13,7 @@
     "import pandas as pd\n",
     "import collections\n",
     "\n",
-    "from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi\n",
+    "from allensdk.brain_observatory.behavior.session_apis.data_fetchers import BehaviorOphysNwbApi\n",
     "from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession\n",
     "\n",
     "%matplotlib notebook"

From 74eecb93aa7d6364aabcf3edd8c9f17267625e54 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 13 Nov 2020 14:33:00 -0800
Subject: [PATCH 31/56] Factor out api utils from behavior_ophys_nwb_api.py

Previously, the behavior_ophys_nwb_api.py file implemented
an `equals` function for comparing two Session objects as well as
a `compare_fields` function for checking that attributes of
Session objects are also equal.

Both these functions have been moved to:
allensdk.brain_observatory.session_api_utils

Additional, the `equals` function has been renamed to `sessions_are_equal`
and the `compare_fields` function has been renamed to
`compare_session_fields`.
---
 .../data_fetchers/behavior_ophys_nwb_api.py   | 68 --------------
 .../behavior/write_nwb/__main__.py            | 15 ++-
 .../brain_observatory/session_api_utils.py    | 94 +++++++++++++++++++
 .../behavior/test_behavior_ophys_session.py   | 13 ++-
 4 files changed, 112 insertions(+), 78 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
index 0067f9f76..e4272cb04 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
@@ -1,5 +1,4 @@
 import datetime
-import math
 import uuid
 import warnings
 
@@ -7,9 +6,7 @@
 import pandas as pd
 import pytz
 import SimpleITK as sitk
-import xarray as xr
 
-from pandas.util.testing import assert_frame_equal
 from pynwb import NWBHDF5IO, NWBFile
 
 import allensdk.brain_observatory.nwb as nwb
@@ -27,7 +24,6 @@
 from allensdk.brain_observatory.nwb.metadata import load_pynwb_extension
 from allensdk.brain_observatory.nwb.nwb_api import NwbApi
 from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time
-from allensdk.core.lazy_property import LazyProperty
 
 load_pynwb_extension(OphysBehaviorMetadataSchema, 'ndx-aibs-behavior-ophys')
 load_pynwb_extension(BehaviorTaskParametersSchema, 'ndx-aibs-behavior-ophys')
@@ -278,67 +274,3 @@ def get_motion_correction(self) -> pd.DataFrame:
         motion_correction_data['y'] = ophys_module.get_data_interface('ophys_motion_correction_y').data[:]
 
         return pd.DataFrame(motion_correction_data)
-
-
-def equals(A, B, reraise=False):
-
-    field_set = set()
-    for key, val in A.__dict__.items():
-        if isinstance(val, LazyProperty):
-            field_set.add(key)
-    for key, val in B.__dict__.items():
-        if isinstance(val, LazyProperty):
-            field_set.add(key)
-
-    try:
-        for field in sorted(field_set):
-            x1, x2 = getattr(A, field), getattr(B, field)
-            err_msg = f"{field} on {A} did not equal {field} on {B} (\n{x1} vs\n{x2}\n)"
-            compare_fields(x1, x2, err_msg)
-
-    except NotImplementedError as e:
-        A_implements_get_field = hasattr(A.api, getattr(type(A), field).getter_name)
-        B_implements_get_field = hasattr(B.api, getattr(type(B), field).getter_name)
-        assert A_implements_get_field == B_implements_get_field == False
-
-    except (AssertionError, AttributeError) as e:
-        if reraise:
-            raise
-        return False
-
-    return True
-
-
-
-def compare_fields(x1, x2, err_msg=""):
-    if isinstance(x1, pd.DataFrame):
-        try:
-            assert_frame_equal(x1, x2, check_like=True)
-        except:
-            print(err_msg)
-            raise
-    elif isinstance(x1, np.ndarray):
-        np.testing.assert_array_almost_equal(x1, x2, err_msg=err_msg)
-    elif isinstance(x1, xr.DataArray):
-        xr.testing.assert_allclose(x1, x2)
-    elif isinstance(x1, (list,)):
-        assert x1 == x2, err_msg
-    elif isinstance(x1, (sitk.Image,)):
-        assert x1.GetSize() == x2.GetSize(), err_msg
-        assert x1 == x2, err_msg
-    elif isinstance(x1, (dict,)):
-        for key in set(x1.keys()).union(set(x2.keys())):
-            key_err_msg = f"mismatch when checking key {key}. {err_msg}"
-
-            if isinstance(x1[key], (np.ndarray,)):
-                np.testing.assert_array_almost_equal(x1[key], x2[key], err_msg=key_err_msg)
-            elif isinstance(x1[key], (float,)):
-                if math.isnan(x1[key]) or math.isnan(x2[key]):
-                    assert math.isnan(x1[key]) and math.isnan(x2[key]), key_err_msg
-                else:
-                    assert x1[key] == x2[key], key_err_msg
-            else:
-                assert x1[key] == x2[key], key_err_msg
-
-    else:
-        assert x1 == x2, err_msg
diff --git a/allensdk/brain_observatory/behavior/write_nwb/__main__.py b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
index 4bb37353f..6382008eb 100644
--- a/allensdk/brain_observatory/behavior/write_nwb/__main__.py
+++ b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
@@ -9,11 +9,15 @@
 import marshmallow
 
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
-from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
+from allensdk.brain_observatory.behavior.behavior_ophys_session import (
+    BehaviorOphysSession)
 from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
-    BehaviorOphysNwbApi, equals)
-from allensdk.brain_observatory.behavior.write_nwb._schemas import InputSchema, OutputSchema
-from allensdk.brain_observatory.argschema_utilities import write_or_print_outputs
+    BehaviorOphysNwbApi)
+from allensdk.brain_observatory.behavior.write_nwb._schemas import (
+    InputSchema, OutputSchema)
+from allensdk.brain_observatory.argschema_utilities import (
+    write_or_print_outputs)
+from allensdk.brain_observatory.session_api_utils import sessions_are_equal
 
 
 class BehaviorOphysJsonApi(BehaviorOphysLimsApi):
@@ -121,7 +125,8 @@ def write_behavior_ophys_nwb(session_data, nwb_filepath):
     try:
         session = BehaviorOphysSession(api=BehaviorOphysJsonApi(session_data))
         BehaviorOphysNwbApi(nwb_filepath_inprogress).save(session)
-        assert equals(session, BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath_inprogress)))
+        api = BehaviorOphysNwbApi(nwb_filepath_inprogress)
+        assert sessions_are_equal(session, BehaviorOphysSession(api=api))
         os.rename(nwb_filepath_inprogress, nwb_filepath)
         return {'output_path': nwb_filepath}
     except Exception as e:
diff --git a/allensdk/brain_observatory/session_api_utils.py b/allensdk/brain_observatory/session_api_utils.py
index 279d43b35..e57732a67 100644
--- a/allensdk/brain_observatory/session_api_utils.py
+++ b/allensdk/brain_observatory/session_api_utils.py
@@ -1,4 +1,5 @@
 import inspect
+import math
 import warnings
 
 from itertools import zip_longest
@@ -6,6 +7,12 @@
 
 import numpy as np
 import pandas as pd
+import xarray as xr
+import SimpleITK as sitk
+
+from pandas.util.testing import assert_frame_equal
+
+from allensdk.core.lazy_property import LazyProperty
 
 
 def is_equal(a: Any, b: Any) -> bool:
@@ -139,3 +146,90 @@ def needs_data_refresh(self, data_params: set) -> bool:
     def clear_updated_params(self, data_params: set):
         """This method clears 'updated params' whose data have been updated"""
         self._updated_params -= data_params
+
+
+def sessions_are_equal(A, B, reraise=False) -> bool:
+    """Check if two Session objects are equal (have same methods and
+    attributes).
+
+    Parameters
+    ----------
+    A : Session A
+        The first session to compare
+    B : Session B
+        The second session to compare
+    reraise : bool, optional
+        Whether to reraise when encountering an Assertion or AttributeError,
+        by default False
+
+    Returns
+    -------
+    bool
+        Whether the two sessions are equal to one another.
+    """
+
+    field_set = set()
+    for key, val in A.__dict__.items():
+        if isinstance(val, LazyProperty):
+            field_set.add(key)
+    for key, val in B.__dict__.items():
+        if isinstance(val, LazyProperty):
+            field_set.add(key)
+
+    try:
+        for field in sorted(field_set):
+            x1, x2 = getattr(A, field), getattr(B, field)
+            err_msg = (f"{field} on {A} did not equal {field} "
+                       f"on {B} (\n{x1} vs\n{x2}\n)")
+            compare_session_fields(x1, x2, err_msg)
+
+    except NotImplementedError:
+        A_implements_get_field = hasattr(A.api, getattr(type(A),
+                                         field).getter_name)
+        B_implements_get_field = hasattr(B.api, getattr(type(B),
+                                         field).getter_name)
+        assert ((A_implements_get_field is False)
+                and (B_implements_get_field is False))
+
+    except (AssertionError, AttributeError):
+        if reraise:
+            raise
+        return False
+
+    return True
+
+
+def compare_session_fields(x1, x2, err_msg=""):
+    if isinstance(x1, pd.DataFrame):
+        try:
+            assert_frame_equal(x1, x2, check_like=True)
+        except Exception:
+            print(err_msg)
+            raise
+    elif isinstance(x1, np.ndarray):
+        np.testing.assert_array_almost_equal(x1, x2, err_msg=err_msg)
+    elif isinstance(x1, xr.DataArray):
+        xr.testing.assert_allclose(x1, x2)
+    elif isinstance(x1, (list,)):
+        assert x1 == x2, err_msg
+    elif isinstance(x1, (sitk.Image,)):
+        assert x1.GetSize() == x2.GetSize(), err_msg
+        assert x1 == x2, err_msg
+    elif isinstance(x1, (dict,)):
+        for key in set(x1.keys()).union(set(x2.keys())):
+            key_err_msg = f"Mismatch when checking key {key}. {err_msg}"
+
+            if isinstance(x1[key], (np.ndarray,)):
+                np.testing.assert_array_almost_equal(x1[key], x2[key],
+                                                     err_msg=key_err_msg)
+            elif isinstance(x1[key], (float,)):
+                if math.isnan(x1[key]) or math.isnan(x2[key]):
+                    both_nan = (math.isnan(x1[key]) and math.isnan(x2[key]))
+                    assert both_nan, key_err_msg
+                else:
+                    assert x1[key] == x2[key], key_err_msg
+            else:
+                assert x1[key] == x2[key], key_err_msg
+
+    else:
+        assert x1 == x2, err_msg
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
index bf721e3b7..557d108b8 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
@@ -12,9 +12,11 @@
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.brain_observatory.behavior.write_nwb.__main__ import BehaviorOphysJsonApi
 from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
-    BehaviorOphysNwbApi, equals, compare_fields)
+    BehaviorOphysNwbApi)
+from allensdk.brain_observatory.session_api_utils import (
+    sessions_are_equal, compare_session_fields)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
-from allensdk.brain_observatory.behavior.behavior_ophys_api import BehaviorOphysApiBase
+from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorOphysBase
 from allensdk.brain_observatory.behavior.image_api import ImageApi
 
 
@@ -39,7 +41,7 @@ def test_session_from_json(tmpdir_factory, session_data, get_expected, get_from_
     expected = get_expected(session_data)
     obtained = get_from_session(session)
 
-    compare_fields(expected, obtained)
+    compare_session_fields(expected, obtained)
 
 
 @pytest.mark.requires_bamboo
@@ -51,7 +53,8 @@ def test_nwb_end_to_end(tmpdir_factory):
     BehaviorOphysNwbApi(nwb_filepath).save(d1)
 
     d2 = BehaviorOphysSession(api=BehaviorOphysNwbApi(nwb_filepath))
-    equals(d1, d2, reraise=True)
+
+    assert sessions_are_equal(d1, d2, reraise=True)
 
 
 @pytest.mark.nightly
@@ -186,7 +189,7 @@ def cell_specimen_table_api():
         [0, 0, 0, 0, 0]
     ])
 
-    class CellSpecimenTableApi(BehaviorOphysApiBase):
+    class CellSpecimenTableApi(BehaviorOphysBase):
 
         def get_cell_specimen_table(self):
             return pd.DataFrame(

From 348615e77727f06142772be2f89d239f6cd50dba Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 13 Nov 2020 14:48:31 -0800
Subject: [PATCH 32/56] Remove extraneous tmp.py file

---
 allensdk/tmp.py | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 allensdk/tmp.py

diff --git a/allensdk/tmp.py b/allensdk/tmp.py
deleted file mode 100644
index 37a406221..000000000
--- a/allensdk/tmp.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import os
-
-from allensdk.brain_observatory.behavior.behavior_ophys_api.behavior_ophys_nwb_api import BehaviorOphysNwbApi
-from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
-
-
-basedir = '/allen/aibs/technology/nicholasc/behavior_ophys'
-
-rel_filepath_list = ['behavior_ophys_session_805784331.nwb',
-                     'behavior_ophys_session_789359614.nwb',
-                     'behavior_ophys_session_803736273.nwb',
-                     'behavior_ophys_session_808621958.nwb',
-                     'behavior_ophys_session_795948257.nwb']
-
-for rel_filepath in rel_filepath_list:
-
-    full_filepath = os.path.join(basedir, rel_filepath)
-    session = BehaviorOphysSession(api=BehaviorOphysNwbApi(full_filepath))
-    print(session.metadata['ophys_experiment_id'])

From 89432a619ad85e77974b20df4b2ab288f4643914 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 13 Nov 2020 16:37:14 -0800
Subject: [PATCH 33/56] Swap deprecated BehaviorOphysApiBase with
 BehaviorOphysBase

---
 .../session_apis/abcs/behavior_ophys_base.py  | 33 +++++++++++++++++--
 .../data_fetchers/behavior_ophys_nwb_api.py   | 23 +++++++++++--
 allensdk/internal/api/behavior_ophys_api.py   |  4 +--
 .../behavior/test_behavior_ophys_session.py   |  5 +--
 4 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
index 1f510554e..29f4ebe60 100644
--- a/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
+++ b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
@@ -10,16 +10,22 @@
 
 class BehaviorOphysBase(BehaviorBase):
     """Abstract base class implementing required methods for interacting with
-    behavior+ophys session data.
+    behavior + ophys session data.
 
     Child classes should be instantiated with a fetch API that implements these
     methods.
     """
 
+    @abc.abstractmethod
+    def get_ophys_experiment_id(self) -> Optional[int]:
+        """Returns the ophys_experiment_id for the instantiated BehaviorOphys
+        Session (or BehaviorOphys data fetcher) if applicable."""
+        raise NotImplementedError()
+
     @abc.abstractmethod
     def get_ophys_session_id(self) -> Optional[int]:
-        """Returns the ophys_session_id associated with this experiment,
-        if applicable.
+        """Returns the behavior + ophys_session_id associated with this
+        experiment, if applicable.
         """
         raise NotImplementedError()
 
@@ -156,3 +162,24 @@ def get_stimulus_presentations(self) -> pd.DataFrame:
             and whose columns are presentation characteristics.
         """
         raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_segmentation_mask_image(self) -> Image:
+        """Get the 'segmentation_mask_image'. This image contains pixels
+        with a value of 1 if they are included in ANY ROI segmented in the
+        behavior + ophys session and 0 otherwise.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_eye_tracking_data(self) -> pd.DataFrame:
+        """Get eye tracking data from behavior + ophys session.
+
+        Returns
+        -------
+        pd.DataFrame
+            A refined eye tracking dataframe that contains information
+            about eye tracking ellipse fits, frame times, eye areas,
+            pupil areas, and frames with likely blinks/outliers.
+        """
+        raise NotImplementedError()
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
index e4272cb04..4b2f70001 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
@@ -118,6 +118,21 @@ def save(self, session_object):
 
         return nwbfile
 
+    def get_ophys_experiment_id(self) -> int:
+        return int(self.nwbfile.identifier)
+
+    # TODO: Implement save and lod of behavior_session_id to/from NWB file
+    def get_behavior_session_id(self) -> int:
+        raise NotImplementedError()
+
+    # TODO: Implement save and load of ophys_session_id to/from NWB file 
+    def get_ophys_session_id(self) -> int:
+        raise NotImplementedError()
+
+    # TODO: Implement save and load of eye_tracking_data to/from NWB file
+    def get_eye_tracking_data(self) -> int:
+        raise NotImplementedError()
+
     def get_running_data_df(self, **kwargs):
 
         running_speed = self.get_running_speed()
@@ -135,11 +150,15 @@ def get_running_data_df(self, **kwargs):
 
         return running_data_df[['speed', 'dx', 'v_sig', 'v_in']]
 
+    def get_ophys_timestamps(self) -> np.ndarray:
+        return self.nwbfile.processing['ophys'].get_data_interface('dff').roi_response_series['traces'].timestamps[:]
+
     def get_stimulus_templates(self, **kwargs):
         return {key: val.data[:] for key, val in self.nwbfile.stimulus_template.items()}
 
-    def get_ophys_timestamps(self) -> np.ndarray:
-        return self.nwbfile.processing['ophys'].get_data_interface('dff').roi_response_series['traces'].timestamps[:]
+    # TODO: Implement save and load of 'raw' unaligned stimulus timestamps to/from NWB file
+    def get_raw_stimulus_timestamps(self) -> np.ndarray:
+        raise NotImplementedError()
 
     def get_stimulus_timestamps(self) -> np.ndarray:
         return self.nwbfile.processing['stimulus'].get_data_interface('timestamps').timestamps[:]
diff --git a/allensdk/internal/api/behavior_ophys_api.py b/allensdk/internal/api/behavior_ophys_api.py
index 3b5d9e3b9..25b5ef11e 100644
--- a/allensdk/internal/api/behavior_ophys_api.py
+++ b/allensdk/internal/api/behavior_ophys_api.py
@@ -24,14 +24,14 @@
 from allensdk.brain_observatory.running_speed import RunningSpeed
 from allensdk.brain_observatory.behavior.image_api import ImageApi
 from allensdk.internal.api import PostgresQueryMixin
-from allensdk.brain_observatory.behavior.behavior_ophys_api import BehaviorOphysApiBase
+from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorOphysBase
 from allensdk.brain_observatory.behavior.trials_processing import get_extended_trials
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
 from allensdk.core.authentication import credential_injector, DbCredentials
 
 
-class BehaviorOphysLimsApi(OphysLimsApi, BehaviorOphysApiBase):
+class BehaviorOphysLimsApi(OphysLimsApi, BehaviorOphysBase):
 
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None):
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
index 557d108b8..2422b2c2a 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
@@ -16,7 +16,6 @@
 from allensdk.brain_observatory.session_api_utils import (
     sessions_are_equal, compare_session_fields)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
-from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorOphysBase
 from allensdk.brain_observatory.behavior.image_api import ImageApi
 
 
@@ -189,7 +188,9 @@ def cell_specimen_table_api():
         [0, 0, 0, 0, 0]
     ])
 
-    class CellSpecimenTableApi(BehaviorOphysBase):
+    # Must implement at least the get_cell_specimen_table
+    # and get_max_projection methods from BehaviorOphysBase
+    class CellSpecimenTableApi:
 
         def get_cell_specimen_table(self):
             return pd.DataFrame(

From 5b74072490d8583f7f251025c206478b7be1771c Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 16 Nov 2020 10:54:29 -0800
Subject: [PATCH 34/56] Rename "data_fetchers" directory to "data_io"

Renames the `data_fetchers` directory to `data_io`. Data API
implementations for sessions and/or project in cases like NWB
not only 'fetch' data but also 'save' so 'fetcher' would be
too narrow a description.

This applies to:
- allensdk.brain_observatory.behavior.project_apis.data_fetchers
- allensdk.brain_observatory.behavior.session_apis.data_fetchers
---
 allensdk/brain_observatory/behavior/behavior_ophys_session.py   | 2 +-
 allensdk/brain_observatory/behavior/behavior_project_cache.py   | 2 +-
 .../behavior/project_apis/data_fetchers/__init__.py             | 1 -
 .../brain_observatory/behavior/project_apis/data_io/__init__.py | 1 +
 .../{data_fetchers => data_io}/behavior_project_lims_api.py     | 0
 .../behavior/session_apis/data_fetchers/__init__.py             | 1 -
 .../brain_observatory/behavior/session_apis/data_io/__init__.py | 1 +
 .../{data_fetchers => data_io}/behavior_ophys_nwb_api.py        | 0
 .../brain_observatory/behavior/swdb/behavior_project_cache.py   | 2 +-
 .../behavior/swdb/save_extended_stimulus_presentations_df.py    | 2 +-
 .../brain_observatory/behavior/swdb/save_flash_response_df.py   | 2 +-
 .../brain_observatory/behavior/swdb/save_trial_response_df.py   | 2 +-
 allensdk/brain_observatory/behavior/write_nwb/__main__.py       | 2 +-
 .../brain_observatory/behavior/test_behavior_ophys_session.py   | 2 +-
 .../behavior/test_behavior_project_lims_api.py                  | 2 +-
 .../brain_observatory/behavior/test_write_nwb_behavior_ophys.py | 2 +-
 .../examples_root/examples/nb/behavior_ophys_session.ipynb      | 2 +-
 17 files changed, 13 insertions(+), 13 deletions(-)
 delete mode 100644 allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py
 create mode 100644 allensdk/brain_observatory/behavior/project_apis/data_io/__init__.py
 rename allensdk/brain_observatory/behavior/project_apis/{data_fetchers => data_io}/behavior_project_lims_api.py (100%)
 delete mode 100644 allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
 rename allensdk/brain_observatory/behavior/session_apis/{data_fetchers => data_io}/behavior_ophys_nwb_api.py (100%)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_session.py b/allensdk/brain_observatory/behavior/behavior_ophys_session.py
index 76cfb1069..001800693 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_session.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_session.py
@@ -7,7 +7,7 @@
 
 from allensdk.brain_observatory.session_api_utils import ParamsMixin
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.deprecated import legacy
 from allensdk.brain_observatory.behavior.trials_processing import (
diff --git a/allensdk/brain_observatory/behavior/behavior_project_cache.py b/allensdk/brain_observatory/behavior/behavior_project_cache.py
index f8520d16e..9d2a860ec 100644
--- a/allensdk/brain_observatory/behavior/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/behavior_project_cache.py
@@ -6,7 +6,7 @@
 
 from allensdk.api.cache import Cache
 
-from allensdk.brain_observatory.behavior.project_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.project_apis.data_io import (
     BehaviorProjectLimsApi)
 from allensdk.brain_observatory.behavior.project_apis.abcs import (
     BehaviorProjectBase)
diff --git a/allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py b/allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py
deleted file mode 100644
index d1cc23be3..000000000
--- a/allensdk/brain_observatory/behavior/project_apis/data_fetchers/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from allensdk.brain_observatory.behavior.project_apis.data_fetchers.behavior_project_lims_api import BehaviorProjectLimsApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/project_apis/data_io/__init__.py b/allensdk/brain_observatory/behavior/project_apis/data_io/__init__.py
new file mode 100644
index 000000000..91b3aa8fc
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/project_apis/data_io/__init__.py
@@ -0,0 +1 @@
+from allensdk.brain_observatory.behavior.project_apis.data_io.behavior_project_lims_api import BehaviorProjectLimsApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/project_apis/data_fetchers/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
similarity index 100%
rename from allensdk/brain_observatory/behavior/project_apis/data_fetchers/behavior_project_lims_api.py
rename to allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py
deleted file mode 100644
index 500e8424c..000000000
--- a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers.behavior_ophys_nwb_api import BehaviorOphysNwbApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
new file mode 100644
index 000000000..4a45810a7
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
@@ -0,0 +1 @@
+from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_nwb_api import BehaviorOphysNwbApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
similarity index 100%
rename from allensdk/brain_observatory/behavior/session_apis/data_fetchers/behavior_ophys_nwb_api.py
rename to allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
diff --git a/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py b/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py
index fc2aeb2df..83490e614 100644
--- a/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py
+++ b/allensdk/brain_observatory/behavior/swdb/behavior_project_cache.py
@@ -5,7 +5,7 @@
 import re
 
 from allensdk import one
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.core.lazy_property import LazyProperty
diff --git a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
index 541e791b9..92b1c83dd 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
@@ -6,7 +6,7 @@
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession,
 )
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 
diff --git a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
index f85af9b71..8a875944f 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
@@ -5,7 +5,7 @@
 import itertools
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc
diff --git a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
index 598d8b03b..c4dcdea6d 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
@@ -6,7 +6,7 @@
 import itertools
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 
diff --git a/allensdk/brain_observatory/behavior/write_nwb/__main__.py b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
index 6382008eb..ca4a6c73d 100644
--- a/allensdk/brain_observatory/behavior/write_nwb/__main__.py
+++ b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
@@ -11,7 +11,7 @@
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession)
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.brain_observatory.behavior.write_nwb._schemas import (
     InputSchema, OutputSchema)
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
index 2422b2c2a..1dd5e6eac 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
@@ -11,7 +11,7 @@
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.brain_observatory.behavior.write_nwb.__main__ import BehaviorOphysJsonApi
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.brain_observatory.session_api_utils import (
     sessions_are_equal, compare_session_fields)
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py
index e4c58a741..d9361c058 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_project_lims_api.py
@@ -1,6 +1,6 @@
 import pytest
 
-from allensdk.brain_observatory.behavior.project_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.project_apis.data_io import (
     BehaviorProjectLimsApi)
 from allensdk.test_utilities.custom_comparators import (
     WhitespaceStrippedString)
diff --git a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
index 9457b2d2e..794003012 100644
--- a/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
+++ b/allensdk/test/brain_observatory/behavior/test_write_nwb_behavior_ophys.py
@@ -7,7 +7,7 @@
 import pytest
 
 import allensdk.brain_observatory.nwb as nwb
-from allensdk.brain_observatory.behavior.session_apis.data_fetchers import (
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
 from allensdk.brain_observatory.behavior.schemas import (
     BehaviorTaskParametersSchema, OphysBehaviorMetadataSchema)
diff --git a/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb b/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb
index 02722bcdc..155011827 100644
--- a/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb
+++ b/doc_template/examples_root/examples/nb/behavior_ophys_session.ipynb
@@ -13,7 +13,7 @@
     "import pandas as pd\n",
     "import collections\n",
     "\n",
-    "from allensdk.brain_observatory.behavior.session_apis.data_fetchers import BehaviorOphysNwbApi\n",
+    "from allensdk.brain_observatory.behavior.session_apis.data_io import BehaviorOphysNwbApi\n",
     "from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession\n",
     "\n",
     "%matplotlib notebook"

From 93b71ec00f393de2d743d3cb2b577d8592089d14 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 16 Nov 2020 14:21:30 -0800
Subject: [PATCH 35/56] Rename BehaviorDataSession to BehaviorSession

This commit will rename the BehaviorDataSession class located in
allensdk.brain_observatory.behavior.behavior_data_session
to just BehaviorSession (and
allensdk.brain_observatory.behavior.behavior_session).

This is to bring the naming in line with the BehaviorOphysSession class
and also to draw a distinction between a 'session' class that
end-users will interact with and a 'data io API' class that loads
session data from data sources like LIMS/NWB.
---
 ...avior_data_session.py => behavior_session.py} |  8 ++++----
 .../project_apis/abcs/behavior_project_base.py   | 10 +++++-----
 .../data_io/behavior_project_lims_api.py         | 12 ++++++------
 .../behavior/test_behavior_data_session.py       | 16 ++++++++--------
 .../internal/Lims Behavior Project Cache.ipynb   |  2 +-
 5 files changed, 24 insertions(+), 24 deletions(-)
 rename allensdk/brain_observatory/behavior/{behavior_data_session.py => behavior_session.py} (97%)

diff --git a/allensdk/brain_observatory/behavior/behavior_data_session.py b/allensdk/brain_observatory/behavior/behavior_session.py
similarity index 97%
rename from allensdk/brain_observatory/behavior/behavior_data_session.py
rename to allensdk/brain_observatory/behavior/behavior_session.py
index e5203bbc1..117ea46c8 100644
--- a/allensdk/brain_observatory/behavior/behavior_data_session.py
+++ b/allensdk/brain_observatory/behavior/behavior_session.py
@@ -11,7 +11,7 @@
 BehaviorDataApi = Type[BehaviorBase]
 
 
-class BehaviorDataSession(object):
+class BehaviorSession(object):
     def __init__(self, api: Optional[BehaviorDataApi] = None):
         self.api = api
         # Initialize attributes to be lazily evaluated
@@ -29,12 +29,12 @@ def __init__(self, api: Optional[BehaviorDataApi] = None):
         self._metadata = None
 
     @classmethod
-    def from_lims(cls, behavior_session_id: int) -> "BehaviorDataSession":
+    def from_lims(cls, behavior_session_id: int) -> "BehaviorSession":
         return cls(api=BehaviorDataLimsApi(behavior_session_id))
 
     @classmethod
     def from_nwb_path(
-            cls, nwb_path: str, **api_kwargs: Any) -> "BehaviorDataSession":
+            cls, nwb_path: str, **api_kwargs: Any) -> "BehaviorSession":
         return NotImplementedError
 
     @property
@@ -301,7 +301,7 @@ def cache_clear(self) -> None:
     def list_api_methods(self) -> List[Tuple[str, str]]:
         """Convenience method to expose list of API `get` methods. These methods
         can be accessed by referencing the API used to initialize this
-        BehaviorDataSession via its `api` instance attribute.
+        BehaviorSession via its `api` instance attribute.
         :rtype: list of tuples, where the first value in the tuple is the
         method name, and the second value is the method docstring.
         """
diff --git a/allensdk/brain_observatory/behavior/project_apis/abcs/behavior_project_base.py b/allensdk/brain_observatory/behavior/project_apis/abcs/behavior_project_base.py
index ead386708..380e68862 100644
--- a/allensdk/brain_observatory/behavior/project_apis/abcs/behavior_project_base.py
+++ b/allensdk/brain_observatory/behavior/project_apis/abcs/behavior_project_base.py
@@ -3,8 +3,8 @@
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession)
-from allensdk.brain_observatory.behavior.behavior_data_session import (
-    BehaviorDataSession)
+from allensdk.brain_observatory.behavior.behavior_session import (
+    BehaviorSession)
 import pandas as pd
 
 
@@ -27,12 +27,12 @@ def get_session_table(self) -> pd.DataFrame:
 
     @abstractmethod
     def get_behavior_only_session_data(
-            self, behavior_session_id: int) -> BehaviorDataSession:
-        """Returns a BehaviorDataSession object that contains methods to
+            self, behavior_session_id: int) -> BehaviorSession:
+        """Returns a BehaviorSession object that contains methods to
         analyze a single behavior session.
         :param behavior_session_id: id that corresponds to a behavior session
         :type behavior_session_id: int
-        :rtype: BehaviorDataSession
+        :rtype: BehaviorSession
         """
         pass
 
diff --git a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
index f2aea0020..5c0e67413 100644
--- a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
@@ -4,8 +4,8 @@
 
 from allensdk.brain_observatory.behavior.project_apis.abcs import (
     BehaviorProjectBase)
-from allensdk.brain_observatory.behavior.behavior_data_session import (
-    BehaviorDataSession)
+from allensdk.brain_observatory.behavior.behavior_session import (
+    BehaviorSession)
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession)
 from allensdk.internal.api.behavior_data_lims_api import BehaviorDataLimsApi
@@ -430,14 +430,14 @@ def get_session_table(
         return table
 
     def get_behavior_only_session_data(
-            self, behavior_session_id: int) -> BehaviorDataSession:
-        """Returns a BehaviorDataSession object that contains methods to
+            self, behavior_session_id: int) -> BehaviorSession:
+        """Returns a BehaviorSession object that contains methods to
         analyze a single behavior session.
         :param behavior_session_id: id that corresponds to a behavior session
         :type behavior_session_id: int
-        :rtype: BehaviorDataSession
+        :rtype: BehaviorSession
         """
-        return BehaviorDataSession(BehaviorDataLimsApi(behavior_session_id))
+        return BehaviorSession(BehaviorDataLimsApi(behavior_session_id))
 
     def get_experiment_table(
             self,
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_data_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_data_session.py
index cd3aa4c2a..b5a3c6e8a 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_data_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_data_session.py
@@ -1,7 +1,7 @@
 import logging
 
-from allensdk.brain_observatory.behavior.behavior_data_session import (
-    BehaviorDataSession)
+from allensdk.brain_observatory.behavior.behavior_session import (
+    BehaviorSession)
 
 
 class DummyApi(object):
@@ -25,18 +25,18 @@ def cache_clear(self):
         pass
 
 
-class TestBehaviorDataSession:
-    """Tests for BehaviorDataSession.
-       The vast majority of methods in BehaviorDataSession are simply calling
+class TestBehaviorSession:
+    """Tests for BehaviorSession.
+       The vast majority of methods in BehaviorSession are simply calling
        functions from the underlying API. The API required for instantiating a
-       BehaviorDataSession is annotated to show that it requires an class that
+       BehaviorSession is annotated to show that it requires an class that
        inherits from BehaviorBase, it is ensured that those methods exist in
        the API class. These methods should be covered by unit tests on the
        API class and will not be re-tested here.
     """
     @classmethod
     def setup_class(cls):
-        cls.behavior_session = BehaviorDataSession(api=DummyApi())
+        cls.behavior_session = BehaviorSession(api=DummyApi())
 
     def test_list_api_methods(self):
         expected = [("get_method", "Method docstring"),
@@ -53,6 +53,6 @@ def test_cache_clear_raises_warning(self, caplog):
 
     def test_cache_clear_no_warning(self, caplog):
         caplog.clear()
-        bs = BehaviorDataSession(api=DummyApiCache())
+        bs = BehaviorSession(api=DummyApiCache())
         bs.cache_clear()
         assert len(caplog.record_tuples) == 0
diff --git a/doc_template/examples_root/examples/internal/Lims Behavior Project Cache.ipynb b/doc_template/examples_root/examples/internal/Lims Behavior Project Cache.ipynb
index f39ffa4ea..280edd008 100644
--- a/doc_template/examples_root/examples/internal/Lims Behavior Project Cache.ipynb	
+++ b/doc_template/examples_root/examples/internal/Lims Behavior Project Cache.ipynb	
@@ -8,7 +8,7 @@
     "A short introduction to analyzing the Visual Behavior data.\n",
     "This notebook uses the LIMS API to access data, so it will only work on the Allen Institute network.\n",
     "\n",
-    "Please note that local caching functionality has not been implemented, as there are currently no NWB files for these data. Currently the `BehaviorProjectCache` acts as an interface to query for all experiments/sessions in the LIMS database, and to produce `BehaviorDataSession` and/or `BehaviorOphysSession` objects (using `get_session_data` and `get_behavior_session_data`, respectively)."
+    "Please note that local caching functionality has not been implemented, as there are currently no NWB files for these data. Currently the `BehaviorProjectCache` acts as an interface to query for all experiments/sessions in the LIMS database, and to produce `BehaviorSession` and/or `BehaviorOphysSession` objects (using `get_session_data` and `get_behavior_session_data`, respectively)."
    ]
   },
   {

From 91f823a5509c35911f9ff27e4f3ae9c71b29f632 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 16 Nov 2020 14:34:20 -0800
Subject: [PATCH 36/56] Move behavior and behavior + ophys session LIMS APIs

This commit moved behavior and behavior + ophys session
LIMS API related classes from `allensdk.internal.api` to:

`allensdk.brain_observatory.behavior.session_apis.data_io`

The files moved include:
- behavior_data_lims_api: Newer version API for behavior only data from LIMS
- behavior_lims_api: Older version API for behavior only data from LIMS
- behavior_ophys_api: API for behavior + ophys data from LIMS
- ophy_lims_api: API for ophys data from LIMS
---
 .../behavior/session_apis/data_io}/behavior_data_lims_api.py      | 0
 .../behavior/session_apis/data_io}/behavior_lims_api.py           | 0
 .../behavior/session_apis/data_io}/behavior_ophys_api.py          | 0
 .../behavior/session_apis/data_io}/ophys_lims_api.py              | 0
 4 files changed, 0 insertions(+), 0 deletions(-)
 rename allensdk/{internal/api => brain_observatory/behavior/session_apis/data_io}/behavior_data_lims_api.py (100%)
 rename allensdk/{internal/api => brain_observatory/behavior/session_apis/data_io}/behavior_lims_api.py (100%)
 rename allensdk/{internal/api => brain_observatory/behavior/session_apis/data_io}/behavior_ophys_api.py (100%)
 rename allensdk/{internal/api => brain_observatory/behavior/session_apis/data_io}/ophys_lims_api.py (100%)

diff --git a/allensdk/internal/api/behavior_data_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_data_lims_api.py
similarity index 100%
rename from allensdk/internal/api/behavior_data_lims_api.py
rename to allensdk/brain_observatory/behavior/session_apis/data_io/behavior_data_lims_api.py
diff --git a/allensdk/internal/api/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
similarity index 100%
rename from allensdk/internal/api/behavior_lims_api.py
rename to allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
diff --git a/allensdk/internal/api/behavior_ophys_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_api.py
similarity index 100%
rename from allensdk/internal/api/behavior_ophys_api.py
rename to allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_api.py
diff --git a/allensdk/internal/api/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
similarity index 100%
rename from allensdk/internal/api/ophys_lims_api.py
rename to allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py

From b604bbf4e1b1a34e9ede6db21caae4d2192b7ad3 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 16 Nov 2020 17:41:44 -0800
Subject: [PATCH 37/56] Merge BehaviorLimsApi duplicates

It turns out that "BehaviorDataLimsApi" located in
allensdk.brain_observatory.behavior.session_apis.data_io.behavior_data_lims_api
(previously allensdk.internal.api.behavior_data_lims_api)
and the "BehaviorLimsApi" located in
allensdk.brain_observatory.bheavior.session_apis.data_io.behavior_lims_api
(previously allensdk.internal.api.behavior_lims_api)
are historically separate implementations of what *should* be the same class.

This commit does the following:
- Add the ability to construct a BehaviorDataLimsApi from a foraging_id.
  This was basically the only unique functionality provided by the
  BehaviorLimsApi class.
- Remove the BehaviorLimsApi class completely.
- Spruce up how database connections are created
- Rename the BehaviorDataLimsApi to BehaviorLimsApi
- Rename behavior_data_lims_api.py to behavior_lims_api.py
---
 .../behavior/behavior_session.py              |   5 +-
 .../data_io/behavior_project_lims_api.py      |   5 +-
 .../behavior/session_apis/data_io/__init__.py |   4 +
 .../data_io/behavior_data_lims_api.py         | 516 ---------------
 .../session_apis/data_io/behavior_lims_api.py | 609 ++++++++++++++++--
 .../session_apis/data_processing/__init__.py  |   0
 allensdk/internal/api/__init__.py             |  55 +-
 allensdk/internal/api/mtrain_api.py           |  12 +-
 .../behavior/test_behavior_data_lims_api.py   |  48 +-
 .../behavior/test_behavior_lims_api.py        |  16 +-
 .../behavior/test_trials_processing.py        |   5 +-
 11 files changed, 640 insertions(+), 635 deletions(-)
 delete mode 100644 allensdk/brain_observatory/behavior/session_apis/data_io/behavior_data_lims_api.py
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/data_processing/__init__.py

diff --git a/allensdk/brain_observatory/behavior/behavior_session.py b/allensdk/brain_observatory/behavior/behavior_session.py
index 117ea46c8..b1d4f68d4 100644
--- a/allensdk/brain_observatory/behavior/behavior_session.py
+++ b/allensdk/brain_observatory/behavior/behavior_session.py
@@ -4,7 +4,8 @@
 import numpy as np
 import inspect
 
-from allensdk.internal.api.behavior_data_lims_api import BehaviorDataLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorLimsApi)
 from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorBase
 from allensdk.brain_observatory.running_speed import RunningSpeed
 
@@ -30,7 +31,7 @@ def __init__(self, api: Optional[BehaviorDataApi] = None):
 
     @classmethod
     def from_lims(cls, behavior_session_id: int) -> "BehaviorSession":
-        return cls(api=BehaviorDataLimsApi(behavior_session_id))
+        return cls(api=BehaviorLimsApi(behavior_session_id))
 
     @classmethod
     def from_nwb_path(
diff --git a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
index 5c0e67413..1e9fa85a6 100644
--- a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
@@ -8,7 +8,8 @@
     BehaviorSession)
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession)
-from allensdk.internal.api.behavior_data_lims_api import BehaviorDataLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorLimsApi)
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.internal.api import PostgresQueryMixin
 from allensdk.brain_observatory.ecephys.ecephys_project_api.http_engine import (
@@ -437,7 +438,7 @@ def get_behavior_only_session_data(
         :type behavior_session_id: int
         :rtype: BehaviorSession
         """
-        return BehaviorSession(BehaviorDataLimsApi(behavior_session_id))
+        return BehaviorSession(BehaviorLimsApi(behavior_session_id))
 
     def get_experiment_table(
             self,
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
index 4a45810a7..9e328cc8d 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
@@ -1 +1,5 @@
+# data_io classes for behavior only
+from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_lims_api import BehaviorLimsApi  # noqa: F401, E501
+
+# data_io classes for behavior + ophys
 from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_nwb_api import BehaviorOphysNwbApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_data_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_data_lims_api.py
deleted file mode 100644
index 939cdce01..000000000
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_data_lims_api.py
+++ /dev/null
@@ -1,516 +0,0 @@
-import numpy as np
-import pandas as pd
-import uuid
-from datetime import datetime
-import pytz
-
-from typing import Dict, Optional, Union, List, Any
-
-from allensdk.core.exceptions import DataFrameIndexError
-from allensdk.brain_observatory.behavior.session_apis.abcs import (
-    BehaviorBase)
-from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
-from allensdk.brain_observatory.behavior.running_processing import (
-    get_running_df)
-from allensdk.brain_observatory.behavior.stimulus_processing import (
-    get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata)
-from allensdk.brain_observatory.running_speed import RunningSpeed
-from allensdk.brain_observatory.behavior.metadata_processing import (
-    get_task_parameters)
-from allensdk.brain_observatory.behavior.sync import frame_time_offset
-from allensdk.brain_observatory.behavior.trials_processing import get_trials
-from allensdk.internal.core.lims_utilities import safe_system_path
-from allensdk.internal.api import PostgresQueryMixin
-from allensdk.api.cache import memoize
-from allensdk.internal.api import (
-    OneResultExpectedError, OneOrMoreResultExpectedError)
-from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin
-from allensdk.core.authentication import DbCredentials, credential_injector
-from allensdk.core.auth_config import (
-    LIMS_DB_CREDENTIAL_MAP, MTRAIN_DB_CREDENTIAL_MAP)
-
-
-class BehaviorDataLimsApi(CachedInstanceMethodMixin, BehaviorBase):
-    def __init__(self, behavior_session_id: int,
-                 lims_credentials: Optional[DbCredentials] = None,
-                 mtrain_credentials: Optional[DbCredentials] = None):
-        super().__init__()
-        if mtrain_credentials:
-            self.mtrain_db = PostgresQueryMixin(
-                dbname=mtrain_credentials.dbname, user=mtrain_credentials.user,
-                host=mtrain_credentials.host, port=mtrain_credentials.port,
-                password=mtrain_credentials.password)
-        else:
-            self.mtrain_db = (credential_injector(MTRAIN_DB_CREDENTIAL_MAP)
-                              (PostgresQueryMixin)())
-        if lims_credentials:
-            self.lims_db = PostgresQueryMixin(
-                dbname=lims_credentials.dbname, user=lims_credentials.user,
-                host=lims_credentials.host, port=lims_credentials.port,
-                password=lims_credentials.password)
-        else:
-            self.lims_db = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
-                            (PostgresQueryMixin)())
-
-        self.behavior_session_id = behavior_session_id
-        ids = self._get_ids()
-        self.ophys_experiment_ids = ids.get("ophys_experiment_ids")
-        self.ophys_session_id = ids.get("ophys_session_id")
-        self.behavior_training_id = ids.get("behavior_training_id")
-        self.foraging_id = ids.get("foraging_id")
-        self.ophys_container_id = ids.get("ophys_container_id")
-
-    def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]:
-        """Fetch ids associated with this behavior_session_id. If there is no
-        id, return None.
-        :returns: Dictionary of ids with the following keys:
-            behavior_training_id: int -- Only if was a training session
-            ophys_session_id: int -- None if have behavior_training_id
-            ophys_experiment_ids: List[int] -- only if have ophys_session_id
-            foraging_id: int
-        :rtype: dict
-        """
-        # Get all ids from the behavior_sessions table
-        query = f"""
-            SELECT
-                ophys_session_id, behavior_training_id, foraging_id
-            FROM
-                behavior_sessions
-            WHERE
-                behavior_sessions.id = {self.behavior_session_id};
-        """
-        ids_response = self.lims_db.select(query)
-        if len(ids_response) > 1:
-            raise OneResultExpectedError
-        ids_dict = ids_response.iloc[0].to_dict()
-
-        #  Get additional ids if also an ophys session
-        #     (experiment_id, container_id)
-        if ids_dict.get("ophys_session_id"):
-            oed_query = f"""
-                SELECT id
-                FROM ophys_experiments
-                WHERE ophys_session_id = {ids_dict["ophys_session_id"]};
-                """
-            oed = self.lims_db.fetchall(oed_query)
-            if len(oed) == 0:
-                oed = None
-
-            container_query = f"""
-            SELECT DISTINCT
-                visual_behavior_experiment_container_id id
-            FROM
-                ophys_experiments_visual_behavior_experiment_containers
-            WHERE
-                ophys_experiment_id IN ({",".join(set(map(str, oed)))});
-            """
-            try:
-                container_id = self.lims_db.fetchone(container_query, strict=True)
-            except OneResultExpectedError:
-                container_id = None
-
-            ids_dict.update({"ophys_experiment_ids": oed,
-                             "ophys_container_id": container_id})
-        else:
-            ids_dict.update({"ophys_experiment_ids": None,
-                             "ophys_container_id": None})
-        return ids_dict
-
-    def get_behavior_session_id(self) -> int:
-        """Getter to be consistent with BehaviorOphysLimsApi."""
-        return self.behavior_session_id
-
-    def get_behavior_session_uuid(self) -> Optional[int]:
-        data = self._behavior_stimulus_file()
-        return data.get("session_uuid")
-
-    def get_behavior_stimulus_file(self) -> str:
-        """Return the path to the StimulusPickle file for a session.
-        :rtype: str
-        """
-        query = f"""
-            SELECT
-                stim.storage_directory || stim.filename AS stim_file
-            FROM
-                well_known_files stim
-            WHERE
-                stim.attachable_id = {self.behavior_session_id}
-                AND stim.attachable_type = 'BehaviorSession'
-                AND stim.well_known_file_type_id IN (
-                    SELECT id
-                    FROM well_known_file_types
-                    WHERE name = 'StimulusPickle');
-        """
-        return safe_system_path(self.lims_db.fetchone(query, strict=True))
-
-    @memoize
-    def _behavior_stimulus_file(self) -> pd.DataFrame:
-        """Helper method to cache stimulus file in memory since it takes about
-        a second to load (and is used in many methods).
-        """
-        return pd.read_pickle(self.get_behavior_stimulus_file())
-
-    def get_licks(self) -> pd.DataFrame:
-        """Get lick data from pkl file.
-        This function assumes that the first sensor in the list of
-        lick_sensors is the desired lick sensor. If this changes we need
-        to update to get the proper line.
-
-        Since licks can occur outside of a trial context, the lick times
-        are extracted from the vsyncs and the frame number in `lick_events`.
-        Since we don't have a timestamp for when in "experiment time" the
-        vsync stream starts (from self.get_stimulus_timestamps), we compute
-        it by fitting a linear regression (frame number x time) for the
-        `start_trial` and `end_trial` events in the `trial_log`, to true
-        up these time streams.
-
-        :returns: pd.DataFrame -- A dataframe containing lick timestamps
-        """
-        # Get licks from pickle file instead of sync
-        data = self._behavior_stimulus_file()
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        lick_frames = (data["items"]["behavior"]["lick_sensors"][0]
-                       ["lick_events"])
-        lick_times = [stimulus_timestamps[frame] for frame in lick_frames]
-        return pd.DataFrame({"time": lick_times})
-
-    def get_rewards(self) -> pd.DataFrame:
-        """Get reward data from pkl file, based on pkl file timestamps
-        (not sync file).
-
-        :returns: pd.DataFrame -- A dataframe containing timestamps of
-            delivered rewards.
-        """
-        data = self._behavior_stimulus_file()
-        offset = frame_time_offset(data)
-        # No sync timestamps to rebase on, but do need to align to
-        # trial events, so add the offset as the "rebase" function
-        return get_rewards(data, lambda x: x + offset)
-
-    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
-        """Get running speed data.
-
-        :returns: pd.DataFrame -- dataframe containing various signals used
-            to compute running speed.
-        """
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        data = self._behavior_stimulus_file()
-        return get_running_df(data, stimulus_timestamps, lowpass=lowpass)
-
-    def get_running_speed(self, lowpass=True) -> RunningSpeed:
-        """Get running speed using timestamps from
-        self.get_stimulus_timestamps.
-
-        NOTE: Do not correct for monitor delay.
-
-        :returns: RunningSpeed -- a NamedTuple containing the subject's
-            timestamps and running speeds (in cm/s)
-        """
-        running_data_df = self.get_running_data_df(lowpass=lowpass)
-        if running_data_df.index.name != "timestamps":
-            raise DataFrameIndexError(
-                f"Expected index to be named 'timestamps' but got "
-                "'{running_data_df.index.name}'.")
-        return RunningSpeed(timestamps=running_data_df.index.values,
-                            values=running_data_df.speed.values)
-
-    def get_stimulus_frame_rate(self) -> float:
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        return np.round(1 / np.mean(np.diff(stimulus_timestamps)), 0)
-
-    def get_stimulus_presentations(self) -> pd.DataFrame:
-        """Get stimulus presentation data.
-
-        NOTE: Uses timestamps that do not account for monitor delay.
-
-        :returns: pd.DataFrame --
-            Table whose rows are stimulus presentations
-            (i.e. a given image, for a given duration, typically 250 ms)
-            and whose columns are presentation characteristics.
-        """
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        data = self._behavior_stimulus_file()
-        raw_stim_pres_df = get_stimulus_presentations(
-            data, stimulus_timestamps)
-
-        # Fill in nulls for image_name
-        # This makes two assumptions:
-        #   1. Nulls in `image_name` should be "gratings_<orientation>"
-        #   2. Gratings are only present (or need to be fixed) when all
-        #      values for `image_name` are null.
-        if pd.isnull(raw_stim_pres_df["image_name"]).all():
-            if ~pd.isnull(raw_stim_pres_df["orientation"]).all():
-                raw_stim_pres_df["image_name"] = (
-                    raw_stim_pres_df["orientation"]
-                    .apply(lambda x: f"gratings_{x}"))
-            else:
-                raise ValueError("All values for 'orentation' and 'image_name'"
-                                 " are null.")
-
-        stimulus_metadata_df = get_stimulus_metadata(data)
-        
-        idx_name = raw_stim_pres_df.index.name
-        stimulus_index_df = (
-            raw_stim_pres_df
-            .reset_index()
-            .merge(stimulus_metadata_df.reset_index(), on=["image_name"])
-            .set_index(idx_name))
-        stimulus_index_df = (
-            stimulus_index_df[["image_set", "image_index", "start_time",
-                               "phase", "spatial_frequency"]]
-            .rename(columns={"start_time": "timestamps"})
-            .sort_index()
-            .set_index("timestamps", drop=True))
-        stim_pres_df = raw_stim_pres_df.merge(
-            stimulus_index_df, left_on="start_time", right_index=True,
-            how="left")
-        if len(raw_stim_pres_df) != len(stim_pres_df):
-            raise ValueError("Length of `stim_pres_df` should not change after"
-                             f" merge; was {len(raw_stim_pres_df)}, now "
-                             f" {len(stim_pres_df)}.")
-        return stim_pres_df[sorted(stim_pres_df)]
-
-    def get_stimulus_templates(self) -> Dict[str, np.ndarray]:
-        """Get stimulus templates (movies, scenes) for behavior session.
-
-        Returns
-        -------
-        Dict[str, np.ndarray]
-            A dictionary containing the stimulus images presented during the
-            session. Keys are data set names, and values are 3D numpy arrays.
-        """
-        data = self._behavior_stimulus_file()
-        return get_stimulus_templates(data)
-
-    def get_stimulus_timestamps(self) -> np.ndarray:
-        """Get stimulus timestamps (vsyncs) from pkl file. Align to the
-        (frame, time) points in the trial events.
-
-        NOTE: Located with behavior_session_id. Does not use the sync_file
-        which requires ophys_session_id.
-
-        Returns
-        -------
-        np.ndarray
-            Timestamps associated with stimulus presentations on the monitor
-            that do no account for monitor delay.
-        """
-        data = self._behavior_stimulus_file()
-        vsyncs = data["items"]["behavior"]["intervalsms"]
-        cum_sum = np.hstack((0, vsyncs)).cumsum() / 1000.0  # cumulative time
-        offset = frame_time_offset(data)
-        return cum_sum + offset
-
-    def get_task_parameters(self) -> dict:
-        """Get task parameters from pkl file.
-
-        Returns
-        -------
-        dict
-            A dictionary containing parameters used to define the task runtime
-            behavior.
-        """
-        data = self._behavior_stimulus_file()
-        return get_task_parameters(data)
-
-    def get_trials(self) -> pd.DataFrame:
-        """Get trials from pkl file
-
-        Returns
-        -------
-        pd.DataFrame
-            A dataframe containing behavioral trial start/stop times,
-            and trial data
-        """
-        licks = self.get_licks()
-        data = self._behavior_stimulus_file()
-        rewards = self.get_rewards()
-        stimulus_presentations = self.get_stimulus_presentations()
-        # Pass a dummy rebase function since we don't have two time streams,
-        # and the frame times are already aligned to trial events in their
-        # respective getters
-        trial_df = get_trials(data, licks, rewards, stimulus_presentations,
-                              lambda x: x)
-
-        return trial_df
-
-    @memoize
-    def get_birth_date(self) -> datetime.date:
-        """Returns the birth date of the animal.
-        :rtype: datetime.date
-        """
-        query = f"""
-        SELECT d.date_of_birth
-        FROM behavior_sessions bs
-        JOIN donors d on d.id = bs.donor_id
-        WHERE bs.id = {self.behavior_session_id}
-        """
-        return self.lims_db.fetchone(query, strict=True).date()
-
-    @memoize
-    def get_sex(self) -> str:
-        """Returns sex of the animal (M/F)
-        :rtype: str
-        """
-        query = f"""
-            SELECT g.name AS sex
-            FROM behavior_sessions bs
-            JOIN donors d ON bs.donor_id = d.id
-            JOIN genders g ON g.id = d.gender_id
-            WHERE bs.id = {self.behavior_session_id};
-            """
-        return self.lims_db.fetchone(query, strict=True)
-
-    @memoize
-    def get_age(self) -> str:
-        """Returns age code of the subject.
-        :rtype: str
-        """
-        query = f"""
-            SELECT a.name AS age
-            FROM behavior_sessions bs
-            JOIN donors d ON d.id = bs.donor_id
-            JOIN ages a ON a.id = d.age_id
-            WHERE bs.id = {self.behavior_session_id};
-        """
-        return self.lims_db.fetchone(query, strict=True)
-
-    @memoize
-    def get_rig_name(self) -> str:
-        """Returns the name of the experimental rig.
-        :rtype: str
-        """
-        query = f"""
-            SELECT e.name AS device_name
-            FROM behavior_sessions bs
-            JOIN equipment e ON e.id = bs.equipment_id
-            WHERE bs.id = {self.behavior_session_id};
-        """
-        return self.lims_db.fetchone(query, strict=True)
-
-    @memoize
-    def get_stimulus_name(self) -> str:
-        """Returns the name of the stimulus set used for the session.
-        :rtype: str
-        """
-        query = f"""
-            SELECT stages.name
-            FROM behavior_sessions bs
-            JOIN stages ON stages.id = bs.state_id
-            WHERE bs.id = '{self.foraging_id}'
-        """
-        return self.mtrain_db.fetchone(query, strict=True)
-
-    @memoize
-    def get_reporter_line(self) -> List[str]:
-        """Returns the genotype name(s) of the reporter line(s).
-        :rtype: list
-        """
-        query = f"""
-            SELECT g.name AS reporter_line
-            FROM behavior_sessions bs
-            JOIN donors d ON bs.donor_id=d.id
-            JOIN donors_genotypes dg ON dg.donor_id=d.id
-            JOIN genotypes g ON g.id=dg.genotype_id
-            JOIN genotype_types gt
-                ON gt.id=g.genotype_type_id AND gt.name = 'reporter'
-            WHERE bs.id={self.behavior_session_id};
-        """
-        result = self.lims_db.fetchall(query)
-        if result is None or len(result) < 1:
-            raise OneOrMoreResultExpectedError(
-                f"Expected one or more, but received: '{result}' "
-                f"from query:\n'{query}'")
-        return result
-
-    @memoize
-    def get_driver_line(self) -> List[str]:
-        """Returns the genotype name(s) of the driver line(s).
-        :rtype: list
-        """
-        query = f"""
-            SELECT g.name AS driver_line
-            FROM behavior_sessions bs
-            JOIN donors d ON bs.donor_id=d.id
-            JOIN donors_genotypes dg ON dg.donor_id=d.id
-            JOIN genotypes g ON g.id=dg.genotype_id
-            JOIN genotype_types gt
-                ON gt.id=g.genotype_type_id AND gt.name = 'driver'
-            WHERE bs.id={self.behavior_session_id};
-        """
-        result = self.lims_db.fetchall(query)
-        if result is None or len(result) < 1:
-            raise OneOrMoreResultExpectedError(
-                f"Expected one or more, but received: '{result}' "
-                f"from query:\n'{query}'")
-        return result
-
-    @memoize
-    def get_external_specimen_name(self) -> int:
-        """Returns the LabTracks ID
-        :rtype: int
-        """
-        # TODO: Should this even be included?
-        # Found sometimes there were entries with NONE which is
-        # why they are filtered out; also many entries in the table
-        # match the donor_id, which is why used DISTINCT
-        query = f"""
-            SELECT DISTINCT(sp.external_specimen_name)
-            FROM behavior_sessions bs
-            JOIN donors d ON bs.donor_id=d.id
-            JOIN specimens sp ON sp.donor_id=d.id
-            WHERE bs.id={self.behavior_session_id}
-            AND sp.external_specimen_name IS NOT NULL;
-            """
-        return int(self.lims_db.fetchone(query, strict=True))
-
-    @memoize
-    def get_full_genotype(self) -> str:
-        """Return the name of the subject's genotype
-        :rtype: str
-        """
-        query = f"""
-                SELECT d.full_genotype
-                FROM behavior_sessions bs
-                JOIN donors d ON d.id=bs.donor_id
-                WHERE bs.id= {self.behavior_session_id};
-                """
-        return self.lims_db.fetchone(query, strict=True)
-
-    @memoize
-    def get_experiment_date(self) -> datetime:
-        """Return timestamp the behavior stimulus file began recording in UTC
-        :rtype: datetime
-        """
-        data = self._behavior_stimulus_file()
-        # Assuming file has local time of computer (Seattle)
-        tz = pytz.timezone("America/Los_Angeles")
-        return tz.localize(data["start_time"]).astimezone(pytz.utc)
-
-    def get_metadata(self) -> Dict[str, Any]:
-        """Return metadata about the session.
-        :rtype: dict
-        """
-        if self.get_behavior_session_uuid() is None:
-            bs_uuid = None
-        else:
-            bs_uuid = uuid.UUID(self.get_behavior_session_uuid())
-        metadata = {
-            "rig_name": self.get_rig_name(),
-            "sex": self.get_sex(),
-            "age": self.get_age(),
-            "ophys_experiment_id": self.ophys_experiment_ids,
-            "experiment_container_id": self.ophys_container_id,
-            "stimulus_frame_rate": self.get_stimulus_frame_rate(),
-            "session_type": self.get_stimulus_name(),
-            "experiment_datetime": self.get_experiment_date(),
-            "reporter_line": self.get_reporter_line(),
-            "driver_line": self.get_driver_line(),
-            "LabTracks_ID": self.get_external_specimen_name(),
-            "full_genotype": self.get_full_genotype(),
-            "behavior_session_uuid": bs_uuid,
-            "foraging_id": self.foraging_id,
-            "behavior_session_id": self.behavior_session_id,
-            "behavior_training_id": self.behavior_training_id,
-        }
-        return metadata
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index 6dbddb2fe..23527f259 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -1,82 +1,555 @@
-
-from typing import Optional
-
-from ..core import lims_utilities
-from . import PostgresQueryMixin
+import numpy as np
 import pandas as pd
+import uuid
+from datetime import datetime
+import pytz
 
-from allensdk.api.cache import memoize
-from allensdk.internal.api import PostgresQueryMixin
-from allensdk.brain_observatory.behavior.trials_processing import get_extended_trials
+from typing import Dict, Optional, Union, List, Any
+
+from allensdk.core.exceptions import DataFrameIndexError
+from allensdk.brain_observatory.behavior.session_apis.abcs import (
+    BehaviorBase)
+from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
+from allensdk.brain_observatory.behavior.running_processing import (
+    get_running_df)
+from allensdk.brain_observatory.behavior.stimulus_processing import (
+    get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata)
+from allensdk.brain_observatory.running_speed import RunningSpeed
+from allensdk.brain_observatory.behavior.metadata_processing import (
+    get_task_parameters)
+from allensdk.brain_observatory.behavior.sync import frame_time_offset
+from allensdk.brain_observatory.behavior.trials_processing import get_trials
 from allensdk.internal.core.lims_utilities import safe_system_path
-from allensdk.core.authentication import DbCredentials, credential_injector
-from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
+from allensdk.internal.api import db_connection_creator
+from allensdk.api.cache import memoize
+from allensdk.internal.api import (
+    OneResultExpectedError, OneOrMoreResultExpectedError)
+from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin
+from allensdk.core.authentication import DbCredentials
+from allensdk.core.auth_config import (
+    LIMS_DB_CREDENTIAL_MAP, MTRAIN_DB_CREDENTIAL_MAP)
+
+
+class BehaviorLimsApi(BehaviorBase, CachedInstanceMethodMixin):
+    """A data fetching class that serves as an API for filling
+    'BehaviorSession' attributes from LIMS.
+    """
+    def __init__(self, behavior_session_id: int,
+                 lims_credentials: Optional[DbCredentials] = None,
+                 mtrain_credentials: Optional[DbCredentials] = None):
+        super().__init__()
+
+        self.mtrain_db = db_connection_creator(
+            credentials=mtrain_credentials,
+            default_credentials=MTRAIN_DB_CREDENTIAL_MAP)
+
+        self.lims_db = db_connection_creator(
+            credentials=lims_credentials,
+            default_credentials=LIMS_DB_CREDENTIAL_MAP)
+
+        self.behavior_session_id = behavior_session_id
+        ids = self._get_ids()
+        self.ophys_experiment_ids = ids.get("ophys_experiment_ids")
+        self.ophys_session_id = ids.get("ophys_session_id")
+        self.behavior_training_id = ids.get("behavior_training_id")
+        self.foraging_id = ids.get("foraging_id")
+        self.ophys_container_id = ids.get("ophys_container_id")
+
+    @classmethod
+    def from_foraging_id(cls,
+                         foraging_id: Union[str, uuid.UUID, int],
+                         lims_credentials: Optional[DbCredentials] = None
+                         ) -> "BehaviorLimsApi":
+        """Create a BehaviorLimsAPI instance from a foraging_id instead of
+        a behavior_session_id.
+
+        NOTE: 'foraging_id' in the LIMS behavior_session table should be
+              the same as the 'behavior_session_uuid' in mtrain which should
+              also be the same as the 'session_uuid' field in the .pkl
+              returned by 'get_behavior_stimulus_file()'.
+        """
+
+        lims_db = db_connection_creator(
+            credentials=lims_credentials,
+            default_credentials=LIMS_DB_CREDENTIAL_MAP)
 
+        if isinstance(foraging_id, uuid.UUID):
+            foraging_id = str(foraging_id)
+        elif isinstance(foraging_id, int):
+            foraging_id = str(uuid.UUID(int=foraging_id))
 
-class BehaviorLimsApi:
+        query = f"""
+            SELECT id
+            FROM behavior_sessions
+            WHERE foraging_id = '{foraging_id}'
+        """
+        session_id = lims_db.fetchone(query, strict=True)
+        return cls(session_id, lims_credentials=lims_credentials)
 
-    def __init__(self, behavior_experiment_id: int,
-                 lims_credentials: Optional[DbCredentials] = None):
+    def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]:
+        """Fetch ids associated with this behavior_session_id. If there is no
+        id, return None.
+        :returns: Dictionary of ids with the following keys:
+            behavior_training_id: int -- Only if was a training session
+            ophys_session_id: int -- None if have behavior_training_id
+            ophys_experiment_ids: List[int] -- only if have ophys_session_id
+            foraging_id: int
+        :rtype: dict
         """
-        Notes
-        -----
-        - behavior_experiment_id is the same as behavior_session_id which is in lims
-        - behavior_experiment_id is associated with foraging_id in lims
-        - foraging_id in lims is the same as behavior_session_uuid in mtrain which is the same
-        as session_uuid in the pickle returned by behavior_stimulus_file
+        # Get all ids from the behavior_sessions table
+        query = f"""
+            SELECT
+                ophys_session_id, behavior_training_id, foraging_id
+            FROM
+                behavior_sessions
+            WHERE
+                behavior_sessions.id = {self.behavior_session_id};
         """
-        self.behavior_experiment_id = behavior_experiment_id
-        if lims_credentials:
-            self.lims_db = PostgresQueryMixin(
-                dbname=lims_credentials.dbname, user=lims_credentials.user,
-                host=lims_credentials.host, password=lims_credentials.password,
-                port=lims_credentials.port)
+        ids_response = self.lims_db.select(query)
+        if len(ids_response) > 1:
+            raise OneResultExpectedError
+        ids_dict = ids_response.iloc[0].to_dict()
+
+        #  Get additional ids if also an ophys session
+        #     (experiment_id, container_id)
+        if ids_dict.get("ophys_session_id"):
+            oed_query = f"""
+                SELECT id
+                FROM ophys_experiments
+                WHERE ophys_session_id = {ids_dict["ophys_session_id"]};
+                """
+            oed = self.lims_db.fetchall(oed_query)
+            if len(oed) == 0:
+                oed = None
+
+            container_query = f"""
+            SELECT DISTINCT
+                visual_behavior_experiment_container_id id
+            FROM
+                ophys_experiments_visual_behavior_experiment_containers
+            WHERE
+                ophys_experiment_id IN ({",".join(set(map(str, oed)))});
+            """
+            try:
+                container_id = self.lims_db.fetchone(container_query,
+                                                     strict=True)
+            except OneResultExpectedError:
+                container_id = None
+
+            ids_dict.update({"ophys_experiment_ids": oed,
+                             "ophys_container_id": container_id})
         else:
-            # Use default credentials from provider
-            # Currying is equivalent to decorator syntactic sugar
-            self.lims_db = (
-                credential_injector(LIMS_DB_CREDENTIAL_MAP)
-                (PostgresQueryMixin)())
+            ids_dict.update({"ophys_experiment_ids": None,
+                             "ophys_container_id": None})
+        return ids_dict
 
-    def get_behavior_experiment_id(self):
-        return self.behavior_experiment_id
+    def get_behavior_session_id(self) -> int:
+        """Getter to be consistent with BehaviorOphysLimsApi."""
+        return self.behavior_session_id
 
-    @memoize
-    def get_behavior_stimulus_file(self):
-        query = '''
-                SELECT stim.storage_directory || stim.filename AS stim_file 
-                FROM behavior_sessions bs 
-                LEFT JOIN well_known_files stim ON stim.attachable_id=bs.id AND stim.attachable_type = 'BehaviorSession' AND stim.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'StimulusPickle') 
-                WHERE bs.id= {};
-                '''.format(self.get_behavior_experiment_id())
+    def get_behavior_session_uuid(self) -> Optional[int]:
+        """Get the universally unique identifier (UUID) number for the
+        current behavior session.
+        """
+        data = self._behavior_stimulus_file()
+        behavior_pkl_uuid = data.get("session_uuid")
+        # Sanity check to ensure that pkl file data matches up with
+        # the behavior session that the pkl file has been associated with.
+        assert_err_msg = (
+            f"The behavior session UUID ({behavior_pkl_uuid}) in the behavior "
+            f"stimulus *.pkl file ({self.get_behavior_stimulus_file()}) does "
+            f"does not match the LIMS UUID ({self.foraging_id}) for "
+            f"behavior session: {self.behavior_session_id}")
+        assert behavior_pkl_uuid == self.foraging_id, assert_err_msg
+        return behavior_pkl_uuid
+
+    def get_behavior_stimulus_file(self) -> str:
+        """Return the path to the StimulusPickle file for a session.
+        :rtype: str
+        """
+        query = f"""
+            SELECT
+                stim.storage_directory || stim.filename AS stim_file
+            FROM
+                well_known_files stim
+            WHERE
+                stim.attachable_id = {self.behavior_session_id}
+                AND stim.attachable_type = 'BehaviorSession'
+                AND stim.well_known_file_type_id IN (
+                    SELECT id
+                    FROM well_known_file_types
+                    WHERE name = 'StimulusPickle');
+        """
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
-    def get_extended_trials(self):
-        filename = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(filename)
-        return get_extended_trials(data)
-
-    @staticmethod
-    def foraging_id_to_behavior_session_id(foraging_id):
-        '''maps foraging_id to behavior_session_id'''
-        api = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
-               (PostgresQueryMixin)())
-        query = '''select id from behavior_sessions where foraging_id = '{}';'''.format(
-            foraging_id)
-        return api.fetchone(query, strict=True)
-
-    @staticmethod
-    def behavior_session_id_to_foraging_id(behavior_session_id):
-        '''maps behavior_session_id to foraging_id'''
-        api = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
-               (PostgresQueryMixin)())
-        query = '''select foraging_id from behavior_sessions where id = '{}';'''.format(
-            behavior_session_id)
-        return api.fetchone(query, strict=True)
+    @memoize
+    def _behavior_stimulus_file(self) -> pd.DataFrame:
+        """Helper method to cache stimulus file in memory since it takes about
+        a second to load (and is used in many methods).
+        """
+        return pd.read_pickle(self.get_behavior_stimulus_file())
 
-    @classmethod
-    def from_foraging_id(cls, foraging_id: str,
-                         lims_credentials: Optional[DbCredentials] = None):
-        return cls(
-            behavior_experiment_id=cls.foraging_id_to_behavior_session_id(
-                foraging_id, lims_credentials))
+    def get_licks(self) -> pd.DataFrame:
+        """Get lick data from pkl file.
+        This function assumes that the first sensor in the list of
+        lick_sensors is the desired lick sensor. If this changes we need
+        to update to get the proper line.
+
+        Since licks can occur outside of a trial context, the lick times
+        are extracted from the vsyncs and the frame number in `lick_events`.
+        Since we don't have a timestamp for when in "experiment time" the
+        vsync stream starts (from self.get_stimulus_timestamps), we compute
+        it by fitting a linear regression (frame number x time) for the
+        `start_trial` and `end_trial` events in the `trial_log`, to true
+        up these time streams.
+
+        :returns: pd.DataFrame -- A dataframe containing lick timestamps
+        """
+        # Get licks from pickle file instead of sync
+        data = self._behavior_stimulus_file()
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        lick_frames = (data["items"]["behavior"]["lick_sensors"][0]
+                       ["lick_events"])
+        lick_times = [stimulus_timestamps[frame] for frame in lick_frames]
+        return pd.DataFrame({"time": lick_times})
+
+    def get_rewards(self) -> pd.DataFrame:
+        """Get reward data from pkl file, based on pkl file timestamps
+        (not sync file).
+
+        :returns: pd.DataFrame -- A dataframe containing timestamps of
+            delivered rewards.
+        """
+        data = self._behavior_stimulus_file()
+        offset = frame_time_offset(data)
+        # No sync timestamps to rebase on, but do need to align to
+        # trial events, so add the offset as the "rebase" function
+        return get_rewards(data, lambda x: x + offset)
+
+    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
+        """Get running speed data.
+
+        :returns: pd.DataFrame -- dataframe containing various signals used
+            to compute running speed.
+        """
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        data = self._behavior_stimulus_file()
+        return get_running_df(data, stimulus_timestamps, lowpass=lowpass)
+
+    def get_running_speed(self, lowpass=True) -> RunningSpeed:
+        """Get running speed using timestamps from
+        self.get_stimulus_timestamps.
+
+        NOTE: Do not correct for monitor delay.
+
+        :returns: RunningSpeed -- a NamedTuple containing the subject's
+            timestamps and running speeds (in cm/s)
+        """
+        running_data_df = self.get_running_data_df(lowpass=lowpass)
+        if running_data_df.index.name != "timestamps":
+            raise DataFrameIndexError(
+                f"Expected index to be named 'timestamps' but got "
+                "'{running_data_df.index.name}'.")
+        return RunningSpeed(timestamps=running_data_df.index.values,
+                            values=running_data_df.speed.values)
+
+    def get_stimulus_frame_rate(self) -> float:
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        return np.round(1 / np.mean(np.diff(stimulus_timestamps)), 0)
+
+    def get_stimulus_presentations(self) -> pd.DataFrame:
+        """Get stimulus presentation data.
+
+        NOTE: Uses timestamps that do not account for monitor delay.
+
+        :returns: pd.DataFrame --
+            Table whose rows are stimulus presentations
+            (i.e. a given image, for a given duration, typically 250 ms)
+            and whose columns are presentation characteristics.
+        """
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        data = self._behavior_stimulus_file()
+        raw_stim_pres_df = get_stimulus_presentations(
+            data, stimulus_timestamps)
+
+        # Fill in nulls for image_name
+        # This makes two assumptions:
+        #   1. Nulls in `image_name` should be "gratings_<orientation>"
+        #   2. Gratings are only present (or need to be fixed) when all
+        #      values for `image_name` are null.
+        if pd.isnull(raw_stim_pres_df["image_name"]).all():
+            if ~pd.isnull(raw_stim_pres_df["orientation"]).all():
+                raw_stim_pres_df["image_name"] = (
+                    raw_stim_pres_df["orientation"]
+                    .apply(lambda x: f"gratings_{x}"))
+            else:
+                raise ValueError("All values for 'orentation' and 'image_name'"
+                                 " are null.")
+
+        stimulus_metadata_df = get_stimulus_metadata(data)
+
+        idx_name = raw_stim_pres_df.index.name
+        stimulus_index_df = (
+            raw_stim_pres_df
+            .reset_index()
+            .merge(stimulus_metadata_df.reset_index(), on=["image_name"])
+            .set_index(idx_name))
+        stimulus_index_df = (
+            stimulus_index_df[["image_set", "image_index", "start_time",
+                               "phase", "spatial_frequency"]]
+            .rename(columns={"start_time": "timestamps"})
+            .sort_index()
+            .set_index("timestamps", drop=True))
+        stim_pres_df = raw_stim_pres_df.merge(
+            stimulus_index_df, left_on="start_time", right_index=True,
+            how="left")
+        if len(raw_stim_pres_df) != len(stim_pres_df):
+            raise ValueError("Length of `stim_pres_df` should not change after"
+                             f" merge; was {len(raw_stim_pres_df)}, now "
+                             f" {len(stim_pres_df)}.")
+        return stim_pres_df[sorted(stim_pres_df)]
+
+    def get_stimulus_templates(self) -> Dict[str, np.ndarray]:
+        """Get stimulus templates (movies, scenes) for behavior session.
+
+        Returns
+        -------
+        Dict[str, np.ndarray]
+            A dictionary containing the stimulus images presented during the
+            session. Keys are data set names, and values are 3D numpy arrays.
+        """
+        data = self._behavior_stimulus_file()
+        return get_stimulus_templates(data)
+
+    def get_stimulus_timestamps(self) -> np.ndarray:
+        """Get stimulus timestamps (vsyncs) from pkl file. Align to the
+        (frame, time) points in the trial events.
+
+        NOTE: Located with behavior_session_id. Does not use the sync_file
+        which requires ophys_session_id.
+
+        Returns
+        -------
+        np.ndarray
+            Timestamps associated with stimulus presentations on the monitor
+            that do no account for monitor delay.
+        """
+        data = self._behavior_stimulus_file()
+        vsyncs = data["items"]["behavior"]["intervalsms"]
+        cum_sum = np.hstack((0, vsyncs)).cumsum() / 1000.0  # cumulative time
+        offset = frame_time_offset(data)
+        return cum_sum + offset
+
+    def get_task_parameters(self) -> dict:
+        """Get task parameters from pkl file.
+
+        Returns
+        -------
+        dict
+            A dictionary containing parameters used to define the task runtime
+            behavior.
+        """
+        data = self._behavior_stimulus_file()
+        return get_task_parameters(data)
+
+    def get_trials(self) -> pd.DataFrame:
+        """Get trials from pkl file
+
+        Returns
+        -------
+        pd.DataFrame
+            A dataframe containing behavioral trial start/stop times,
+            and trial data
+        """
+        licks = self.get_licks()
+        data = self._behavior_stimulus_file()
+        rewards = self.get_rewards()
+        stimulus_presentations = self.get_stimulus_presentations()
+        # Pass a dummy rebase function since we don't have two time streams,
+        # and the frame times are already aligned to trial events in their
+        # respective getters
+        trial_df = get_trials(data, licks, rewards, stimulus_presentations,
+                              lambda x: x)
+
+        return trial_df
+
+    @memoize
+    def get_birth_date(self) -> datetime.date:
+        """Returns the birth date of the animal.
+        :rtype: datetime.date
+        """
+        query = f"""
+        SELECT d.date_of_birth
+        FROM behavior_sessions bs
+        JOIN donors d on d.id = bs.donor_id
+        WHERE bs.id = {self.behavior_session_id}
+        """
+        return self.lims_db.fetchone(query, strict=True).date()
+
+    @memoize
+    def get_sex(self) -> str:
+        """Returns sex of the animal (M/F)
+        :rtype: str
+        """
+        query = f"""
+            SELECT g.name AS sex
+            FROM behavior_sessions bs
+            JOIN donors d ON bs.donor_id = d.id
+            JOIN genders g ON g.id = d.gender_id
+            WHERE bs.id = {self.behavior_session_id};
+            """
+        return self.lims_db.fetchone(query, strict=True)
+
+    @memoize
+    def get_age(self) -> str:
+        """Returns age code of the subject.
+        :rtype: str
+        """
+        query = f"""
+            SELECT a.name AS age
+            FROM behavior_sessions bs
+            JOIN donors d ON d.id = bs.donor_id
+            JOIN ages a ON a.id = d.age_id
+            WHERE bs.id = {self.behavior_session_id};
+        """
+        return self.lims_db.fetchone(query, strict=True)
+
+    @memoize
+    def get_rig_name(self) -> str:
+        """Returns the name of the experimental rig.
+        :rtype: str
+        """
+        query = f"""
+            SELECT e.name AS device_name
+            FROM behavior_sessions bs
+            JOIN equipment e ON e.id = bs.equipment_id
+            WHERE bs.id = {self.behavior_session_id};
+        """
+        return self.lims_db.fetchone(query, strict=True)
+
+    @memoize
+    def get_stimulus_name(self) -> str:
+        """Returns the name of the stimulus set used for the session.
+        :rtype: str
+        """
+        query = f"""
+            SELECT stages.name
+            FROM behavior_sessions bs
+            JOIN stages ON stages.id = bs.state_id
+            WHERE bs.id = '{self.foraging_id}'
+        """
+        return self.mtrain_db.fetchone(query, strict=True)
+
+    @memoize
+    def get_reporter_line(self) -> List[str]:
+        """Returns the genotype name(s) of the reporter line(s).
+        :rtype: list
+        """
+        query = f"""
+            SELECT g.name AS reporter_line
+            FROM behavior_sessions bs
+            JOIN donors d ON bs.donor_id=d.id
+            JOIN donors_genotypes dg ON dg.donor_id=d.id
+            JOIN genotypes g ON g.id=dg.genotype_id
+            JOIN genotype_types gt
+                ON gt.id=g.genotype_type_id AND gt.name = 'reporter'
+            WHERE bs.id={self.behavior_session_id};
+        """
+        result = self.lims_db.fetchall(query)
+        if result is None or len(result) < 1:
+            raise OneOrMoreResultExpectedError(
+                f"Expected one or more, but received: '{result}' "
+                f"from query:\n'{query}'")
+        return result
+
+    @memoize
+    def get_driver_line(self) -> List[str]:
+        """Returns the genotype name(s) of the driver line(s).
+        :rtype: list
+        """
+        query = f"""
+            SELECT g.name AS driver_line
+            FROM behavior_sessions bs
+            JOIN donors d ON bs.donor_id=d.id
+            JOIN donors_genotypes dg ON dg.donor_id=d.id
+            JOIN genotypes g ON g.id=dg.genotype_id
+            JOIN genotype_types gt
+                ON gt.id=g.genotype_type_id AND gt.name = 'driver'
+            WHERE bs.id={self.behavior_session_id};
+        """
+        result = self.lims_db.fetchall(query)
+        if result is None or len(result) < 1:
+            raise OneOrMoreResultExpectedError(
+                f"Expected one or more, but received: '{result}' "
+                f"from query:\n'{query}'")
+        return result
+
+    @memoize
+    def get_external_specimen_name(self) -> int:
+        """Returns the LabTracks ID
+        :rtype: int
+        """
+        # TODO: Should this even be included?
+        # Found sometimes there were entries with NONE which is
+        # why they are filtered out; also many entries in the table
+        # match the donor_id, which is why used DISTINCT
+        query = f"""
+            SELECT DISTINCT(sp.external_specimen_name)
+            FROM behavior_sessions bs
+            JOIN donors d ON bs.donor_id=d.id
+            JOIN specimens sp ON sp.donor_id=d.id
+            WHERE bs.id={self.behavior_session_id}
+            AND sp.external_specimen_name IS NOT NULL;
+            """
+        return int(self.lims_db.fetchone(query, strict=True))
+
+    @memoize
+    def get_full_genotype(self) -> str:
+        """Return the name of the subject's genotype
+        :rtype: str
+        """
+        query = f"""
+                SELECT d.full_genotype
+                FROM behavior_sessions bs
+                JOIN donors d ON d.id=bs.donor_id
+                WHERE bs.id= {self.behavior_session_id};
+                """
+        return self.lims_db.fetchone(query, strict=True)
+
+    @memoize
+    def get_experiment_date(self) -> datetime:
+        """Return timestamp the behavior stimulus file began recording in UTC
+        :rtype: datetime
+        """
+        data = self._behavior_stimulus_file()
+        # Assuming file has local time of computer (Seattle)
+        tz = pytz.timezone("America/Los_Angeles")
+        return tz.localize(data["start_time"]).astimezone(pytz.utc)
+
+    def get_metadata(self) -> Dict[str, Any]:
+        """Return metadata about the session.
+        :rtype: dict
+        """
+        if self.get_behavior_session_uuid() is None:
+            bs_uuid = None
+        else:
+            bs_uuid = uuid.UUID(self.get_behavior_session_uuid())
+        metadata = {
+            "rig_name": self.get_rig_name(),
+            "sex": self.get_sex(),
+            "age": self.get_age(),
+            "ophys_experiment_id": self.ophys_experiment_ids,
+            "experiment_container_id": self.ophys_container_id,
+            "stimulus_frame_rate": self.get_stimulus_frame_rate(),
+            "session_type": self.get_stimulus_name(),
+            "experiment_datetime": self.get_experiment_date(),
+            "reporter_line": self.get_reporter_line(),
+            "driver_line": self.get_driver_line(),
+            "LabTracks_ID": self.get_external_specimen_name(),
+            "full_genotype": self.get_full_genotype(),
+            "behavior_session_uuid": bs_uuid,
+            "foraging_id": self.foraging_id,
+            "behavior_session_id": self.behavior_session_id,
+            "behavior_training_id": self.behavior_training_id,
+        }
+        return metadata
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_processing/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_processing/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/allensdk/internal/api/__init__.py b/allensdk/internal/api/__init__.py
index 5ea790c6c..bb2fbd7bd 100644
--- a/allensdk/internal/api/__init__.py
+++ b/allensdk/internal/api/__init__.py
@@ -1,9 +1,12 @@
+from typing import Optional
+
 import psycopg2
 import psycopg2.extras
 import pandas as pd
 
 from allensdk import one, OneResultExpectedError
-from allensdk.core.authentication import credential_injector
+from allensdk.core.authentication import DbCredentials, credential_injector
+
 
 class OneOrMoreResultExpectedError(RuntimeError):
     pass
@@ -12,7 +15,8 @@ class OneOrMoreResultExpectedError(RuntimeError):
 def psycopg2_select(query, database, host, port, username, password):
 
     connection = psycopg2.connect(
-        host=host, port=port, dbname=database, user=username, password=password,
+        host=host, port=port, dbname=database,
+        user=username, password=password,
         cursor_factory=psycopg2.extras.RealDictCursor
     )
     cursor = connection.cursor()
@@ -40,7 +44,9 @@ def get_cursor(self):
         return self.get_connection().cursor()
 
     def get_connection(self):
-        return psycopg2.connect(dbname=self.dbname, user=self.user, host=self.host, password=self.password, port=self.port)
+        return psycopg2.connect(dbname=self.dbname, user=self.user,
+                                host=self.host, password=self.password,
+                                port=self.port)
 
     def fetchone(self, query, strict=True):
         response = one(list(self.select(query).to_dict().values()))
@@ -53,11 +59,12 @@ def fetchall(self, query, strict=True):
         return [one(x) for x in response.values.flat]
 
     def select(self, query):
-        return psycopg2_select(query, 
-            database=self.dbname, 
-            host=self.host, 
-            port=self.port, 
-            username=self.user, 
+        return psycopg2_select(
+            query,
+            database=self.dbname,
+            host=self.host,
+            port=self.port,
+            username=self.user,
             password=self.password
         )
 
@@ -66,3 +73,35 @@ def select_one(self, query):
         if len(data) == 1:
             return data[0]
         return {}
+
+
+def db_connection_creator(default_credentials: dict,
+                          credentials: Optional[DbCredentials] = None,
+                          ) -> PostgresQueryMixin:
+    """Create a db connection using either default credentials or
+    custom credentials.
+
+    Parameters
+    ----------
+    default_credentials : dict
+        Default credentials to use for creating the DB connection.
+        Some options can be found in allensdk.core.auth_config.
+    credentials : Optional[DbCredentials], optional
+        User specified credentials, by default None
+
+    Returns
+    -------
+    PostgresQueryMixin
+        A DB connection instance which can execute queries to the DB
+        specified by credentials or default_credentials.
+    """
+    if credentials:
+        db_conn = PostgresQueryMixin(
+            dbname=credentials.dbname, user=credentials.user,
+            host=credentials.host, port=credentials.port,
+            password=credentials.password)
+    else:
+        db_conn = (credential_injector(default_credentials)
+                   (PostgresQueryMixin)())
+
+    return db_conn
diff --git a/allensdk/internal/api/mtrain_api.py b/allensdk/internal/api/mtrain_api.py
index ce41ecae9..4f805d633 100644
--- a/allensdk/internal/api/mtrain_api.py
+++ b/allensdk/internal/api/mtrain_api.py
@@ -7,7 +7,8 @@
 import uuid
 
 from . import PostgresQueryMixin
-from .behavior_lims_api import BehaviorLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorLimsApi)
 from allensdk.brain_observatory.behavior.trials_processing import EDF_COLUMNS
 from allensdk.core.auth_config import MTRAIN_DB_CREDENTIAL_MAP
 from allensdk.core.authentication import credential_injector
@@ -51,15 +52,14 @@ def get_session(self, behavior_session_uuid=None, behavior_session_id=None):
         assert not all(v is None for v in [
                        behavior_session_uuid, behavior_session_id]), 'must enter either a behavior_session_uuid or a behavior_session_id'
 
+        behavior_data = BehaviorLimsApi(behavior_session_id)
         if behavior_session_uuid is None and behavior_session_id is not None:
             # get a behavior session uuid if a lims ID was entered
-            behavior_session_uuid = BehaviorLimsApi.behavior_session_id_to_foraging_id(
-                behavior_session_id)
-
+            behavior_session_uuid = behavior_data.get_behavior_session_uuid()
         if behavior_session_uuid is not None and behavior_session_id is not None:
             # if both a behavior session uuid and a lims id are entered, ensure that they match
-            assert behavior_session_uuid == BehaviorLimsApi.behavior_session_id_to_foraging_id(
-                behavior_session_id), 'behavior_session {} does not match behavior_session_id {}'.format(behavior_session_uuid, behavior_session_id)
+            assert behavior_session_uuid == behavior_data.get_behavior_session_uuid(), 'behavior_session {} does not match behavior_session_id {}'.format(behavior_session_uuid, behavior_session_id)
+
         filters = [{"name": "id", "op": "eq", "val": behavior_session_uuid}]
         behavior_df = self.get_df('behavior_sessions', filters=filters).rename(columns={'id': 'behavior_session_uuid'})
         state_df = self.get_df('states').rename(columns={'id': 'state_id'})
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
index 6eb3a07cb..a0724c502 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
@@ -5,7 +5,9 @@
 import pytz
 import math
 
-from allensdk.internal.api.behavior_data_lims_api import BehaviorDataLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorLimsApi
+)
 from allensdk.core.authentication import DbCredentials
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.running_speed import RunningSpeed
@@ -17,9 +19,9 @@
 
 
 @pytest.fixture
-def MockBehaviorDataLimsApi():
+def MockBehaviorLimsApi():
 
-    class MockBehaviorDataLimsApi(BehaviorDataLimsApi):
+    class MockBehaviorLimsApi(BehaviorLimsApi):
         """
         Mock class that overrides some functions to provide test data and
         initialize without calls to db.
@@ -68,14 +70,14 @@ def get_running_data_df(self, lowpass=True):
                 {"timestamps": [0.0, 0.1, 0.2],
                  "speed": [8.0, 15.0, 16.0]}).set_index("timestamps")
 
-    api = MockBehaviorDataLimsApi()
+    api = MockBehaviorLimsApi()
     yield api
     api.cache_clear()
 
 
 @pytest.fixture
 def MockApiRunSpeedExpectedError():
-    class MockApiRunSpeedExpectedError(BehaviorDataLimsApi):
+    class MockApiRunSpeedExpectedError(BehaviorLimsApi):
         """
         Mock class that overrides some functions to provide test data and 
         initialize without calls to db.
@@ -106,39 +108,39 @@ def get_running_data_df(self, lowpass=True):
 #    get_trials
 # Does not include test for get_metadata since it just collects data from
 # methods covered in other unit tests, or data derived from sql queries.
-def test_get_stimulus_timestamps(MockBehaviorDataLimsApi):
-    api = MockBehaviorDataLimsApi
+def test_get_stimulus_timestamps(MockBehaviorLimsApi):
+    api = MockBehaviorLimsApi
     expected = np.array([0.016 * i for i in range(11)])
     assert np.allclose(expected, api.get_stimulus_timestamps())
 
 
-def test_get_licks(MockBehaviorDataLimsApi):
-    api = MockBehaviorDataLimsApi
+def test_get_licks(MockBehaviorLimsApi):
+    api = MockBehaviorLimsApi
     expected = pd.DataFrame({"time": [0.016 * i for i in [2., 6., 9.]]})
     pd.testing.assert_frame_equal(expected, api.get_licks())
 
 
-def test_get_behavior_session_uuid(MockBehaviorDataLimsApi):
-    api = MockBehaviorDataLimsApi
+def test_get_behavior_session_uuid(MockBehaviorLimsApi):
+    api = MockBehaviorLimsApi
     assert 123456 == api.get_behavior_session_uuid()
 
 
-def test_get_stimulus_frame_rate(MockBehaviorDataLimsApi):
-    api = MockBehaviorDataLimsApi
+def test_get_stimulus_frame_rate(MockBehaviorLimsApi):
+    api = MockBehaviorLimsApi
     assert 62.0 == api.get_stimulus_frame_rate()
 
 
-def test_get_experiment_date(MockBehaviorDataLimsApi):
-    api = MockBehaviorDataLimsApi
+def test_get_experiment_date(MockBehaviorLimsApi):
+    api = MockBehaviorLimsApi
     expected = datetime(2019, 9, 26, 16, tzinfo=pytz.UTC)
     actual = api.get_experiment_date()
     assert expected == actual
 
 
-def test_get_running_speed(MockBehaviorDataLimsApi):
+def test_get_running_speed(MockBehaviorLimsApi):
     expected = RunningSpeed(timestamps=[0.0, 0.1, 0.2],
                             values=[8.0, 15.0, 16.0])
-    api = MockBehaviorDataLimsApi
+    api = MockBehaviorLimsApi
     actual = api.get_running_speed()
     assert expected == actual
 
@@ -148,8 +150,8 @@ def test_get_running_speed_raises_index_error(MockApiRunSpeedExpectedError):
         MockApiRunSpeedExpectedError.get_running_speed()
 
 
-# def test_get_stimulus_presentations(MockBehaviorDataLimsApi):
-#     api = MockBehaviorDataLimsApi
+# def test_get_stimulus_presentations(MockBehaviorLimsApi):
+#     api = MockBehaviorLimsApi
 #     #  TODO. This function is a monster with multiple dependencies,
 #     #  no tests, and no documentation (for any of its dependencies).
 #     #  Needs to be broken out into testable parts.
@@ -159,7 +161,7 @@ def test_get_running_speed_raises_index_error(MockApiRunSpeedExpectedError):
 class TestBehaviorRegression:
     """
     Test whether behavior sessions (that are also ophys) loaded with
-    BehaviorDataLimsApi return the same results as sessions loaded
+    BehaviorLimsApi return the same results as sessions loaded
     with BehaviorOphysLimsApi, for relevant functions. Do not check for
     timestamps, which are from different files so will not be the same.
     Also not checking for experiment_date, since they're from two different
@@ -167,11 +169,11 @@ class TestBehaviorRegression:
 
     Do not test `get_licks` regression because the licks come from two
     different sources and are recorded differently (behavior pickle file in 
-    BehaviorDataLimsApi; sync file in BehaviorOphysLimeApi)
+    BehaviorLimsApi; sync file in BehaviorOphysLimeApi)
     """
     @classmethod
     def setup_class(cls):
-        cls.bd = BehaviorDataLimsApi(976012750)
+        cls.bd = BehaviorLimsApi(976012750)
         cls.od = BehaviorOphysLimsApi(976255949)
 
     @classmethod
@@ -287,7 +289,7 @@ def test_get_full_genotype_regression(self):
     def test_get_experiment_date_regression(self):
         """Just testing the date since it comes from two different sources;
         We expect that BehaviorOphysLimsApi will be earlier (more like when
-        rig was started up), while BehaviorDataLimsApi returns the start of
+        rig was started up), while BehaviorLimsApi returns the start of
         the actual behavior (from pkl file)"""
         assert (self.bd.get_experiment_date().date()
                 == self.od.get_experiment_date().date())
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py
index 2afec5403..00afc052d 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py
@@ -1,7 +1,8 @@
 import pytest
 
 from allensdk import OneResultExpectedError
-from allensdk.internal.api.behavior_lims_api import BehaviorLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorLimsApi)
 from allensdk.brain_observatory.behavior.mtrain import ExtendedTrialSchema
 from marshmallow.schema import ValidationError
 
@@ -30,18 +31,17 @@ def test_get_behavior_stimulus_file(behavior_experiment_id, compare_val):
     pytest.param('394a910e-94c7-4472-9838-5345aff59ed8'),
 ])
 def test_foraging_id_to_behavior_session_id(behavior_session_uuid):
-    behavior_session_id = BehaviorLimsApi.foraging_id_to_behavior_session_id(
-        behavior_session_uuid)
-    assert behavior_session_id == 823847007
+    session = BehaviorLimsApi.from_foraging_id(behavior_session_uuid)
+    assert session.behavior_session_id == 823847007
 
 
 @pytest.mark.requires_bamboo
-@pytest.mark.parametrize('behavior_session_uuid', [
+@pytest.mark.parametrize('behavior_session_id', [
     pytest.param(823847007),
 ])
-def test_behavior_session_id_to_foraging_id(behavior_session_uuid):
-    foraging_id = BehaviorLimsApi.behavior_session_id_to_foraging_id(
-        behavior_session_uuid)
+def test_behavior_session_id_to_foraging_id(behavior_session_id):
+    session = BehaviorLimsApi(behavior_session_id=behavior_session_id)
+    foraging_id = session.get_behavior_session_uuid()
     assert foraging_id == '394a910e-94c7-4472-9838-5345aff59ed8'
 
 
diff --git a/allensdk/test/brain_observatory/behavior/test_trials_processing.py b/allensdk/test/brain_observatory/behavior/test_trials_processing.py
index cfd8c1ad4..1ab1b25b7 100644
--- a/allensdk/test/brain_observatory/behavior/test_trials_processing.py
+++ b/allensdk/test/brain_observatory/behavior/test_trials_processing.py
@@ -2,7 +2,8 @@
 import pandas as pd
 import numpy as np
 
-from allensdk.internal.api.behavior_lims_api import BehaviorLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorLimsApi)
 from allensdk.brain_observatory.behavior import trials_processing
 
 
@@ -22,7 +23,7 @@ def test_get_ori_info_from_trial(behavior_experiment_id, ti, expected, exception
     - i may be rewriting code here but its more a sanity check really...
     """
     stim_output = pd.read_pickle(
-        BehaviorLimsApi(behavior_experiment_id).get_behavior_stimulus_file()
+        BehaviorLimsApi(behavior_session_id=behavior_experiment_id).get_behavior_stimulus_file()
     )
     trial_log = stim_output['items']['behavior']['trial_log']
 

From b4048633fbe4f3c8505008000a84ff1551195965 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 17 Nov 2020 11:58:54 -0800
Subject: [PATCH 38/56] Fix imports of BehaviorOphysLimsApi

This commits fixes imports of the BehaviorOphysLimsApi
which previously lived at:
allensdk.internal.api.behavior_ophys_api.py

Its new location is:
allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_lims_api.py
---
 allensdk/brain_observatory/behavior/behavior_ophys_session.py | 3 +--
 .../project_apis/data_io/behavior_project_lims_api.py         | 3 +--
 .../behavior/session_apis/data_io/__init__.py                 | 1 +
 .../{behavior_ophys_api.py => behavior_ophys_lims_api.py}     | 2 +-
 .../behavior/swdb/save_extended_stimulus_presentations_df.py  | 3 +--
 .../brain_observatory/behavior/swdb/save_flash_response_df.py | 3 ++-
 .../brain_observatory/behavior/swdb/save_trial_response_df.py | 4 +---
 allensdk/brain_observatory/behavior/write_nwb/__main__.py     | 3 +--
 .../brain_observatory/behavior/test_behavior_data_lims_api.py | 3 ++-
 .../behavior/test_behavior_ophys_lims_api.py                  | 3 ++-
 .../brain_observatory/behavior/test_behavior_ophys_session.py | 3 +--
 .../examples/nb/api_modernization/updated_vs_legacy_api.ipynb | 2 +-
 12 files changed, 15 insertions(+), 18 deletions(-)
 rename allensdk/brain_observatory/behavior/session_apis/data_io/{behavior_ophys_api.py => behavior_ophys_lims_api.py} (99%)

diff --git a/allensdk/brain_observatory/behavior/behavior_ophys_session.py b/allensdk/brain_observatory/behavior/behavior_ophys_session.py
index 001800693..03d1dd995 100644
--- a/allensdk/brain_observatory/behavior/behavior_ophys_session.py
+++ b/allensdk/brain_observatory/behavior/behavior_ophys_session.py
@@ -6,9 +6,8 @@
 
 
 from allensdk.brain_observatory.session_api_utils import ParamsMixin
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorOphysNwbApi)
+    BehaviorOphysNwbApi, BehaviorOphysLimsApi)
 from allensdk.deprecated import legacy
 from allensdk.brain_observatory.behavior.trials_processing import (
     calculate_reward_rate)
diff --git a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
index 1e9fa85a6..21a5ce87a 100644
--- a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
@@ -9,8 +9,7 @@
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession)
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorLimsApi)
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
+    BehaviorLimsApi, BehaviorOphysLimsApi)
 from allensdk.internal.api import PostgresQueryMixin
 from allensdk.brain_observatory.ecephys.ecephys_project_api.http_engine import (
     HttpEngine)
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
index 9e328cc8d..d36e07de9 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
@@ -3,3 +3,4 @@
 
 # data_io classes for behavior + ophys
 from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_nwb_api import BehaviorOphysNwbApi  # noqa: F401, E501
+from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_lims_api import BehaviorOphysLimsApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
similarity index 99%
rename from allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_api.py
rename to allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index 25b5ef11e..278a34667 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -31,7 +31,7 @@
 from allensdk.core.authentication import credential_injector, DbCredentials
 
 
-class BehaviorOphysLimsApi(OphysLimsApi, BehaviorOphysBase):
+class BehaviorOphysLimsApi(BehaviorOphysBase, OphysLimsApi):
 
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None):
diff --git a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
index 92b1c83dd..9011abf05 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_extended_stimulus_presentations_df.py
@@ -7,8 +7,7 @@
     BehaviorOphysSession,
 )
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorOphysNwbApi)
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
+    BehaviorOphysNwbApi, BehaviorOphysLimsApi)
 
 import behavior_project_cache as bpc
 from importlib import reload
diff --git a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
index 8a875944f..3b0c708f1 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_flash_response_df.py
@@ -7,7 +7,8 @@
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorOphysNwbApi)
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorOphysLimsApi)
 from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc
 from allensdk.brain_observatory.behavior.swdb.analysis_tools import get_nearest_frame, get_trace_around_timepoint, get_mean_in_window
 
diff --git a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
index c4dcdea6d..f3048b77b 100644
--- a/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
+++ b/allensdk/brain_observatory/behavior/swdb/save_trial_response_df.py
@@ -7,9 +7,7 @@
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorOphysNwbApi)
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
-
+    BehaviorOphysNwbApi, BehaviorOphysLimsApi)
 from allensdk.brain_observatory.behavior.swdb import behavior_project_cache as bpc
 from importlib import reload; reload(bpc)
 from allensdk.brain_observatory.behavior.swdb.analysis_tools import get_nearest_frame, get_trace_around_timepoint, get_mean_in_window
diff --git a/allensdk/brain_observatory/behavior/write_nwb/__main__.py b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
index ca4a6c73d..62ea5d9a6 100644
--- a/allensdk/brain_observatory/behavior/write_nwb/__main__.py
+++ b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
@@ -8,11 +8,10 @@
 import argschema
 import marshmallow
 
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession)
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorOphysNwbApi)
+    BehaviorOphysNwbApi, BehaviorOphysLimsApi)
 from allensdk.brain_observatory.behavior.write_nwb._schemas import (
     InputSchema, OutputSchema)
 from allensdk.brain_observatory.argschema_utilities import (
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
index a0724c502..c6c19f34f 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
@@ -9,7 +9,8 @@
     BehaviorLimsApi
 )
 from allensdk.core.authentication import DbCredentials
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorOphysLimsApi)
 from allensdk.brain_observatory.running_speed import RunningSpeed
 from allensdk.core.exceptions import DataFrameIndexError
 
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
index 2592d1e2a..6d6b5a796 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
@@ -4,7 +4,8 @@
 from contextlib import contextmanager
 
 from allensdk.internal.api import OneResultExpectedError
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorOphysLimsApi)
 from allensdk.brain_observatory.behavior.mtrain import ExtendedTrialSchema
 from marshmallow.schema import ValidationError
 
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
index 1dd5e6eac..9301781e6 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
@@ -12,10 +12,9 @@
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 from allensdk.brain_observatory.behavior.write_nwb.__main__ import BehaviorOphysJsonApi
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorOphysNwbApi)
+    BehaviorOphysNwbApi, BehaviorOphysLimsApi)
 from allensdk.brain_observatory.session_api_utils import (
     sessions_are_equal, compare_session_fields)
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.behavior.image_api import ImageApi
 
 
diff --git a/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb b/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb
index bbe717dc9..33fe74959 100644
--- a/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb
+++ b/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb
@@ -17,7 +17,7 @@
    "source": [
     "import matplotlib.pyplot as plt\n",
     "\n",
-    "from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi\n",
+    "from from allensdk.brain_observatory.behavior.session_apis.data_io import BehaviorOphysLimsApi\n",
     "from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession\n",
     "\n",
     "\n",

From dcfcc6b8a84ad525fe4aeb48f41ead8ee5e08468 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 17 Nov 2020 13:15:34 -0800
Subject: [PATCH 39/56] Fix imports for OphysLimsApi

This commit fixes imports for the OphysLimsApi that previously lived at:
allensdk.internal.api.ophys_lims_api.py

It now lives at:
allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api.py
---
 .../session_apis/data_io/behavior_ophys_lims_api.py         | 3 ++-
 allensdk/brain_observatory/behavior/validation.py           | 6 ++++--
 allensdk/brain_observatory/nwb/__init__.py                  | 3 ++-
 .../test/brain_observatory/behavior/test_ophys_lims_api.py  | 3 ++-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index 278a34667..d6f669961 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -9,7 +9,8 @@
 import matplotlib.image as mpimg  # NOQA: E402
 
 from allensdk.api.cache import memoize
-from allensdk.internal.api.ophys_lims_api import OphysLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api \
+    import OphysLimsApi
 from allensdk.brain_observatory.behavior.sync import (
     get_sync_data, get_stimulus_rebase_function, frame_time_offset)
 from allensdk.brain_observatory.sync_dataset import Dataset
diff --git a/allensdk/brain_observatory/behavior/validation.py b/allensdk/brain_observatory/behavior/validation.py
index cdc58454a..528e0a6a6 100644
--- a/allensdk/brain_observatory/behavior/validation.py
+++ b/allensdk/brain_observatory/behavior/validation.py
@@ -1,8 +1,10 @@
 import h5py
 import os
 
-from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
-from allensdk.internal.api.ophys_lims_api import OphysLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorOphysLimsApi)
+from allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api \
+    import OphysLimsApi
 from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession
 
 class ValidationError(AssertionError):
diff --git a/allensdk/brain_observatory/nwb/__init__.py b/allensdk/brain_observatory/nwb/__init__.py
index 20e5976c3..f0e5118f3 100644
--- a/allensdk/brain_observatory/nwb/__init__.py
+++ b/allensdk/brain_observatory/nwb/__init__.py
@@ -896,7 +896,8 @@ def add_cell_specimen_table(nwbfile: NWBFile,
     for cell_roi_id, table_row in cell_roi_table.iterrows():
 
         # NOTE: The 'image_mask' in this cell_roi_table has already been
-        # processing by the allensdk.internal.api.ophys_lims_api
+        # processing by the function from 
+        # allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api
         # get_cell_specimen_table() method. As a result, the ROI is stored in
         # an array that is the same shape as the FULL field of view of the
         # experiment (e.g. 512 x 512).
diff --git a/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py b/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py
index 14f9f3ba7..6fd623e7d 100644
--- a/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py
@@ -3,7 +3,8 @@
 import pytest
 
 from allensdk.internal.api import OneResultExpectedError, OneOrMoreResultExpectedError
-from allensdk.internal.api.ophys_lims_api import OphysLimsApi
+from allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api \
+    import OphysLimsApi
 
 
 @pytest.fixture(scope="function")

From 0143677e7d2c8f843aa592d71977d670fa2634a9 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Wed, 18 Nov 2020 08:44:39 -0800
Subject: [PATCH 40/56] Move the BehaviorOphysJsonApi

The commit moves the BehaviorOphysJsonApi defined in
allensdk.brain_observatory.behavior.write_nwb.__main__ to
allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_json_api.

This commit also fixes a bug with multiple inheritence and method
resolution order. ABC classes should always be inherited last so that
the first parent of a class will be able to fill the subclass with
defined methods (and not abstract methods).
---
 .../behavior/session_apis/data_io/__init__.py |  1 +
 .../session_apis/data_io/behavior_lims_api.py | 17 +++-
 .../data_io/behavior_ophys_json_api.py        | 98 +++++++++++++++++++
 .../data_io/behavior_ophys_lims_api.py        | 20 ++--
 .../behavior/write_nwb/__main__.py            | 98 +------------------
 5 files changed, 128 insertions(+), 106 deletions(-)
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
index d36e07de9..686eaa15e 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/__init__.py
@@ -4,3 +4,4 @@
 # data_io classes for behavior + ophys
 from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_nwb_api import BehaviorOphysNwbApi  # noqa: F401, E501
 from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_lims_api import BehaviorOphysLimsApi  # noqa: F401, E501
+from allensdk.brain_observatory.behavior.session_apis.data_io.behavior_ophys_json_api import BehaviorOphysJsonApi  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index 23527f259..a8975a413 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -18,7 +18,8 @@
 from allensdk.brain_observatory.behavior.metadata_processing import (
     get_task_parameters)
 from allensdk.brain_observatory.behavior.sync import frame_time_offset
-from allensdk.brain_observatory.behavior.trials_processing import get_trials
+from allensdk.brain_observatory.behavior.trials_processing import (
+    get_trials, get_extended_trials)
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.internal.api import db_connection_creator
 from allensdk.api.cache import memoize
@@ -30,7 +31,7 @@
     LIMS_DB_CREDENTIAL_MAP, MTRAIN_DB_CREDENTIAL_MAP)
 
 
-class BehaviorLimsApi(BehaviorBase, CachedInstanceMethodMixin):
+class BehaviorLimsApi(CachedInstanceMethodMixin, BehaviorBase):
     """A data fetching class that serves as an API for filling
     'BehaviorSession' attributes from LIMS.
     """
@@ -373,6 +374,18 @@ def get_trials(self) -> pd.DataFrame:
 
         return trial_df
 
+    def get_extended_trials(self) -> pd.DataFrame:
+        """Get extended trials from pkl file
+
+        Returns
+        -------
+        pd.DataFrame
+            A dataframe containing extended behavior trial information.
+        """
+        filename = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(filename)
+        return get_extended_trials(data)
+
     @memoize
     def get_birth_date(self) -> datetime.date:
         """Returns the birth date of the animal.
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
new file mode 100644
index 000000000..3366b63b0
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
@@ -0,0 +1,98 @@
+import datetime
+import pytz
+
+
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorOphysLimsApi)
+
+
+class BehaviorOphysJsonApi(BehaviorOphysLimsApi):
+    """
+        This class is used by both Scientifica and Mesoscope ophys experiments.
+    """
+
+    def __init__(self, data):
+        self.data = data
+
+    def get_ophys_experiment_id(self):
+        return self.data['ophys_experiment_id']
+
+    def get_surface_2p_pixel_size_um(self):
+        return self.data['surface_2p_pixel_size_um']
+
+    def get_max_projection_file(self):
+        return self.data['max_projection_file']
+
+    def get_sync_file(self):
+        return self.data['sync_file']
+
+    def get_rig_name(self):
+        return self.data['rig_name']
+
+    def get_sex(self):
+        return self.data['sex']
+
+    def get_age(self):
+        return self.data['age']
+
+    def get_field_of_view_shape(self):
+        return {'height': self.data['movie_height'], 'width': self.data['movie_width']}
+
+    def get_experiment_container_id(self):
+        return self.data['container_id']
+
+    def get_targeted_structure(self):
+        return self.data['targeted_structure']
+
+    def get_imaging_depth(self):
+        return self.data['targeted_depth']
+
+    def get_stimulus_name(self):
+        return self.data['stimulus_name']
+
+    def get_experiment_date(self):
+        return pytz.utc.localize(datetime.datetime.strptime(self.data['date_of_acquisition'], "%Y-%m-%d %H:%M:%S"))
+
+    def get_reporter_line(self):
+        return self.data['reporter_line']
+
+    def get_driver_line(self):
+        return self.data['driver_line']
+
+    def external_specimen_name(self):
+        return self.data['external_specimen_name']
+
+    def get_full_genotype(self):
+        return self.data['full_genotype']
+
+    def get_behavior_stimulus_file(self):
+        return self.data['behavior_stimulus_file']
+
+    def get_dff_file(self):
+        return self.data['dff_file']
+
+    def get_ophys_cell_segmentation_run_id(self):
+        return self.data['ophys_cell_segmentation_run_id']
+
+    def get_raw_cell_specimen_table_dict(self):
+        return self.data['cell_specimen_table_dict']
+
+    def get_demix_file(self):
+        return self.data['demix_file']
+
+    def get_average_intensity_projection_image_file(self):
+        return self.data['average_intensity_projection_image_file']
+
+    def get_rigid_motion_transform_file(self):
+        return self.data['rigid_motion_transform_file']
+
+    def get_external_specimen_name(self):
+        return self.data['external_specimen_name']
+
+    def get_imaging_plane_group(self):
+        try:
+            # Will only contain the "imaging_plane_group" key if we are
+            # dealing with Mesoscope data
+            return self.data["imaging_plane_group"]
+        except KeyError:
+            return None
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index d6f669961..e0f663f1d 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -16,23 +16,29 @@
 from allensdk.brain_observatory.sync_dataset import Dataset
 from allensdk.brain_observatory import sync_utilities
 from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner
-from allensdk.brain_observatory.behavior.stimulus_processing import get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata
-from allensdk.brain_observatory.behavior.metadata_processing import get_task_parameters
-from allensdk.brain_observatory.behavior.running_processing import get_running_df
+from allensdk.brain_observatory.behavior.stimulus_processing import (
+    get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata)
+from allensdk.brain_observatory.behavior.metadata_processing import (
+    get_task_parameters)
+from allensdk.brain_observatory.behavior.running_processing import (
+    get_running_df)
 from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
 from allensdk.brain_observatory.behavior.trials_processing import get_trials
-from allensdk.brain_observatory.behavior.eye_tracking_processing import load_eye_tracking_hdf, process_eye_tracking_data
+from allensdk.brain_observatory.behavior.eye_tracking_processing import (
+    load_eye_tracking_hdf, process_eye_tracking_data)
 from allensdk.brain_observatory.running_speed import RunningSpeed
 from allensdk.brain_observatory.behavior.image_api import ImageApi
 from allensdk.internal.api import PostgresQueryMixin
-from allensdk.brain_observatory.behavior.session_apis.abcs import BehaviorOphysBase
-from allensdk.brain_observatory.behavior.trials_processing import get_extended_trials
+from allensdk.brain_observatory.behavior.session_apis.abcs import (
+    BehaviorOphysBase)
+from allensdk.brain_observatory.behavior.trials_processing import (
+    get_extended_trials)
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
 from allensdk.core.authentication import credential_injector, DbCredentials
 
 
-class BehaviorOphysLimsApi(BehaviorOphysBase, OphysLimsApi):
+class BehaviorOphysLimsApi(OphysLimsApi, BehaviorOphysBase):
 
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None):
diff --git a/allensdk/brain_observatory/behavior/write_nwb/__main__.py b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
index 62ea5d9a6..1d30bf16c 100644
--- a/allensdk/brain_observatory/behavior/write_nwb/__main__.py
+++ b/allensdk/brain_observatory/behavior/write_nwb/__main__.py
@@ -1,17 +1,13 @@
-import datetime
-import pytz
 import os
 import logging
 import sys
-import argparse
-import json
 import argschema
 import marshmallow
 
 from allensdk.brain_observatory.behavior.behavior_ophys_session import (
     BehaviorOphysSession)
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorOphysNwbApi, BehaviorOphysLimsApi)
+    BehaviorOphysNwbApi, BehaviorOphysJsonApi)
 from allensdk.brain_observatory.behavior.write_nwb._schemas import (
     InputSchema, OutputSchema)
 from allensdk.brain_observatory.argschema_utilities import (
@@ -19,98 +15,6 @@
 from allensdk.brain_observatory.session_api_utils import sessions_are_equal
 
 
-class BehaviorOphysJsonApi(BehaviorOphysLimsApi):
-    """
-        This class is used by both Scientifica and Mesoscope ophys experiments.
-    """
-
-    def __init__(self, data):
-        self.data = data
-
-    def get_ophys_experiment_id(self):
-        return self.data['ophys_experiment_id']
-
-    def get_surface_2p_pixel_size_um(self):
-        return self.data['surface_2p_pixel_size_um']
-
-    def get_max_projection_file(self):
-        return self.data['max_projection_file']
-
-    def get_sync_file(self):
-        return self.data['sync_file']
-
-    def get_rig_name(self):
-        return self.data['rig_name']
-
-    def get_sex(self):
-        return self.data['sex']
-
-    def get_age(self):
-        return self.data['age']
-
-    def get_field_of_view_shape(self):
-        return {'height': self.data['movie_height'], 'width': self.data['movie_width']}
-
-    def get_experiment_container_id(self):
-        return self.data['container_id']
-
-    def get_targeted_structure(self):
-        return self.data['targeted_structure']
-
-    def get_imaging_depth(self):
-        return self.data['targeted_depth']
-
-    def get_stimulus_name(self):
-        return self.data['stimulus_name']
-
-    def get_experiment_date(self):
-        return pytz.utc.localize(datetime.datetime.strptime(self.data['date_of_acquisition'], "%Y-%m-%d %H:%M:%S"))
-
-    def get_reporter_line(self):
-        return self.data['reporter_line']
-
-    def get_driver_line(self):
-        return self.data['driver_line']
-
-    def external_specimen_name(self):
-        return self.data['external_specimen_name']
-
-    def get_full_genotype(self):
-        return self.data['full_genotype']
-
-    def get_behavior_stimulus_file(self):
-        return self.data['behavior_stimulus_file']
-
-    def get_dff_file(self):
-        return self.data['dff_file']
-
-    def get_ophys_cell_segmentation_run_id(self):
-        return self.data['ophys_cell_segmentation_run_id']
-
-    def get_raw_cell_specimen_table_dict(self):
-        return self.data['cell_specimen_table_dict']
-
-    def get_demix_file(self):
-        return self.data['demix_file']
-
-    def get_average_intensity_projection_image_file(self):
-        return self.data['average_intensity_projection_image_file']
-
-    def get_rigid_motion_transform_file(self):
-        return self.data['rigid_motion_transform_file']
-
-    def get_external_specimen_name(self):
-        return self.data['external_specimen_name']
-
-    def get_imaging_plane_group(self):
-        try:
-            # Will only contain the "imaging_plane_group" key if we are
-            # dealing with Mesoscope data
-            return self.data["imaging_plane_group"]
-        except KeyError:
-            return None
-
-
 def write_behavior_ophys_nwb(session_data, nwb_filepath):
 
     nwb_filepath_inprogress = nwb_filepath+'.inprogress'

From 402e2daf5440480382fb941047364d980e6d48a2 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Thu, 19 Nov 2020 16:56:09 -0800
Subject: [PATCH 41/56] Factor out data only ophys / behavior + ophys methods

This commit factors out 'data only' (LIMS/BehaviorOphysJson/NWB) API
methods from methods that are required by the OphysSession class.

This commit creates a new set of classes (BehaviorDataXforms, BehaviorOphysDataXforms)
that transforms data obtained from 'data only' methods into sources for
the OphysSession class.
---
 .../session_apis/abcs/behavior_ophys_base.py  |  22 +-
 .../session_apis/data_io/behavior_lims_api.py | 292 +--------------
 .../data_io/behavior_ophys_json_api.py        |  16 +-
 .../data_io/behavior_ophys_lims_api.py        | 312 ++--------------
 .../data_io/behavior_ophys_nwb_api.py         |   2 +-
 .../session_apis/data_io/ophys_lims_api.py    |   6 -
 .../session_apis/data_processing/__init__.py  |   0
 .../session_apis/data_transforms/__init__.py  |   2 +
 .../data_transforms/behavior_data_xforms.py   | 287 +++++++++++++++
 .../behavior_ophys_data_xforms.py             | 344 ++++++++++++++++++
 .../behavior/test_behavior_data_lims_api.py   |  12 +-
 11 files changed, 694 insertions(+), 601 deletions(-)
 delete mode 100644 allensdk/brain_observatory/behavior/session_apis/data_processing/__init__.py
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/data_transforms/__init__.py
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
 create mode 100644 allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py

diff --git a/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
index 29f4ebe60..f0852c95b 100644
--- a/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
+++ b/allensdk/brain_observatory/behavior/session_apis/abcs/behavior_ophys_base.py
@@ -124,18 +124,6 @@ def get_ophys_timestamps(self) -> np.ndarray:
         """
         raise NotImplementedError()
 
-    @abc.abstractmethod
-    def get_raw_stimulus_timestamps(self) -> np.ndarray:
-        """Get raw stimulus timestamps.
-
-        Returns
-        -------
-        np.ndarray
-            Timestamps associated with stimulus presentations on the monitor
-            without accounting for monitor delay.
-        """
-        raise NotImplementedError()
-
     @abc.abstractmethod
     def get_stimulus_timestamps(self) -> np.ndarray:
         """Get stimulus timestamps.
@@ -164,15 +152,7 @@ def get_stimulus_presentations(self) -> pd.DataFrame:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def get_segmentation_mask_image(self) -> Image:
-        """Get the 'segmentation_mask_image'. This image contains pixels
-        with a value of 1 if they are included in ANY ROI segmented in the
-        behavior + ophys session and 0 otherwise.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_eye_tracking_data(self) -> pd.DataFrame:
+    def get_eye_tracking(self) -> pd.DataFrame:
         """Get eye tracking data from behavior + ophys session.
 
         Returns
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index a8975a413..26722c9dc 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -1,25 +1,8 @@
-import numpy as np
-import pandas as pd
-import uuid
-from datetime import datetime
-import pytz
+from typing import Dict, Optional, Union, List
 
-from typing import Dict, Optional, Union, List, Any
+from datetime import datetime
+import uuid
 
-from allensdk.core.exceptions import DataFrameIndexError
-from allensdk.brain_observatory.behavior.session_apis.abcs import (
-    BehaviorBase)
-from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
-from allensdk.brain_observatory.behavior.running_processing import (
-    get_running_df)
-from allensdk.brain_observatory.behavior.stimulus_processing import (
-    get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata)
-from allensdk.brain_observatory.running_speed import RunningSpeed
-from allensdk.brain_observatory.behavior.metadata_processing import (
-    get_task_parameters)
-from allensdk.brain_observatory.behavior.sync import frame_time_offset
-from allensdk.brain_observatory.behavior.trials_processing import (
-    get_trials, get_extended_trials)
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.internal.api import db_connection_creator
 from allensdk.api.cache import memoize
@@ -29,17 +12,17 @@
 from allensdk.core.authentication import DbCredentials
 from allensdk.core.auth_config import (
     LIMS_DB_CREDENTIAL_MAP, MTRAIN_DB_CREDENTIAL_MAP)
+from allensdk.brain_observatory.behavior.session_apis.data_transforms import (
+    BehaviorDataXforms)
 
 
-class BehaviorLimsApi(CachedInstanceMethodMixin, BehaviorBase):
+class BehaviorLimsApi(BehaviorDataXforms, CachedInstanceMethodMixin):
     """A data fetching class that serves as an API for filling
     'BehaviorSession' attributes from LIMS.
     """
     def __init__(self, behavior_session_id: int,
                  lims_credentials: Optional[DbCredentials] = None,
                  mtrain_credentials: Optional[DbCredentials] = None):
-        super().__init__()
-
         self.mtrain_db = db_connection_creator(
             credentials=mtrain_credentials,
             default_credentials=MTRAIN_DB_CREDENTIAL_MAP)
@@ -144,26 +127,6 @@ def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]:
                              "ophys_container_id": None})
         return ids_dict
 
-    def get_behavior_session_id(self) -> int:
-        """Getter to be consistent with BehaviorOphysLimsApi."""
-        return self.behavior_session_id
-
-    def get_behavior_session_uuid(self) -> Optional[int]:
-        """Get the universally unique identifier (UUID) number for the
-        current behavior session.
-        """
-        data = self._behavior_stimulus_file()
-        behavior_pkl_uuid = data.get("session_uuid")
-        # Sanity check to ensure that pkl file data matches up with
-        # the behavior session that the pkl file has been associated with.
-        assert_err_msg = (
-            f"The behavior session UUID ({behavior_pkl_uuid}) in the behavior "
-            f"stimulus *.pkl file ({self.get_behavior_stimulus_file()}) does "
-            f"does not match the LIMS UUID ({self.foraging_id}) for "
-            f"behavior session: {self.behavior_session_id}")
-        assert behavior_pkl_uuid == self.foraging_id, assert_err_msg
-        return behavior_pkl_uuid
-
     def get_behavior_stimulus_file(self) -> str:
         """Return the path to the StimulusPickle file for a session.
         :rtype: str
@@ -184,210 +147,7 @@ def get_behavior_stimulus_file(self) -> str:
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def _behavior_stimulus_file(self) -> pd.DataFrame:
-        """Helper method to cache stimulus file in memory since it takes about
-        a second to load (and is used in many methods).
-        """
-        return pd.read_pickle(self.get_behavior_stimulus_file())
-
-    def get_licks(self) -> pd.DataFrame:
-        """Get lick data from pkl file.
-        This function assumes that the first sensor in the list of
-        lick_sensors is the desired lick sensor. If this changes we need
-        to update to get the proper line.
-
-        Since licks can occur outside of a trial context, the lick times
-        are extracted from the vsyncs and the frame number in `lick_events`.
-        Since we don't have a timestamp for when in "experiment time" the
-        vsync stream starts (from self.get_stimulus_timestamps), we compute
-        it by fitting a linear regression (frame number x time) for the
-        `start_trial` and `end_trial` events in the `trial_log`, to true
-        up these time streams.
-
-        :returns: pd.DataFrame -- A dataframe containing lick timestamps
-        """
-        # Get licks from pickle file instead of sync
-        data = self._behavior_stimulus_file()
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        lick_frames = (data["items"]["behavior"]["lick_sensors"][0]
-                       ["lick_events"])
-        lick_times = [stimulus_timestamps[frame] for frame in lick_frames]
-        return pd.DataFrame({"time": lick_times})
-
-    def get_rewards(self) -> pd.DataFrame:
-        """Get reward data from pkl file, based on pkl file timestamps
-        (not sync file).
-
-        :returns: pd.DataFrame -- A dataframe containing timestamps of
-            delivered rewards.
-        """
-        data = self._behavior_stimulus_file()
-        offset = frame_time_offset(data)
-        # No sync timestamps to rebase on, but do need to align to
-        # trial events, so add the offset as the "rebase" function
-        return get_rewards(data, lambda x: x + offset)
-
-    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
-        """Get running speed data.
-
-        :returns: pd.DataFrame -- dataframe containing various signals used
-            to compute running speed.
-        """
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        data = self._behavior_stimulus_file()
-        return get_running_df(data, stimulus_timestamps, lowpass=lowpass)
-
-    def get_running_speed(self, lowpass=True) -> RunningSpeed:
-        """Get running speed using timestamps from
-        self.get_stimulus_timestamps.
-
-        NOTE: Do not correct for monitor delay.
-
-        :returns: RunningSpeed -- a NamedTuple containing the subject's
-            timestamps and running speeds (in cm/s)
-        """
-        running_data_df = self.get_running_data_df(lowpass=lowpass)
-        if running_data_df.index.name != "timestamps":
-            raise DataFrameIndexError(
-                f"Expected index to be named 'timestamps' but got "
-                "'{running_data_df.index.name}'.")
-        return RunningSpeed(timestamps=running_data_df.index.values,
-                            values=running_data_df.speed.values)
-
-    def get_stimulus_frame_rate(self) -> float:
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        return np.round(1 / np.mean(np.diff(stimulus_timestamps)), 0)
-
-    def get_stimulus_presentations(self) -> pd.DataFrame:
-        """Get stimulus presentation data.
-
-        NOTE: Uses timestamps that do not account for monitor delay.
-
-        :returns: pd.DataFrame --
-            Table whose rows are stimulus presentations
-            (i.e. a given image, for a given duration, typically 250 ms)
-            and whose columns are presentation characteristics.
-        """
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        data = self._behavior_stimulus_file()
-        raw_stim_pres_df = get_stimulus_presentations(
-            data, stimulus_timestamps)
-
-        # Fill in nulls for image_name
-        # This makes two assumptions:
-        #   1. Nulls in `image_name` should be "gratings_<orientation>"
-        #   2. Gratings are only present (or need to be fixed) when all
-        #      values for `image_name` are null.
-        if pd.isnull(raw_stim_pres_df["image_name"]).all():
-            if ~pd.isnull(raw_stim_pres_df["orientation"]).all():
-                raw_stim_pres_df["image_name"] = (
-                    raw_stim_pres_df["orientation"]
-                    .apply(lambda x: f"gratings_{x}"))
-            else:
-                raise ValueError("All values for 'orentation' and 'image_name'"
-                                 " are null.")
-
-        stimulus_metadata_df = get_stimulus_metadata(data)
-
-        idx_name = raw_stim_pres_df.index.name
-        stimulus_index_df = (
-            raw_stim_pres_df
-            .reset_index()
-            .merge(stimulus_metadata_df.reset_index(), on=["image_name"])
-            .set_index(idx_name))
-        stimulus_index_df = (
-            stimulus_index_df[["image_set", "image_index", "start_time",
-                               "phase", "spatial_frequency"]]
-            .rename(columns={"start_time": "timestamps"})
-            .sort_index()
-            .set_index("timestamps", drop=True))
-        stim_pres_df = raw_stim_pres_df.merge(
-            stimulus_index_df, left_on="start_time", right_index=True,
-            how="left")
-        if len(raw_stim_pres_df) != len(stim_pres_df):
-            raise ValueError("Length of `stim_pres_df` should not change after"
-                             f" merge; was {len(raw_stim_pres_df)}, now "
-                             f" {len(stim_pres_df)}.")
-        return stim_pres_df[sorted(stim_pres_df)]
-
-    def get_stimulus_templates(self) -> Dict[str, np.ndarray]:
-        """Get stimulus templates (movies, scenes) for behavior session.
-
-        Returns
-        -------
-        Dict[str, np.ndarray]
-            A dictionary containing the stimulus images presented during the
-            session. Keys are data set names, and values are 3D numpy arrays.
-        """
-        data = self._behavior_stimulus_file()
-        return get_stimulus_templates(data)
-
-    def get_stimulus_timestamps(self) -> np.ndarray:
-        """Get stimulus timestamps (vsyncs) from pkl file. Align to the
-        (frame, time) points in the trial events.
-
-        NOTE: Located with behavior_session_id. Does not use the sync_file
-        which requires ophys_session_id.
-
-        Returns
-        -------
-        np.ndarray
-            Timestamps associated with stimulus presentations on the monitor
-            that do no account for monitor delay.
-        """
-        data = self._behavior_stimulus_file()
-        vsyncs = data["items"]["behavior"]["intervalsms"]
-        cum_sum = np.hstack((0, vsyncs)).cumsum() / 1000.0  # cumulative time
-        offset = frame_time_offset(data)
-        return cum_sum + offset
-
-    def get_task_parameters(self) -> dict:
-        """Get task parameters from pkl file.
-
-        Returns
-        -------
-        dict
-            A dictionary containing parameters used to define the task runtime
-            behavior.
-        """
-        data = self._behavior_stimulus_file()
-        return get_task_parameters(data)
-
-    def get_trials(self) -> pd.DataFrame:
-        """Get trials from pkl file
-
-        Returns
-        -------
-        pd.DataFrame
-            A dataframe containing behavioral trial start/stop times,
-            and trial data
-        """
-        licks = self.get_licks()
-        data = self._behavior_stimulus_file()
-        rewards = self.get_rewards()
-        stimulus_presentations = self.get_stimulus_presentations()
-        # Pass a dummy rebase function since we don't have two time streams,
-        # and the frame times are already aligned to trial events in their
-        # respective getters
-        trial_df = get_trials(data, licks, rewards, stimulus_presentations,
-                              lambda x: x)
-
-        return trial_df
-
-    def get_extended_trials(self) -> pd.DataFrame:
-        """Get extended trials from pkl file
-
-        Returns
-        -------
-        pd.DataFrame
-            A dataframe containing extended behavior trial information.
-        """
-        filename = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(filename)
-        return get_extended_trials(data)
-
-    @memoize
-    def get_birth_date(self) -> datetime.date:
+    def get_birth_date(self) -> datetime:
         """Returns the birth date of the animal.
         :rtype: datetime.date
         """
@@ -528,41 +288,3 @@ def get_full_genotype(self) -> str:
                 WHERE bs.id= {self.behavior_session_id};
                 """
         return self.lims_db.fetchone(query, strict=True)
-
-    @memoize
-    def get_experiment_date(self) -> datetime:
-        """Return timestamp the behavior stimulus file began recording in UTC
-        :rtype: datetime
-        """
-        data = self._behavior_stimulus_file()
-        # Assuming file has local time of computer (Seattle)
-        tz = pytz.timezone("America/Los_Angeles")
-        return tz.localize(data["start_time"]).astimezone(pytz.utc)
-
-    def get_metadata(self) -> Dict[str, Any]:
-        """Return metadata about the session.
-        :rtype: dict
-        """
-        if self.get_behavior_session_uuid() is None:
-            bs_uuid = None
-        else:
-            bs_uuid = uuid.UUID(self.get_behavior_session_uuid())
-        metadata = {
-            "rig_name": self.get_rig_name(),
-            "sex": self.get_sex(),
-            "age": self.get_age(),
-            "ophys_experiment_id": self.ophys_experiment_ids,
-            "experiment_container_id": self.ophys_container_id,
-            "stimulus_frame_rate": self.get_stimulus_frame_rate(),
-            "session_type": self.get_stimulus_name(),
-            "experiment_datetime": self.get_experiment_date(),
-            "reporter_line": self.get_reporter_line(),
-            "driver_line": self.get_driver_line(),
-            "LabTracks_ID": self.get_external_specimen_name(),
-            "full_genotype": self.get_full_genotype(),
-            "behavior_session_uuid": bs_uuid,
-            "foraging_id": self.foraging_id,
-            "behavior_session_id": self.behavior_session_id,
-            "behavior_training_id": self.behavior_training_id,
-        }
-        return metadata
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
index 3366b63b0..3c5c4fb6d 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
@@ -1,12 +1,11 @@
-import datetime
+from datetime import datetime
 import pytz
 
+from allensdk.brain_observatory.behavior.session_apis.data_transforms import (
+    BehaviorOphysDataXforms)
 
-from allensdk.brain_observatory.behavior.session_apis.data_io import (
-    BehaviorOphysLimsApi)
 
-
-class BehaviorOphysJsonApi(BehaviorOphysLimsApi):
+class BehaviorOphysJsonApi(BehaviorOphysDataXforms):
     """
         This class is used by both Scientifica and Mesoscope ophys experiments.
     """
@@ -36,7 +35,8 @@ def get_age(self):
         return self.data['age']
 
     def get_field_of_view_shape(self):
-        return {'height': self.data['movie_height'], 'width': self.data['movie_width']}
+        return {'height': self.data['movie_height'],
+                'width': self.data['movie_width']}
 
     def get_experiment_container_id(self):
         return self.data['container_id']
@@ -51,7 +51,9 @@ def get_stimulus_name(self):
         return self.data['stimulus_name']
 
     def get_experiment_date(self):
-        return pytz.utc.localize(datetime.datetime.strptime(self.data['date_of_acquisition'], "%Y-%m-%d %H:%M:%S"))
+        return pytz.utc.localize(
+            datetime.strptime(self.data['date_of_acquisition'],
+                              "%Y-%m-%d %H:%M:%S"))
 
     def get_reporter_line(self):
         return self.data['reporter_line']
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index e0f663f1d..d03bb3852 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -1,94 +1,37 @@
 import logging
 from typing import Optional
-from pathlib import Path
-
-import numpy as np
-import h5py
 import pandas as pd
-import uuid
-import matplotlib.image as mpimg  # NOQA: E402
 
 from allensdk.api.cache import memoize
 from allensdk.brain_observatory.behavior.session_apis.data_io.ophys_lims_api \
     import OphysLimsApi
-from allensdk.brain_observatory.behavior.sync import (
-    get_sync_data, get_stimulus_rebase_function, frame_time_offset)
-from allensdk.brain_observatory.sync_dataset import Dataset
-from allensdk.brain_observatory import sync_utilities
-from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner
-from allensdk.brain_observatory.behavior.stimulus_processing import (
-    get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata)
-from allensdk.brain_observatory.behavior.metadata_processing import (
-    get_task_parameters)
-from allensdk.brain_observatory.behavior.running_processing import (
-    get_running_df)
-from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
-from allensdk.brain_observatory.behavior.trials_processing import get_trials
-from allensdk.brain_observatory.behavior.eye_tracking_processing import (
-    load_eye_tracking_hdf, process_eye_tracking_data)
-from allensdk.brain_observatory.running_speed import RunningSpeed
-from allensdk.brain_observatory.behavior.image_api import ImageApi
-from allensdk.internal.api import PostgresQueryMixin
-from allensdk.brain_observatory.behavior.session_apis.abcs import (
-    BehaviorOphysBase)
-from allensdk.brain_observatory.behavior.trials_processing import (
-    get_extended_trials)
+from allensdk.brain_observatory.behavior.session_apis.data_io import (
+    BehaviorLimsApi)
+from allensdk.internal.api import db_connection_creator, PostgresQueryMixin
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
 from allensdk.core.authentication import credential_injector, DbCredentials
+from allensdk.brain_observatory.behavior.session_apis.data_transforms import (
+    BehaviorOphysDataXforms)
 
 
-class BehaviorOphysLimsApi(OphysLimsApi, BehaviorOphysBase):
+class BehaviorOphysLimsApi(BehaviorLimsApi, OphysLimsApi,
+                           BehaviorOphysDataXforms):
 
     def __init__(self, ophys_experiment_id: int,
-                 lims_credentials: Optional[DbCredentials] = None):
-        super().__init__(ophys_experiment_id, lims_credentials)
+                 lims_credentials: Optional[DbCredentials] = None,
+                 mtrain_credentials: Optional[DbCredentials] = None):
+
         self.logger = logging.getLogger(self.__class__.__name__)
 
-    @memoize
-    def get_sync_data(self):
-        sync_path = self.get_sync_file()
-        return get_sync_data(sync_path)
+        self.lims_db = db_connection_creator(
+            credentials=lims_credentials,
+            default_credentials=LIMS_DB_CREDENTIAL_MAP)
 
-    @memoize
-    def get_stimulus_timestamps(self):
-        sync_path = self.get_sync_file()
-        timestamps, _, _ = (OphysTimeAligner(sync_file=sync_path)
-                            .corrected_stim_timestamps)
-        return timestamps
-
-    @staticmethod
-    def _process_ophys_plane_timestamps(
-            ophys_timestamps: np.ndarray, plane_group: Optional[int],
-            group_count: int):
-        """
-        On mesoscope rigs each frame corresponds to a different imaging plane;
-        the laser moves between N pairs of planes. So, every Nth 2P
-        frame time in the sync file corresponds to a given plane (and
-        its multiplexed pair). The order in which the planes are
-        acquired dictates which timestamps should be assigned to which
-        plane pairs. The planes are acquired in ascending order, where
-        plane_group=0 is the first group of planes.
-
-        If the plane group is None (indicating it does not belong to
-        a plane group), then the plane was not collected concurrently
-        and the data do not need to be resampled. This is the case for
-        Scientifica 2p data, for example.
-
-        Parameters
-        ----------
-        ophys_timestamps: np.ndarray
-            Array of timestamps for 2p data
-        plane_group: int
-            The plane group this experiment belongs to. Signals the
-            order of acquisition.
-        group_count: int
-            The total number of plane groups acquired.
-        """
-        if (group_count == 0) or (plane_group is None):
-            return ophys_timestamps
-        resampled = ophys_timestamps[plane_group::group_count]
-        return resampled
+        self.ophys_experiment_id = ophys_experiment_id
+        super().__init__(behavior_session_id=self.get_behavior_session_id(),
+                         lims_credentials=lims_credentials,
+                         mtrain_credentials=mtrain_credentials)
 
     @memoize
     def get_ophys_timestamps(self):
@@ -133,6 +76,24 @@ def get_ophys_timestamps(self):
                     f"number of split timestamps (len={num_of_timestamps}).")
         return ophys_timestamps
 
+    def get_behavior_session_id(self):
+        query = '''
+                SELECT bs.id FROM behavior_sessions bs
+                JOIN ophys_sessions os ON os.id = bs.ophys_session_id
+                JOIN ophys_experiments oe ON oe.ophys_session_id = os.id
+                WHERE oe.id = {};
+                '''.format(self.get_ophys_experiment_id())
+        return self.lims_db.fetchone(query, strict=True)
+
+    @memoize
+    def get_ophys_session_id(self):
+        query = '''
+                SELECT os.id FROM ophys_sessions os
+                JOIN ophys_experiment oe ON oe.ophys_session_id = os.id
+                WHERE oe.id = {};
+                '''.format(self.get_ophys_experiment_id())
+        return self.lims_db.fetchone(query, strict=False)
+
     @memoize
     def get_experiment_container_id(self):
         query = '''
@@ -154,172 +115,6 @@ def get_behavior_stimulus_file(self):
                 '''.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
-    def get_behavior_session_uuid(self):
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        return data['session_uuid']
-
-    @memoize
-    def get_stimulus_frame_rate(self):
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        return np.round(1 / np.mean(np.diff(stimulus_timestamps)), 0)
-
-    @memoize
-    def get_ophys_frame_rate(self):
-        ophys_timestamps = self.get_ophys_timestamps()
-        return np.round(1 / np.mean(np.diff(ophys_timestamps)), 0)
-
-    @memoize
-    def get_metadata(self):
-
-        metadata = super().get_metadata()
-        metadata['ophys_experiment_id'] = self.get_ophys_experiment_id()
-        metadata['experiment_container_id'] = self.get_experiment_container_id()
-        metadata['ophys_frame_rate'] = self.get_ophys_frame_rate()
-        metadata['stimulus_frame_rate'] = self.get_stimulus_frame_rate()
-        metadata['targeted_structure'] = self.get_targeted_structure()
-        metadata['imaging_depth'] = self.get_imaging_depth()
-        metadata['session_type'] = self.get_stimulus_name()
-        metadata['experiment_datetime'] = self.get_experiment_date()
-        metadata['reporter_line'] = self.get_reporter_line()
-        metadata['driver_line'] = self.get_driver_line()
-        metadata['LabTracks_ID'] = self.get_external_specimen_name()
-        metadata['full_genotype'] = self.get_full_genotype()
-        metadata['behavior_session_uuid'] = uuid.UUID(self.get_behavior_session_uuid())
-        metadata["imaging_plane_group"] = self.get_imaging_plane_group()
-
-        return metadata
-
-    @memoize
-    def get_dff_traces(self):
-        dff_traces = self.get_raw_dff_data()
-        cell_roi_id_list = self.get_cell_roi_ids()
-        df = pd.DataFrame({'dff': [x for x in dff_traces]}, index=pd.Index(cell_roi_id_list, name='cell_roi_id'))
-
-        cell_specimen_table = self.get_cell_specimen_table()
-        df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id')
-        return df
-
-    @memoize
-    def get_running_data_df(self, lowpass=True):
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        return get_running_df(data, stimulus_timestamps, lowpass=lowpass)
-
-    @memoize
-    def get_running_speed(self, lowpass=True):
-        running_data_df = self.get_running_data_df(lowpass=lowpass)
-        assert running_data_df.index.name == 'timestamps'
-        return RunningSpeed(timestamps=running_data_df.index.values,
-                            values=running_data_df.speed.values)
-
-    @memoize
-    def get_stimulus_presentations(self):
-        stimulus_timestamps = self.get_stimulus_timestamps()
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        stimulus_presentations_df_pre = get_stimulus_presentations(data, stimulus_timestamps)
-        stimulus_metadata_df = get_stimulus_metadata(data)
-        idx_name = stimulus_presentations_df_pre.index.name
-        stimulus_index_df = stimulus_presentations_df_pre.reset_index().merge(stimulus_metadata_df.reset_index(), on=['image_name']).set_index(idx_name)
-        stimulus_index_df.sort_index(inplace=True)
-        stimulus_index_df = stimulus_index_df[['image_set', 'image_index', 'start_time',
-                                               'phase', 'spatial_frequency']].rename(columns={'start_time': 'timestamps'})
-        stimulus_index_df.set_index('timestamps', inplace=True, drop=True)
-        stimulus_presentations_df = stimulus_presentations_df_pre.merge(stimulus_index_df, left_on='start_time', right_index=True, how='left')
-        assert len(stimulus_presentations_df_pre) == len(stimulus_presentations_df)
-
-        return stimulus_presentations_df[sorted(stimulus_presentations_df.columns)]
-
-    @memoize
-    def get_stimulus_templates(self):
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        return get_stimulus_templates(data)
-
-    @memoize
-    def get_sync_licks(self):
-        lick_times = self.get_sync_data()['lick_times']
-        return pd.DataFrame({'time': lick_times})
-
-    @memoize
-    def get_licks(self):
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        rebase_function = self.get_stimulus_rebase_function()
-        # Get licks from pickle file (need to add an offset to align with
-        # the trial_log time stream)
-        lick_frames = (data["items"]["behavior"]["lick_sensors"][0]
-                       ["lick_events"])
-        vsyncs = data["items"]["behavior"]["intervalsms"]
-        vsync_times_raw = np.hstack((0, vsyncs)).cumsum() / 1000.0  # cumulative time
-        vsync_offset = frame_time_offset(data)
-        vsync_times = vsync_times_raw + vsync_offset
-        lick_times = [vsync_times[frame] for frame in lick_frames]
-        # Align pickle data with sync time stream
-        return pd.DataFrame({"time": list(map(rebase_function, lick_times))})
-
-    @memoize
-    def get_rewards(self):
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        rebase_function = self.get_stimulus_rebase_function()
-        return get_rewards(data, rebase_function)
-
-    @memoize
-    def get_task_parameters(self):
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        return get_task_parameters(data)
-
-    @memoize
-    def get_trials(self):
-
-        licks = self.get_licks()
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        rewards = self.get_rewards()
-        stimulus_presentations = self.get_stimulus_presentations()
-        rebase_function = self.get_stimulus_rebase_function()
-        trial_df = get_trials(data, licks, rewards, stimulus_presentations, rebase_function)
-
-        return trial_df
-
-    @memoize
-    def get_corrected_fluorescence_traces(self):
-        demix_file = self.get_demix_file()
-
-        g = h5py.File(demix_file)
-        corrected_fluorescence_trace_array = np.asarray(g['data'])
-        g.close()
-
-        cell_roi_id_list = self.get_cell_roi_ids()
-        ophys_timestamps = self.get_ophys_timestamps()
-        assert corrected_fluorescence_trace_array.shape[1], ophys_timestamps.shape[0]
-        df = pd.DataFrame({'corrected_fluorescence': list(corrected_fluorescence_trace_array)}, index=pd.Index(cell_roi_id_list, name='cell_roi_id'))
-
-        cell_specimen_table = self.get_cell_specimen_table()
-        df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id')
-        return df
-
-    @memoize
-    def get_average_projection(self, image_api=None):
-
-        if image_api is None:
-            image_api = ImageApi
-
-        avgint_a1X_file = self.get_average_intensity_projection_image_file()
-        pixel_size = self.get_surface_2p_pixel_size_um()
-        average_image = mpimg.imread(avgint_a1X_file)
-        return ImageApi.serialize(average_image, [pixel_size / 1000., pixel_size / 1000.], 'mm')
-
-    @memoize
-    def get_motion_correction(self):
-        motion_correction_filepath = self.get_rigid_motion_transform_file()
-        motion_correction = pd.read_csv(motion_correction_filepath)
-        return motion_correction[['x', 'y']]
-
     @memoize
     def get_nwb_filepath(self):
 
@@ -331,18 +126,6 @@ def get_nwb_filepath(self):
                 '''.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
-    def get_stimulus_rebase_function(self):
-        stimulus_timestamps_no_monitor_delay = self.get_sync_data()['stimulus_times_no_delay']
-        behavior_stimulus_file = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(behavior_stimulus_file)
-        stimulus_rebase_function = get_stimulus_rebase_function(data, stimulus_timestamps_no_monitor_delay)
-        return stimulus_rebase_function
-
-    def get_extended_trials(self):
-        filename = self.get_behavior_stimulus_file()
-        data = pd.read_pickle(filename)
-        return get_extended_trials(data)
-
     @memoize
     def get_eye_tracking_filepath(self):
         query = '''SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file
@@ -354,31 +137,6 @@ def get_eye_tracking_filepath(self):
                    '''.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
-    def get_eye_tracking(self,
-                         z_threshold: float = 3.0,
-                         dilation_frames: int = 2):
-        logger = logging.getLogger("BehaviorOphysLimsApi")
-
-        logger.info(f"Getting eye_tracking_data with "
-                    f"'z_threshold={z_threshold}', "
-                    f"'dilation_frames={dilation_frames}'")
-
-        filepath = Path(self.get_eye_tracking_filepath())
-        sync_path = Path(self.get_sync_file())
-
-        eye_tracking_data = load_eye_tracking_hdf(filepath)
-        frame_times = sync_utilities.get_synchronized_frame_times(
-            session_sync_file=sync_path,
-            sync_line_label_keys=Dataset.EYE_TRACKING_KEYS,
-            trim_after_spike=False)
-
-        eye_tracking_data = process_eye_tracking_data(eye_tracking_data,
-                                                      frame_times,
-                                                      z_threshold,
-                                                      dilation_frames)
-
-        return eye_tracking_data
-
     @staticmethod
     def get_ophys_experiment_df():
 
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
index 4b2f70001..72e42208e 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
@@ -130,7 +130,7 @@ def get_ophys_session_id(self) -> int:
         raise NotImplementedError()
 
     # TODO: Implement save and load of eye_tracking_data to/from NWB file
-    def get_eye_tracking_data(self) -> int:
+    def get_eye_tracking(self) -> int:
         raise NotImplementedError()
 
     def get_running_data_df(self, **kwargs):
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
index e6683ced5..e210a7ae6 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
@@ -281,12 +281,6 @@ def get_dff_file(self):
                 '''.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
-    @memoize
-    def get_cell_roi_ids(self):
-        cell_specimen_table = self.get_cell_specimen_table()
-        assert cell_specimen_table.index.name == 'cell_specimen_id'
-        return cell_specimen_table['cell_roi_id'].values
-
     @memoize
     def get_objectlist_file(self):
         query = '''
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_processing/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_processing/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/__init__.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/__init__.py
new file mode 100644
index 000000000..6a8954618
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/__init__.py
@@ -0,0 +1,2 @@
+from allensdk.brain_observatory.behavior.session_apis.data_transforms.behavior_data_xforms import BehaviorDataXforms  # noqa: F401, E501
+from allensdk.brain_observatory.behavior.session_apis.data_transforms.behavior_ophys_data_xforms import BehaviorOphysDataXforms  # noqa: F401, E501
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
new file mode 100644
index 000000000..e180fc7df
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
@@ -0,0 +1,287 @@
+from typing import Dict, Optional, Any
+from datetime import datetime
+import pytz
+import uuid
+
+import numpy as np
+import pandas as pd
+
+from allensdk.core.exceptions import DataFrameIndexError
+from allensdk.api.cache import memoize
+from allensdk.brain_observatory.behavior.session_apis.abcs import (
+    BehaviorBase)
+from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
+from allensdk.brain_observatory.behavior.running_processing import (
+    get_running_df)
+from allensdk.brain_observatory.behavior.stimulus_processing import (
+    get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata)
+from allensdk.brain_observatory.running_speed import RunningSpeed
+from allensdk.brain_observatory.behavior.metadata_processing import (
+    get_task_parameters)
+from allensdk.brain_observatory.behavior.sync import frame_time_offset
+from allensdk.brain_observatory.behavior.trials_processing import (
+    get_trials, get_extended_trials)
+
+
+class BehaviorDataXforms(BehaviorBase):
+
+    def get_behavior_session_id(self) -> int:
+        """Getter to be consistent with BehaviorOphysLimsApi."""
+        return self.behavior_session_id
+
+    @memoize
+    def _behavior_stimulus_file(self) -> pd.DataFrame:
+        """Helper method to cache stimulus file in memory since it takes about
+        a second to load (and is used in many methods).
+        """
+        return pd.read_pickle(self.get_behavior_stimulus_file())
+
+    def get_behavior_session_uuid(self) -> Optional[int]:
+        """Get the universally unique identifier (UUID) number for the
+        current behavior session.
+        """
+        data = self._behavior_stimulus_file()
+        behavior_pkl_uuid = data.get("session_uuid")
+        # Sanity check to ensure that pkl file data matches up with
+        # the behavior session that the pkl file has been associated with.
+        assert_err_msg = (
+            f"The behavior session UUID ({behavior_pkl_uuid}) in the behavior "
+            f"stimulus *.pkl file ({self.get_behavior_stimulus_file()}) does "
+            f"does not match the LIMS UUID ({self.foraging_id}) for "
+            f"behavior session: {self.behavior_session_id}")
+        assert behavior_pkl_uuid == self.foraging_id, assert_err_msg
+        return behavior_pkl_uuid
+
+    def get_licks(self) -> pd.DataFrame:
+        """Get lick data from pkl file.
+        This function assumes that the first sensor in the list of
+        lick_sensors is the desired lick sensor. If this changes we need
+        to update to get the proper line.
+
+        Since licks can occur outside of a trial context, the lick times
+        are extracted from the vsyncs and the frame number in `lick_events`.
+        Since we don't have a timestamp for when in "experiment time" the
+        vsync stream starts (from self.get_stimulus_timestamps), we compute
+        it by fitting a linear regression (frame number x time) for the
+        `start_trial` and `end_trial` events in the `trial_log`, to true
+        up these time streams.
+
+        :returns: pd.DataFrame -- A dataframe containing lick timestamps
+        """
+        # Get licks from pickle file instead of sync
+        data = self._behavior_stimulus_file()
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        lick_frames = (data["items"]["behavior"]["lick_sensors"][0]
+                       ["lick_events"])
+        lick_times = [stimulus_timestamps[frame] for frame in lick_frames]
+        return pd.DataFrame({"time": lick_times})
+
+    def get_rewards(self) -> pd.DataFrame:
+        """Get reward data from pkl file, based on pkl file timestamps
+        (not sync file).
+
+        :returns: pd.DataFrame -- A dataframe containing timestamps of
+            delivered rewards.
+        """
+        data = self._behavior_stimulus_file()
+        offset = frame_time_offset(data)
+        # No sync timestamps to rebase on, but do need to align to
+        # trial events, so add the offset as the "rebase" function
+        return get_rewards(data, lambda x: x + offset)
+
+    def get_running_data_df(self, lowpass=True) -> pd.DataFrame:
+        """Get running speed data.
+
+        :returns: pd.DataFrame -- dataframe containing various signals used
+            to compute running speed.
+        """
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        data = self._behavior_stimulus_file()
+        return get_running_df(data, stimulus_timestamps, lowpass=lowpass)
+
+    def get_running_speed(self, lowpass=True) -> RunningSpeed:
+        """Get running speed using timestamps from
+        self.get_stimulus_timestamps.
+
+        NOTE: Do not correct for monitor delay.
+
+        :returns: RunningSpeed -- a NamedTuple containing the subject's
+            timestamps and running speeds (in cm/s)
+        """
+        running_data_df = self.get_running_data_df(lowpass=lowpass)
+        if running_data_df.index.name != "timestamps":
+            raise DataFrameIndexError(
+                f"Expected index to be named 'timestamps' but got "
+                f"'{running_data_df.index.name}'.")
+        return RunningSpeed(timestamps=running_data_df.index.values,
+                            values=running_data_df.speed.values)
+
+    def get_stimulus_frame_rate(self) -> float:
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        return np.round(1 / np.mean(np.diff(stimulus_timestamps)), 0)
+
+    def get_stimulus_presentations(self) -> pd.DataFrame:
+        """Get stimulus presentation data.
+
+        NOTE: Uses timestamps that do not account for monitor delay.
+
+        :returns: pd.DataFrame --
+            Table whose rows are stimulus presentations
+            (i.e. a given image, for a given duration, typically 250 ms)
+            and whose columns are presentation characteristics.
+        """
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        data = self._behavior_stimulus_file()
+        raw_stim_pres_df = get_stimulus_presentations(
+            data, stimulus_timestamps)
+
+        # Fill in nulls for image_name
+        # This makes two assumptions:
+        #   1. Nulls in `image_name` should be "gratings_<orientation>"
+        #   2. Gratings are only present (or need to be fixed) when all
+        #      values for `image_name` are null.
+        if pd.isnull(raw_stim_pres_df["image_name"]).all():
+            if ~pd.isnull(raw_stim_pres_df["orientation"]).all():
+                raw_stim_pres_df["image_name"] = (
+                    raw_stim_pres_df["orientation"]
+                    .apply(lambda x: f"gratings_{x}"))
+            else:
+                raise ValueError("All values for 'orentation' and 'image_name'"
+                                 " are null.")
+
+        stimulus_metadata_df = get_stimulus_metadata(data)
+
+        idx_name = raw_stim_pres_df.index.name
+        stimulus_index_df = (
+            raw_stim_pres_df
+            .reset_index()
+            .merge(stimulus_metadata_df.reset_index(), on=["image_name"])
+            .set_index(idx_name))
+        stimulus_index_df = (
+            stimulus_index_df[["image_set", "image_index", "start_time",
+                               "phase", "spatial_frequency"]]
+            .rename(columns={"start_time": "timestamps"})
+            .sort_index()
+            .set_index("timestamps", drop=True))
+        stim_pres_df = raw_stim_pres_df.merge(
+            stimulus_index_df, left_on="start_time", right_index=True,
+            how="left")
+        if len(raw_stim_pres_df) != len(stim_pres_df):
+            raise ValueError("Length of `stim_pres_df` should not change after"
+                             f" merge; was {len(raw_stim_pres_df)}, now "
+                             f" {len(stim_pres_df)}.")
+        return stim_pres_df[sorted(stim_pres_df)]
+
+    def get_stimulus_templates(self) -> Dict[str, np.ndarray]:
+        """Get stimulus templates (movies, scenes) for behavior session.
+
+        Returns
+        -------
+        Dict[str, np.ndarray]
+            A dictionary containing the stimulus images presented during the
+            session. Keys are data set names, and values are 3D numpy arrays.
+        """
+        data = self._behavior_stimulus_file()
+        return get_stimulus_templates(data)
+
+    def get_stimulus_timestamps(self) -> np.ndarray:
+        """Get stimulus timestamps (vsyncs) from pkl file. Align to the
+        (frame, time) points in the trial events.
+
+        NOTE: Located with behavior_session_id. Does not use the sync_file
+        which requires ophys_session_id.
+
+        Returns
+        -------
+        np.ndarray
+            Timestamps associated with stimulus presentations on the monitor
+            that do no account for monitor delay.
+        """
+        data = self._behavior_stimulus_file()
+        vsyncs = data["items"]["behavior"]["intervalsms"]
+        cum_sum = np.hstack((0, vsyncs)).cumsum() / 1000.0  # cumulative time
+        offset = frame_time_offset(data)
+        return cum_sum + offset
+
+    def get_task_parameters(self) -> dict:
+        """Get task parameters from pkl file.
+
+        Returns
+        -------
+        dict
+            A dictionary containing parameters used to define the task runtime
+            behavior.
+        """
+        data = self._behavior_stimulus_file()
+        return get_task_parameters(data)
+
+    def get_trials(self) -> pd.DataFrame:
+        """Get trials from pkl file
+
+        Returns
+        -------
+        pd.DataFrame
+            A dataframe containing behavioral trial start/stop times,
+            and trial data
+        """
+        licks = self.get_licks()
+        data = self._behavior_stimulus_file()
+        rewards = self.get_rewards()
+        stimulus_presentations = self.get_stimulus_presentations()
+        # Pass a dummy rebase function since we don't have two time streams,
+        # and the frame times are already aligned to trial events in their
+        # respective getters
+        trial_df = get_trials(data, licks, rewards, stimulus_presentations,
+                              lambda x: x)
+
+        return trial_df
+
+    def get_extended_trials(self) -> pd.DataFrame:
+        """Get extended trials from pkl file
+
+        Returns
+        -------
+        pd.DataFrame
+            A dataframe containing extended behavior trial information.
+        """
+        filename = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(filename)
+        return get_extended_trials(data)
+
+    @memoize
+    def get_experiment_date(self) -> datetime:
+        """Return timestamp the behavior stimulus file began recording in UTC
+        :rtype: datetime
+        """
+        data = self._behavior_stimulus_file()
+        # Assuming file has local time of computer (Seattle)
+        tz = pytz.timezone("America/Los_Angeles")
+        return tz.localize(data["start_time"]).astimezone(pytz.utc)
+
+    def get_metadata(self) -> Dict[str, Any]:
+        """Return metadata about the session.
+        :rtype: dict
+        """
+        if self.get_behavior_session_uuid() is None:
+            bs_uuid = None
+        else:
+            bs_uuid = uuid.UUID(self.get_behavior_session_uuid())
+        metadata = {
+            "rig_name": self.get_rig_name(),
+            "sex": self.get_sex(),
+            "age": self.get_age(),
+            "ophys_experiment_id": self.ophys_experiment_ids,
+            "experiment_container_id": self.ophys_container_id,
+            "stimulus_frame_rate": self.get_stimulus_frame_rate(),
+            "session_type": self.get_stimulus_name(),
+            "experiment_datetime": self.get_experiment_date(),
+            "reporter_line": self.get_reporter_line(),
+            "driver_line": self.get_driver_line(),
+            "LabTracks_ID": self.get_external_specimen_name(),
+            "full_genotype": self.get_full_genotype(),
+            "behavior_session_uuid": bs_uuid,
+            "foraging_id": self.foraging_id,
+            "behavior_session_id": self.behavior_session_id,
+            "behavior_training_id": self.behavior_training_id,
+        }
+        return metadata
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
new file mode 100644
index 000000000..0b8e2a888
--- /dev/null
+++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
@@ -0,0 +1,344 @@
+import logging
+from pathlib import Path
+from typing import Optional
+import uuid
+
+import h5py
+import matplotlib.image as mpimg  # NOQA: E402
+import numpy as np
+import pandas as pd
+
+from allensdk.api.cache import memoize
+from allensdk.brain_observatory.behavior.session_apis.abcs import (
+    BehaviorOphysBase)
+
+from allensdk.brain_observatory.behavior.trials_processing import (
+    get_extended_trials)
+from allensdk.brain_observatory.behavior.sync import (
+    get_sync_data, get_stimulus_rebase_function, frame_time_offset)
+from allensdk.brain_observatory.sync_dataset import Dataset
+from allensdk.brain_observatory import sync_utilities
+from allensdk.internal.brain_observatory.time_sync import OphysTimeAligner
+from allensdk.brain_observatory.behavior.stimulus_processing import (
+    get_stimulus_presentations, get_stimulus_templates, get_stimulus_metadata)
+from allensdk.brain_observatory.behavior.metadata_processing import (
+    get_task_parameters)
+from allensdk.brain_observatory.behavior.running_processing import (
+    get_running_df)
+from allensdk.brain_observatory.behavior.rewards_processing import get_rewards
+from allensdk.brain_observatory.behavior.trials_processing import get_trials
+from allensdk.brain_observatory.behavior.eye_tracking_processing import (
+    load_eye_tracking_hdf, process_eye_tracking_data)
+from allensdk.brain_observatory.running_speed import RunningSpeed
+from allensdk.brain_observatory.behavior.image_api import ImageApi
+
+
+class BehaviorOphysDataXforms(BehaviorOphysBase):
+
+    @memoize
+    def get_sync_data(self):
+        sync_path = self.get_sync_file()
+        return get_sync_data(sync_path)
+
+    @memoize
+    def get_stimulus_timestamps(self):
+        sync_path = self.get_sync_file()
+        timestamps, _, _ = (OphysTimeAligner(sync_file=sync_path)
+                            .corrected_stim_timestamps)
+        return timestamps
+
+    @staticmethod
+    def _process_ophys_plane_timestamps(
+            ophys_timestamps: np.ndarray, plane_group: Optional[int],
+            group_count: int):
+        """
+        On mesoscope rigs each frame corresponds to a different imaging plane;
+        the laser moves between N pairs of planes. So, every Nth 2P
+        frame time in the sync file corresponds to a given plane (and
+        its multiplexed pair). The order in which the planes are
+        acquired dictates which timestamps should be assigned to which
+        plane pairs. The planes are acquired in ascending order, where
+        plane_group=0 is the first group of planes.
+
+        If the plane group is None (indicating it does not belong to
+        a plane group), then the plane was not collected concurrently
+        and the data do not need to be resampled. This is the case for
+        Scientifica 2p data, for example.
+
+        Parameters
+        ----------
+        ophys_timestamps: np.ndarray
+            Array of timestamps for 2p data
+        plane_group: int
+            The plane group this experiment belongs to. Signals the
+            order of acquisition.
+        group_count: int
+            The total number of plane groups acquired.
+        """
+        if (group_count == 0) or (plane_group is None):
+            return ophys_timestamps
+        resampled = ophys_timestamps[plane_group::group_count]
+        return resampled
+
+    @memoize
+    def get_ophys_timestamps(self):
+
+        ophys_timestamps = self.get_sync_data()['ophys_frames']
+        dff_traces = self.get_raw_dff_data()
+        number_of_cells, number_of_dff_frames = dff_traces.shape
+        num_of_timestamps = len(ophys_timestamps)
+        if number_of_dff_frames < num_of_timestamps:
+            ophys_timestamps = ophys_timestamps[:number_of_dff_frames]
+        elif number_of_dff_frames == num_of_timestamps:
+            pass
+        else:
+            raise RuntimeError('dff_frames is longer than timestamps')
+
+        # Resample if collecting multiple concurrent planes (e.g. mesoscope)
+        plane_group = self.get_imaging_plane_group()
+        if plane_group is not None:
+            group_count = self.get_plane_group_count()
+            ophys_timestamps = self._process_ophys_plane_timestamps(
+                ophys_timestamps, plane_group, group_count)
+        return ophys_timestamps
+
+    def get_behavior_session_uuid(self):
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        return data['session_uuid']
+
+    @memoize
+    def get_stimulus_frame_rate(self):
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        return np.round(1 / np.mean(np.diff(stimulus_timestamps)), 0)
+
+    @memoize
+    def get_ophys_frame_rate(self):
+        ophys_timestamps = self.get_ophys_timestamps()
+        return np.round(1 / np.mean(np.diff(ophys_timestamps)), 0)
+
+    @memoize
+    def get_metadata(self):
+        mdata = dict()
+        mdata['ophys_experiment_id'] = self.get_ophys_experiment_id()
+        mdata['experiment_container_id'] = self.get_experiment_container_id()
+        mdata['ophys_frame_rate'] = self.get_ophys_frame_rate()
+        mdata['stimulus_frame_rate'] = self.get_stimulus_frame_rate()
+        mdata['targeted_structure'] = self.get_targeted_structure()
+        mdata['imaging_depth'] = self.get_imaging_depth()
+        mdata['session_type'] = self.get_stimulus_name()
+        mdata['experiment_datetime'] = self.get_experiment_date()
+        mdata['reporter_line'] = self.get_reporter_line()
+        mdata['driver_line'] = self.get_driver_line()
+        mdata['LabTracks_ID'] = self.get_external_specimen_name()
+        mdata['full_genotype'] = self.get_full_genotype()
+        behavior_session_uuid = self.get_behavior_session_uuid()
+        mdata['behavior_session_uuid'] = uuid.UUID(behavior_session_uuid)
+        mdata["imaging_plane_group"] = self.get_imaging_plane_group()
+
+        return mdata
+
+    @memoize
+    def get_cell_roi_ids(self):
+        cell_specimen_table = self.get_cell_specimen_table()
+        assert cell_specimen_table.index.name == 'cell_specimen_id'
+        return cell_specimen_table['cell_roi_id'].values
+
+    @memoize
+    def get_dff_traces(self):
+        dff_traces = self.get_raw_dff_data()
+        cell_roi_id_list = self.get_cell_roi_ids()
+        df = pd.DataFrame({'dff': [x for x in dff_traces]},
+                          index=pd.Index(cell_roi_id_list,
+                          name='cell_roi_id'))
+
+        cell_specimen_table = self.get_cell_specimen_table()
+        df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id')
+        return df
+
+    @memoize
+    def get_running_data_df(self, lowpass=True):
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        return get_running_df(data, stimulus_timestamps, lowpass=lowpass)
+
+    @memoize
+    def get_running_speed(self, lowpass=True):
+        running_data_df = self.get_running_data_df(lowpass=lowpass)
+        assert running_data_df.index.name == 'timestamps'
+        return RunningSpeed(timestamps=running_data_df.index.values,
+                            values=running_data_df.speed.values)
+
+    @memoize
+    def get_stimulus_presentations(self):
+        stimulus_timestamps = self.get_stimulus_timestamps()
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        stim_presentations_df_pre = get_stimulus_presentations(
+            data, stimulus_timestamps)
+        stimulus_metadata_df = get_stimulus_metadata(data)
+        idx_name = stim_presentations_df_pre.index.name
+
+        stimulus_index_df = (
+            stim_presentations_df_pre.reset_index().merge(
+                stimulus_metadata_df.reset_index(),
+                on=['image_name']).set_index(idx_name))
+
+        stimulus_index_df.sort_index(inplace=True)
+
+        columns_to_select = ['image_set', 'image_index', 'start_time',
+                             'phase', 'spatial_frequency']
+
+        stimulus_index_df = (
+            stimulus_index_df[columns_to_select].rename(
+                columns={'start_time': 'timestamps'}))
+
+        stimulus_index_df.set_index('timestamps', inplace=True, drop=True)
+        stim_presentations_df = (
+            stim_presentations_df_pre.merge(stimulus_index_df,
+                                            left_on='start_time',
+                                            right_index=True, how='left'))
+
+        assert len(stim_presentations_df_pre) == len(stim_presentations_df)
+
+        sorted_column_names = sorted(stim_presentations_df.columns)
+        return stim_presentations_df[sorted_column_names]
+
+    @memoize
+    def get_stimulus_templates(self):
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        return get_stimulus_templates(data)
+
+    @memoize
+    def get_sync_licks(self):
+        lick_times = self.get_sync_data()['lick_times']
+        return pd.DataFrame({'time': lick_times})
+
+    @memoize
+    def get_licks(self):
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        rebase_function = self.get_stimulus_rebase_function()
+        # Get licks from pickle file (need to add an offset to align with
+        # the trial_log time stream)
+        lick_frames = (data["items"]["behavior"]["lick_sensors"][0]
+                       ["lick_events"])
+        vsyncs = data["items"]["behavior"]["intervalsms"]
+
+        # Cumulative time
+        vsync_times_raw = np.hstack((0, vsyncs)).cumsum() / 1000.0
+
+        vsync_offset = frame_time_offset(data)
+        vsync_times = vsync_times_raw + vsync_offset
+        lick_times = [vsync_times[frame] for frame in lick_frames]
+        # Align pickle data with sync time stream
+        return pd.DataFrame({"time": list(map(rebase_function, lick_times))})
+
+    @memoize
+    def get_rewards(self):
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        rebase_function = self.get_stimulus_rebase_function()
+        return get_rewards(data, rebase_function)
+
+    @memoize
+    def get_task_parameters(self):
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        return get_task_parameters(data)
+
+    @memoize
+    def get_trials(self):
+
+        licks = self.get_licks()
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        rewards = self.get_rewards()
+        stimulus_presentations = self.get_stimulus_presentations()
+        rebase_function = self.get_stimulus_rebase_function()
+        trial_df = get_trials(data, licks, rewards,
+                              stimulus_presentations, rebase_function)
+
+        return trial_df
+
+    @memoize
+    def get_corrected_fluorescence_traces(self):
+        demix_file = self.get_demix_file()
+
+        g = h5py.File(demix_file)
+        corrected_fluorescence_traces = np.asarray(g['data'])
+        g.close()
+
+        cell_roi_id_list = self.get_cell_roi_ids()
+        ophys_timestamps = self.get_ophys_timestamps()
+
+        num_trace_timepoints = corrected_fluorescence_traces.shape[1]
+        assert num_trace_timepoints, ophys_timestamps.shape[0]
+        df = pd.DataFrame(
+            {'corrected_fluorescence': list(corrected_fluorescence_traces)},
+            index=pd.Index(cell_roi_id_list, name='cell_roi_id'))
+
+        cell_specimen_table = self.get_cell_specimen_table()
+        df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id')
+        return df
+
+    @memoize
+    def get_average_projection(self, image_api=None):
+
+        if image_api is None:
+            image_api = ImageApi
+
+        avgint_a1X_file = self.get_average_intensity_projection_image_file()
+        pixel_size = self.get_surface_2p_pixel_size_um()
+        average_image = mpimg.imread(avgint_a1X_file)
+        return ImageApi.serialize(average_image, [pixel_size / 1000.,
+                                                  pixel_size / 1000.], 'mm')
+
+    @memoize
+    def get_motion_correction(self):
+        motion_correction_filepath = self.get_rigid_motion_transform_file()
+        motion_correction = pd.read_csv(motion_correction_filepath)
+        return motion_correction[['x', 'y']]
+
+    def get_stimulus_rebase_function(self):
+        stimulus_timestamps_no_monitor_delay = (
+            self.get_sync_data()['stimulus_times_no_delay'])
+
+        behavior_stimulus_file = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(behavior_stimulus_file)
+        stimulus_rebase_function = get_stimulus_rebase_function(
+            data, stimulus_timestamps_no_monitor_delay)
+
+        return stimulus_rebase_function
+
+    def get_extended_trials(self):
+        filename = self.get_behavior_stimulus_file()
+        data = pd.read_pickle(filename)
+        return get_extended_trials(data)
+
+    def get_eye_tracking(self,
+                         z_threshold: float = 3.0,
+                         dilation_frames: int = 2):
+        logger = logging.getLogger("BehaviorOphysLimsApi")
+
+        logger.info(f"Getting eye_tracking_data with "
+                    f"'z_threshold={z_threshold}', "
+                    f"'dilation_frames={dilation_frames}'")
+
+        filepath = Path(self.get_eye_tracking_filepath())
+        sync_path = Path(self.get_sync_file())
+
+        eye_tracking_data = load_eye_tracking_hdf(filepath)
+        frame_times = sync_utilities.get_synchronized_frame_times(
+            session_sync_file=sync_path,
+            sync_line_label_keys=Dataset.EYE_TRACKING_KEYS,
+            trim_after_spike=False)
+
+        eye_tracking_data = process_eye_tracking_data(eye_tracking_data,
+                                                      frame_times,
+                                                      z_threshold,
+                                                      dilation_frames)
+
+        return eye_tracking_data
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
index c6c19f34f..05526abb3 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py
@@ -15,8 +15,8 @@
 from allensdk.core.exceptions import DataFrameIndexError
 
 mock_db_credentials = DbCredentials(dbname='mock_db', user='mock_user',
-                                      host='mock_host', port='mock_port',
-                                      password='mock')
+                                    host='mock_host', port='mock_port',
+                                    password='mock')
 
 
 @pytest.fixture
@@ -31,10 +31,14 @@ def __init__(self):
             super().__init__(behavior_session_id=8675309,
                              lims_credentials=mock_db_credentials,
                              mtrain_credentials=mock_db_credentials)
+            self.foraging_id = 123456
 
         def _get_ids(self):
             return {}
 
+        def get_behavior_stimulus_file(self):
+            return "dummy_stimulus_file.pkl"
+
         def _behavior_stimulus_file(self):
             data = {
                 "items": {
@@ -80,7 +84,7 @@ def get_running_data_df(self, lowpass=True):
 def MockApiRunSpeedExpectedError():
     class MockApiRunSpeedExpectedError(BehaviorLimsApi):
         """
-        Mock class that overrides some functions to provide test data and 
+        Mock class that overrides some functions to provide test data and
         initialize without calls to db.
         """
         def __init__(self):
@@ -169,7 +173,7 @@ class TestBehaviorRegression:
     sources (and I'm not sure how it's uploaded in the database).
 
     Do not test `get_licks` regression because the licks come from two
-    different sources and are recorded differently (behavior pickle file in 
+    different sources and are recorded differently (behavior pickle file in
     BehaviorLimsApi; sync file in BehaviorOphysLimeApi)
     """
     @classmethod

From b2bc77f89e547812dc14481fc8307931c5a57122 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Fri, 20 Nov 2020 18:13:50 -0800
Subject: [PATCH 42/56] Fix tests; Move transforms from OphysLimsApi to
 BehaviorOphysDataXforms

---
 .../session_apis/data_io/behavior_lims_api.py |  10 +-
 .../data_io/behavior_ophys_json_api.py        |  12 ++
 .../data_io/behavior_ophys_lims_api.py        |  54 ++------
 .../session_apis/data_io/ophys_lims_api.py    |  65 ----------
 .../data_transforms/behavior_data_xforms.py   |   4 -
 .../behavior_ophys_data_xforms.py             | 118 ++++++++++++++----
 .../behavior/test_behavior_ophys_lims_api.py  |  13 +-
 .../behavior/test_behavior_ophys_session.py   |   2 +-
 8 files changed, 134 insertions(+), 144 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index 26722c9dc..912ebadd0 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -1,3 +1,4 @@
+import logging
 from typing import Dict, Optional, Union, List
 
 from datetime import datetime
@@ -23,6 +24,9 @@ class BehaviorLimsApi(BehaviorDataXforms, CachedInstanceMethodMixin):
     def __init__(self, behavior_session_id: int,
                  lims_credentials: Optional[DbCredentials] = None,
                  mtrain_credentials: Optional[DbCredentials] = None):
+
+        self.logger = logging.getLogger(self.__class__.__name__)
+
         self.mtrain_db = db_connection_creator(
             credentials=mtrain_credentials,
             default_credentials=MTRAIN_DB_CREDENTIAL_MAP)
@@ -90,7 +94,7 @@ def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]:
                 behavior_sessions.id = {self.behavior_session_id};
         """
         ids_response = self.lims_db.select(query)
-        if len(ids_response) > 1:
+        if len(ids_response) > 1 or len(ids_response) < 1:
             raise OneResultExpectedError
         ids_dict = ids_response.iloc[0].to_dict()
 
@@ -127,6 +131,10 @@ def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]:
                              "ophys_container_id": None})
         return ids_dict
 
+    def get_behavior_session_id(self) -> int:
+        """Getter to be consistent with BehaviorOphysLimsApi."""
+        return self.behavior_session_id
+
     def get_behavior_stimulus_file(self) -> str:
         """Return the path to the StimulusPickle file for a session.
         :rtype: str
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
index 3c5c4fb6d..d4371266e 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 import pytz
 
@@ -12,10 +13,21 @@ class BehaviorOphysJsonApi(BehaviorOphysDataXforms):
 
     def __init__(self, data):
         self.data = data
+        self.logger = logging.getLogger(self.__class__.__name__)
 
     def get_ophys_experiment_id(self):
         return self.data['ophys_experiment_id']
 
+    # TODO: This should be replaced with a dict lookup after the
+    # behavior_ophys_write_nwb LIMS strategy has been updated
+    def get_behavior_session_id(self):
+        NotImplementedError()
+
+    # TODO: This should be replaced with a dict lookup after the
+    # behavior_ophys_write_nwb LIMS strategy has been updated
+    def get_ophys_session_id(self):
+        NotImplementedError()
+
     def get_surface_2p_pixel_size_um(self):
         return self.data['surface_2p_pixel_size_um']
 
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index d03bb3852..b097989af 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -15,8 +15,8 @@
     BehaviorOphysDataXforms)
 
 
-class BehaviorOphysLimsApi(BehaviorLimsApi, OphysLimsApi,
-                           BehaviorOphysDataXforms):
+class BehaviorOphysLimsApi(BehaviorOphysDataXforms, BehaviorLimsApi,
+                           OphysLimsApi):
 
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None,
@@ -29,52 +29,12 @@ def __init__(self, ophys_experiment_id: int,
             default_credentials=LIMS_DB_CREDENTIAL_MAP)
 
         self.ophys_experiment_id = ophys_experiment_id
-        super().__init__(behavior_session_id=self.get_behavior_session_id(),
+        super().__init__(self.get_behavior_session_id(),
                          lims_credentials=lims_credentials,
                          mtrain_credentials=mtrain_credentials)
 
-    @memoize
-    def get_ophys_timestamps(self):
-
-        ophys_timestamps = self.get_sync_data()['ophys_frames']
-        dff_traces = self.get_raw_dff_data()
-        plane_group = self.get_imaging_plane_group()
-
-        number_of_cells, number_of_dff_frames = dff_traces.shape
-        # Scientifica data has extra frames in the sync file relative
-        # to the number of frames in the video. These sentinel frames
-        # should be removed.
-        # NOTE: This fix does not apply to mesoscope data.
-        # See http://confluence.corp.alleninstitute.org/x/9DVnAg
-        if plane_group is None:    # non-mesoscope
-            num_of_timestamps = len(ophys_timestamps)
-            if (number_of_dff_frames < num_of_timestamps):
-                self.logger.info(
-                    "Truncating acquisition frames ('ophys_frames') "
-                    f"(len={num_of_timestamps}) to the number of frames "
-                    f"in the df/f trace ({number_of_dff_frames}).")
-                ophys_timestamps = ophys_timestamps[:number_of_dff_frames]
-            elif number_of_dff_frames > num_of_timestamps:
-                raise RuntimeError(
-                    f"dff_frames (len={number_of_dff_frames}) is longer "
-                    f"than timestamps (len={num_of_timestamps}).")
-        # Mesoscope data
-        # Resample if collecting multiple concurrent planes (e.g. mesoscope)
-        # because the frames are interleaved
-        else:
-            group_count = self.get_plane_group_count()
-            self.logger.info(
-                "Mesoscope data detected. Splitting timestamps "
-                f"(len={len(ophys_timestamps)} over {group_count} "
-                "plane group(s).")
-            ophys_timestamps = self._process_ophys_plane_timestamps(
-                ophys_timestamps, plane_group, group_count)
-            num_of_timestamps = len(ophys_timestamps)
-            if number_of_dff_frames != num_of_timestamps:
-                raise RuntimeError(
-                    f"dff_frames (len={number_of_dff_frames}) is not equal to "
-                    f"number of split timestamps (len={num_of_timestamps}).")
-        return ophys_timestamps
+    def get_ophys_experiment_id(self):
+        return self.ophys_experiment_id
 
     def get_behavior_session_id(self):
         query = '''
@@ -83,7 +43,7 @@ def get_behavior_session_id(self):
                 JOIN ophys_experiments oe ON oe.ophys_session_id = os.id
                 WHERE oe.id = {};
                 '''.format(self.get_ophys_experiment_id())
-        return self.lims_db.fetchone(query, strict=True)
+        return self.lims_db.fetchone(query, strict=False)
 
     @memoize
     def get_ophys_session_id(self):
@@ -92,7 +52,7 @@ def get_ophys_session_id(self):
                 JOIN ophys_experiment oe ON oe.ophys_session_id = os.id
                 WHERE oe.id = {};
                 '''.format(self.get_ophys_experiment_id())
-        return self.lims_db.fetchone(query, strict=False)
+        return self.lims_db.fetchone(query, strict=True)
 
     @memoize
     def get_experiment_container_id(self):
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
index e210a7ae6..25eb13793 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
@@ -1,8 +1,3 @@
-import matplotlib.image as mpimg  # NOQA: D102
-import json
-import numpy as np
-import os
-import h5py
 import pytz
 import pandas as pd
 from typing import Optional
@@ -10,8 +5,6 @@
 from allensdk.internal.api import (
     PostgresQueryMixin, OneOrMoreResultExpectedError)
 from allensdk.api.cache import memoize
-from allensdk.brain_observatory.behavior.image_api import ImageApi
-import allensdk.brain_observatory.roi_masks as roi
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin
 from allensdk.core.authentication import credential_injector, DbCredentials
@@ -144,17 +137,6 @@ def get_max_projection_file(self):
                 '''.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
-    @memoize
-    def get_max_projection(self, image_api=None):
-
-        if image_api is None:
-            image_api = ImageApi
-
-        maxInt_a13_file = self.get_max_projection_file()
-        pixel_size = self.get_surface_2p_pixel_size_um()
-        max_projection = mpimg.imread(maxInt_a13_file)
-        return ImageApi.serialize(max_projection, [pixel_size / 1000., pixel_size / 1000.], 'mm')
-
     @memoize
     def get_targeted_structure(self):
         query = '''
@@ -344,12 +326,6 @@ def get_foraging_id(self):
                 '''.format(self.get_ophys_experiment_id())        
         return self.lims_db.fetchone(query, strict=True)
 
-    def get_raw_dff_data(self):
-        dff_path = self.get_dff_file()
-        with h5py.File(dff_path, 'r') as raw_file:
-            dff_traces = np.asarray(raw_file['data'])
-        return dff_traces
-
     @memoize
     def get_rig_name(self):
         query = '''
@@ -371,21 +347,6 @@ def get_field_of_view_shape(self):
         X = {c: self.lims_db.fetchone(query.format('oe.movie_{}'.format(c), self.get_ophys_experiment_id()), strict=True) for c in ['width', 'height']}
         return X
 
-    @memoize
-    def get_metadata(self):
-
-        metadata = {}
-        metadata['rig_name'] = self.get_rig_name()
-        metadata['sex'] = self.get_sex()
-        metadata['age'] = self.get_age()
-        metadata['excitation_lambda'] = 910.
-        metadata['emission_lambda'] = 520.
-        metadata['indicator'] = 'GCAMP6f'
-        metadata['field_of_view_width'] = self.get_field_of_view_shape()['width']
-        metadata['field_of_view_height'] = self.get_field_of_view_shape()['height']
-
-        return metadata
-
     @memoize
     def get_ophys_cell_segmentation_run_id(self):
         query = '''
@@ -408,32 +369,6 @@ def get_raw_cell_specimen_table_dict(self):
         cell_specimen_table.drop(['ophys_experiment_id', 'ophys_cell_segmentation_run_id'], inplace=True, axis=1)
         return cell_specimen_table.to_dict()
 
-    @memoize
-    def get_cell_specimen_table(self):
-        cell_specimen_table = pd.DataFrame.from_dict(self.get_raw_cell_specimen_table_dict()).set_index('cell_roi_id').sort_index()
-        fov_width, fov_height = self.get_field_of_view_shape()['width'], self.get_field_of_view_shape()['height']
-
-        # Convert cropped ROI masks to uncropped versions
-        image_mask_list = []
-        for cell_roi_id, table_row in cell_specimen_table.iterrows():
-            # Deserialize roi data into AllenSDK RoiMask object
-            curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height,
-                                   label=None, mask_group=-1)
-            curr_roi.x = table_row['x']
-            curr_roi.y = table_row['y']
-            curr_roi.width = table_row['width']
-            curr_roi.height = table_row['height']
-            curr_roi.mask = np.array(table_row['image_mask'])
-            image_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
-
-        cell_specimen_table['image_mask'] = image_mask_list
-        cell_specimen_table = cell_specimen_table[sorted(cell_specimen_table.columns)]
-
-        cell_specimen_table.index.rename('cell_roi_id', inplace=True)
-        cell_specimen_table.reset_index(inplace=True)
-        cell_specimen_table.set_index('cell_specimen_id', inplace=True)
-        return cell_specimen_table
-
     @memoize
     def get_surface_2p_pixel_size_um(self):
         query = '''
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
index e180fc7df..674fd2a9e 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
@@ -25,10 +25,6 @@
 
 class BehaviorDataXforms(BehaviorBase):
 
-    def get_behavior_session_id(self) -> int:
-        """Getter to be consistent with BehaviorOphysLimsApi."""
-        return self.behavior_session_id
-
     @memoize
     def _behavior_stimulus_file(self) -> pd.DataFrame:
         """Helper method to cache stimulus file in memory since it takes about
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
index 0b8e2a888..8d293ca13 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
@@ -31,10 +31,80 @@
     load_eye_tracking_hdf, process_eye_tracking_data)
 from allensdk.brain_observatory.running_speed import RunningSpeed
 from allensdk.brain_observatory.behavior.image_api import ImageApi
+import allensdk.brain_observatory.roi_masks as roi
 
 
 class BehaviorOphysDataXforms(BehaviorOphysBase):
 
+    @memoize
+    def get_cell_specimen_table(self):
+        cell_specimen_table = pd.DataFrame.from_dict(self.get_raw_cell_specimen_table_dict()).set_index('cell_roi_id').sort_index()
+        fov_width = self.get_field_of_view_shape()['width']
+        fov_height = self.get_field_of_view_shape()['height']
+
+        # Convert cropped ROI masks to uncropped versions
+        image_mask_list = []
+        for cell_roi_id, table_row in cell_specimen_table.iterrows():
+            # Deserialize roi data into AllenSDK RoiMask object
+            curr_roi = roi.RoiMask(image_w=fov_width, image_h=fov_height,
+                                   label=None, mask_group=-1)
+            curr_roi.x = table_row['x']
+            curr_roi.y = table_row['y']
+            curr_roi.width = table_row['width']
+            curr_roi.height = table_row['height']
+            curr_roi.mask = np.array(table_row['image_mask'])
+            image_mask_list.append(curr_roi.get_mask_plane().astype(np.bool))
+
+        cell_specimen_table['image_mask'] = image_mask_list
+        cell_specimen_table = cell_specimen_table[sorted(cell_specimen_table.columns)]
+
+        cell_specimen_table.index.rename('cell_roi_id', inplace=True)
+        cell_specimen_table.reset_index(inplace=True)
+        cell_specimen_table.set_index('cell_specimen_id', inplace=True)
+        return cell_specimen_table
+
+    @memoize
+    def get_ophys_timestamps(self):
+        ophys_timestamps = self.get_sync_data()['ophys_frames']
+        dff_traces = self.get_raw_dff_data()
+        plane_group = self.get_imaging_plane_group()
+
+        number_of_cells, number_of_dff_frames = dff_traces.shape
+        # Scientifica data has extra frames in the sync file relative
+        # to the number of frames in the video. These sentinel frames
+        # should be removed.
+        # NOTE: This fix does not apply to mesoscope data.
+        # See http://confluence.corp.alleninstitute.org/x/9DVnAg
+        if plane_group is None:    # non-mesoscope
+            num_of_timestamps = len(ophys_timestamps)
+            if (number_of_dff_frames < num_of_timestamps):
+                self.logger.info(
+                    "Truncating acquisition frames ('ophys_frames') "
+                    f"(len={num_of_timestamps}) to the number of frames "
+                    f"in the df/f trace ({number_of_dff_frames}).")
+                ophys_timestamps = ophys_timestamps[:number_of_dff_frames]
+            elif number_of_dff_frames > num_of_timestamps:
+                raise RuntimeError(
+                    f"dff_frames (len={number_of_dff_frames}) is longer "
+                    f"than timestamps (len={num_of_timestamps}).")
+        # Mesoscope data
+        # Resample if collecting multiple concurrent planes (e.g. mesoscope)
+        # because the frames are interleaved
+        else:
+            group_count = self.get_plane_group_count()
+            self.logger.info(
+                "Mesoscope data detected. Splitting timestamps "
+                f"(len={len(ophys_timestamps)} over {group_count} "
+                "plane group(s).")
+            ophys_timestamps = self._process_ophys_plane_timestamps(
+                ophys_timestamps, plane_group, group_count)
+            num_of_timestamps = len(ophys_timestamps)
+            if number_of_dff_frames != num_of_timestamps:
+                raise RuntimeError(
+                    f"dff_frames (len={number_of_dff_frames}) is not equal to "
+                    f"number of split timestamps (len={num_of_timestamps}).")
+        return ophys_timestamps
+
     @memoize
     def get_sync_data(self):
         sync_path = self.get_sync_file()
@@ -80,28 +150,6 @@ def _process_ophys_plane_timestamps(
         resampled = ophys_timestamps[plane_group::group_count]
         return resampled
 
-    @memoize
-    def get_ophys_timestamps(self):
-
-        ophys_timestamps = self.get_sync_data()['ophys_frames']
-        dff_traces = self.get_raw_dff_data()
-        number_of_cells, number_of_dff_frames = dff_traces.shape
-        num_of_timestamps = len(ophys_timestamps)
-        if number_of_dff_frames < num_of_timestamps:
-            ophys_timestamps = ophys_timestamps[:number_of_dff_frames]
-        elif number_of_dff_frames == num_of_timestamps:
-            pass
-        else:
-            raise RuntimeError('dff_frames is longer than timestamps')
-
-        # Resample if collecting multiple concurrent planes (e.g. mesoscope)
-        plane_group = self.get_imaging_plane_group()
-        if plane_group is not None:
-            group_count = self.get_plane_group_count()
-            ophys_timestamps = self._process_ophys_plane_timestamps(
-                ophys_timestamps, plane_group, group_count)
-        return ophys_timestamps
-
     def get_behavior_session_uuid(self):
         behavior_stimulus_file = self.get_behavior_stimulus_file()
         data = pd.read_pickle(behavior_stimulus_file)
@@ -135,6 +183,14 @@ def get_metadata(self):
         behavior_session_uuid = self.get_behavior_session_uuid()
         mdata['behavior_session_uuid'] = uuid.UUID(behavior_session_uuid)
         mdata["imaging_plane_group"] = self.get_imaging_plane_group()
+        mdata['rig_name'] = self.get_rig_name()
+        mdata['sex'] = self.get_sex()
+        mdata['age'] = self.get_age()
+        mdata['excitation_lambda'] = 910.
+        mdata['emission_lambda'] = 520.
+        mdata['indicator'] = 'GCAMP6f'
+        mdata['field_of_view_width'] = self.get_field_of_view_shape()['width']
+        mdata['field_of_view_height'] = self.get_field_of_view_shape()['height']
 
         return mdata
 
@@ -144,6 +200,12 @@ def get_cell_roi_ids(self):
         assert cell_specimen_table.index.name == 'cell_specimen_id'
         return cell_specimen_table['cell_roi_id'].values
 
+    def get_raw_dff_data(self):
+        dff_path = self.get_dff_file()
+        with h5py.File(dff_path, 'r') as raw_file:
+            dff_traces = np.asarray(raw_file['data'])
+        return dff_traces
+
     @memoize
     def get_dff_traces(self):
         dff_traces = self.get_raw_dff_data()
@@ -284,6 +346,18 @@ def get_corrected_fluorescence_traces(self):
         df = cell_specimen_table[['cell_roi_id']].join(df, on='cell_roi_id')
         return df
 
+    @memoize
+    def get_max_projection(self, image_api=None):
+
+        if image_api is None:
+            image_api = ImageApi
+
+        maxInt_a13_file = self.get_max_projection_file()
+        pixel_size = self.get_surface_2p_pixel_size_um()
+        max_projection = mpimg.imread(maxInt_a13_file)
+        return ImageApi.serialize(max_projection, [pixel_size / 1000.,
+                                                   pixel_size / 1000.], 'mm')
+
     @memoize
     def get_average_projection(self, image_api=None):
 
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
index 6d6b5a796..bf9a44652 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
@@ -27,16 +27,16 @@ def does_not_raise(enter_result=None):
 ])
 def test_get_behavior_stimulus_file(ophys_experiment_id, compare_val):
 
-    api = BehaviorOphysLimsApi(ophys_experiment_id)
-
     if compare_val is None:
         expected_fail = False
         try:
+            api = BehaviorOphysLimsApi(ophys_experiment_id)
             api.get_behavior_stimulus_file()
         except OneResultExpectedError:
             expected_fail = True
         assert expected_fail is True
     else:
+        api = BehaviorOphysLimsApi(ophys_experiment_id)
         assert api.get_behavior_stimulus_file() == compare_val
 
 
@@ -104,7 +104,7 @@ def test_process_ophys_plane_timestamps(
         (0, np.arange(20), np.arange(5).reshape(1, 5),
          None, pytest.raises(RuntimeError))
     ],
-    ids=["scientifica-trunate", "scientifica-raise", "mesoscope-good",
+    ids=["scientifica-truncate", "scientifica-raise", "mesoscope-good",
          "mesoscope-raise"]
 )
 def test_get_ophys_timestamps(monkeypatch, plane_group, ophys_timestamps,
@@ -112,6 +112,11 @@ def test_get_ophys_timestamps(monkeypatch, plane_group, ophys_timestamps,
     """Test the acquisition frame truncation only happens for
     non-mesoscope data (and raises error for scientifica data with
     longer trace frames than acquisition frames (ophys_timestamps))."""
+
+    monkeypatch.setattr(BehaviorOphysLimsApi,
+                        "get_behavior_session_id", lambda x: 123)
+    monkeypatch.setattr(BehaviorOphysLimsApi, "_get_ids", lambda x: {})
+
     api = BehaviorOphysLimsApi(123)
     # Mocking any db calls
     monkeypatch.setattr(api, "get_sync_data",
@@ -122,4 +127,4 @@ def test_get_ophys_timestamps(monkeypatch, plane_group, ophys_timestamps,
     with context:
         actual = api.get_ophys_timestamps()
         if expected is not None:
-            np.testing.assert_array_equal(expected, actual)
\ No newline at end of file
+            np.testing.assert_array_equal(expected, actual)
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
index 9301781e6..5c5dcd580 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
@@ -86,7 +86,7 @@ def test_visbeh_ophys_data_set():
                                  'session_type': 'OPHYS_6_images_B',
                                  'driver_line': ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'],
                                  'behavior_session_uuid': uuid.UUID('69cdbe09-e62b-4b42-aab1-54b5773dfe78'),
-                                 'experiment_datetime': pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 28, 37)),
+                                 'experiment_datetime': pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 58, 50, 325000)),
                                  'ophys_frame_rate': 31.0,
                                  'imaging_depth': 375,
                                  'LabTracks_ID': 416369,

From 75bbf6629f478fc5c745e18f14da0d374cfa4470 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 23 Nov 2020 16:42:17 -0800
Subject: [PATCH 43/56] Fix BehaviorOphysLimsApi inheritance issues

This commits fixes an inheritance order issue that was causing tests to
fail. Specifically:

- The BehaviorLimsApi inherits from BehaviorDataXforms so
  BehaviorOphysDataXforms should always be inherited before
  BehaviorLimsApi

- The OphysLimsApi methods should be used before BehaviorLimsApi methods

- The BehaviorOphysLimsApi class should not be calling super() in its
  __init__. Rather it should set and provide self.ophys_experiment_id,
  self.behavior_session_id, self.lims_db, and self.mtrain_db
---
 .../data_io/behavior_ophys_lims_api.py        | 24 +++++++------------
 .../session_apis/data_io/ophys_lims_api.py    | 15 ++++--------
 .../behavior/test_behavior_lims_api.py        |  3 ++-
 .../behavior/test_behavior_ophys_lims_api.py  | 10 ++++++++
 .../behavior/test_behavior_ophys_session.py   |  2 +-
 .../behavior/test_ophys_lims_api.py           |  9 -------
 6 files changed, 27 insertions(+), 36 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index b097989af..76f1c8427 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -9,14 +9,15 @@
     BehaviorLimsApi)
 from allensdk.internal.api import db_connection_creator, PostgresQueryMixin
 from allensdk.internal.core.lims_utilities import safe_system_path
-from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
+from allensdk.core.auth_config import (
+    LIMS_DB_CREDENTIAL_MAP, MTRAIN_DB_CREDENTIAL_MAP)
 from allensdk.core.authentication import credential_injector, DbCredentials
 from allensdk.brain_observatory.behavior.session_apis.data_transforms import (
     BehaviorOphysDataXforms)
 
 
-class BehaviorOphysLimsApi(BehaviorOphysDataXforms, BehaviorLimsApi,
-                           OphysLimsApi):
+class BehaviorOphysLimsApi(BehaviorOphysDataXforms,  OphysLimsApi,
+                           BehaviorLimsApi):
 
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None,
@@ -28,23 +29,16 @@ def __init__(self, ophys_experiment_id: int,
             credentials=lims_credentials,
             default_credentials=LIMS_DB_CREDENTIAL_MAP)
 
+        self.mtrain_db = db_connection_creator(
+            credentials=mtrain_credentials,
+            default_credentials=MTRAIN_DB_CREDENTIAL_MAP)
+
         self.ophys_experiment_id = ophys_experiment_id
-        super().__init__(self.get_behavior_session_id(),
-                         lims_credentials=lims_credentials,
-                         mtrain_credentials=mtrain_credentials)
+        self.behavior_session_id = self.get_behavior_session_id()
 
     def get_ophys_experiment_id(self):
         return self.ophys_experiment_id
 
-    def get_behavior_session_id(self):
-        query = '''
-                SELECT bs.id FROM behavior_sessions bs
-                JOIN ophys_sessions os ON os.id = bs.ophys_session_id
-                JOIN ophys_experiments oe ON oe.ophys_session_id = os.id
-                WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
-        return self.lims_db.fetchone(query, strict=False)
-
     @memoize
     def get_ophys_session_id(self):
         query = '''
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
index 25eb13793..030ef2783 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
@@ -3,7 +3,7 @@
 from typing import Optional
 
 from allensdk.internal.api import (
-    PostgresQueryMixin, OneOrMoreResultExpectedError)
+    PostgresQueryMixin, OneOrMoreResultExpectedError, db_connection_creator)
 from allensdk.api.cache import memoize
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin
@@ -16,15 +16,10 @@ class OphysLimsApi(CachedInstanceMethodMixin):
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None):
         self.ophys_experiment_id = ophys_experiment_id
-        if lims_credentials:
-            self.lims_db = PostgresQueryMixin(
-                dbname=lims_credentials.dbname, user=lims_credentials.user,
-                host=lims_credentials.host, password=lims_credentials.password,
-                port=lims_credentials.port)
-        else:
-            # Currying is equivalent to decorator syntactic sugar
-            self.lims_db = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
-                            (PostgresQueryMixin)())
+
+        self.lims_db = db_connection_creator(
+            credentials=lims_credentials,
+            default_credentials=LIMS_DB_CREDENTIAL_MAP)
 
     def get_ophys_experiment_id(self):
         return self.ophys_experiment_id
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py
index 00afc052d..b1a6a9767 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_lims_api.py
@@ -13,16 +13,17 @@
     pytest.param(0, None)
 ])
 def test_get_behavior_stimulus_file(behavior_experiment_id, compare_val):
-    api = BehaviorLimsApi(behavior_experiment_id)
 
     if compare_val is None:
         expected_fail = False
         try:
+            api = BehaviorLimsApi(behavior_experiment_id)
             api.get_behavior_stimulus_file()
         except OneResultExpectedError:
             expected_fail = True
         assert expected_fail is True
     else:
+        api = BehaviorLimsApi(behavior_experiment_id)
         assert api.get_behavior_stimulus_file() == compare_val
 
 
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
index bf9a44652..d79099dc8 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_lims_api.py
@@ -20,6 +20,16 @@ def does_not_raise(enter_result=None):
     yield enter_result
 
 
+
+@pytest.mark.requires_bamboo
+@pytest.mark.parametrize('ophys_experiment_id', [
+    pytest.param(511458874),
+])
+def test_get_cell_roi_table(ophys_experiment_id):
+    api = BehaviorOphysLimsApi(ophys_experiment_id)
+    assert len(api.get_cell_specimen_table()) == 128
+
+
 @pytest.mark.requires_bamboo
 @pytest.mark.parametrize('ophys_experiment_id, compare_val', [
     pytest.param(789359614, '/allen/programs/braintv/production/visualbehavior/prod0/specimen_756577249/behavior_session_789295700/789220000.pkl'),
diff --git a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
index 5c5dcd580..9301781e6 100644
--- a/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
+++ b/allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py
@@ -86,7 +86,7 @@ def test_visbeh_ophys_data_set():
                                  'session_type': 'OPHYS_6_images_B',
                                  'driver_line': ['Camk2a-tTA', 'Slc17a7-IRES2-Cre'],
                                  'behavior_session_uuid': uuid.UUID('69cdbe09-e62b-4b42-aab1-54b5773dfe78'),
-                                 'experiment_datetime': pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 58, 50, 325000)),
+                                 'experiment_datetime': pytz.utc.localize(datetime.datetime(2018, 11, 30, 23, 28, 37)),
                                  'ophys_frame_rate': 31.0,
                                  'imaging_depth': 375,
                                  'LabTracks_ID': 416369,
diff --git a/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py b/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py
index 6fd623e7d..0d5566e9b 100644
--- a/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py
+++ b/allensdk/test/brain_observatory/behavior/test_ophys_lims_api.py
@@ -220,15 +220,6 @@ def test_get_ophys_segmentation_run_id(ophys_experiment_id):
     _ = ophys_lims_api.get_ophys_cell_segmentation_run_id()
 
 
-@pytest.mark.requires_bamboo
-@pytest.mark.parametrize('ophys_experiment_id', [
-    pytest.param(511458874),
-])
-def test_get_cell_roi_table(ophys_experiment_id):
-    ophys_lims_api = OphysLimsApi(ophys_experiment_id)
-    assert len(ophys_lims_api.get_cell_specimen_table()) == 128
-
-
 @pytest.mark.requires_bamboo
 @pytest.mark.parametrize('ophys_lims_experiment_id, compare_val', [
     pytest.param(511458874, 0.785203),

From ca0c472cefbce8c7c904f553a2b7c5982725a065 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Mon, 23 Nov 2020 18:40:17 -0800
Subject: [PATCH 44/56] Fix failing mtrain_api test involving BehaviorLimsApi

---
 allensdk/internal/api/mtrain_api.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/allensdk/internal/api/mtrain_api.py b/allensdk/internal/api/mtrain_api.py
index 4f805d633..290576e04 100644
--- a/allensdk/internal/api/mtrain_api.py
+++ b/allensdk/internal/api/mtrain_api.py
@@ -52,13 +52,14 @@ def get_session(self, behavior_session_uuid=None, behavior_session_id=None):
         assert not all(v is None for v in [
                        behavior_session_uuid, behavior_session_id]), 'must enter either a behavior_session_uuid or a behavior_session_id'
 
-        behavior_data = BehaviorLimsApi(behavior_session_id)
-        if behavior_session_uuid is None and behavior_session_id is not None:
-            # get a behavior session uuid if a lims ID was entered
-            behavior_session_uuid = behavior_data.get_behavior_session_uuid()
         if behavior_session_uuid is not None and behavior_session_id is not None:
             # if both a behavior session uuid and a lims id are entered, ensure that they match
-            assert behavior_session_uuid == behavior_data.get_behavior_session_uuid(), 'behavior_session {} does not match behavior_session_id {}'.format(behavior_session_uuid, behavior_session_id)
+            behavior_api = BehaviorLimsApi(behavior_session_id)
+            assert behavior_session_uuid == behavior_api.get_behavior_session_uuid(), 'behavior_session {} does not match behavior_session_id {}'.format(behavior_session_uuid, behavior_session_id)
+        if behavior_session_uuid is None and behavior_session_id is not None:
+            # get a behavior session uuid if a lims ID was entered
+            behavior_api = BehaviorLimsApi(behavior_session_id)
+            behavior_session_uuid = behavior_api.get_behavior_session_uuid()
 
         filters = [{"name": "id", "op": "eq", "val": behavior_session_uuid}]
         behavior_df = self.get_df('behavior_sessions', filters=filters).rename(columns={'id': 'behavior_session_uuid'})

From 3d1c3e3e12162637fb6e5f12475114cbd52d09ac Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 8 Dec 2020 11:44:42 -0800
Subject: [PATCH 45/56] Improve docstring for session_api_utils

---
 allensdk/brain_observatory/session_api_utils.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/allensdk/brain_observatory/session_api_utils.py b/allensdk/brain_observatory/session_api_utils.py
index e57732a67..5322508be 100644
--- a/allensdk/brain_observatory/session_api_utils.py
+++ b/allensdk/brain_observatory/session_api_utils.py
@@ -199,7 +199,20 @@ def sessions_are_equal(A, B, reraise=False) -> bool:
     return True
 
 
-def compare_session_fields(x1, x2, err_msg=""):
+def compare_session_fields(x1: Any, x2: Any, err_msg=""):
+    """Helper function to compare if two fields (attributes) from a
+    Session object are equal to one another.
+
+    Parameters
+    ----------
+    x1 : Any
+        The field from the first session to compare
+    x2 : Any
+        The corresponding field from the second session to compare
+    err_msg : str, optional
+        The error message to display if two compared fields do not equal
+        one another, by default "" (an empty string)
+    """
     if isinstance(x1, pd.DataFrame):
         try:
             assert_frame_equal(x1, x2, check_like=True)

From 352ce95873da63690ac13ce2c225e3af62548917 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 8 Dec 2020 15:06:14 -0800
Subject: [PATCH 46/56] Update db_connection_creator documentation and params

This commit aims to improve the clarity of documentation
for the db_connection_creator helper function. This includes
changing the 'default_credentials' param to 'fallback_credentials'.
---
 .../data_io/behavior_project_lims_api.py      | 31 +++++----------
 .../session_apis/data_io/behavior_lims_api.py |  6 +--
 .../data_io/behavior_ophys_lims_api.py        |  4 +-
 .../session_apis/data_io/ophys_lims_api.py    |  7 ++--
 allensdk/internal/api/__init__.py             | 39 ++++++++++++++-----
 5 files changed, 47 insertions(+), 40 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
index 21a5ce87a..40ada8d9c 100644
--- a/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
+++ b/allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py
@@ -10,11 +10,11 @@
     BehaviorOphysSession)
 from allensdk.brain_observatory.behavior.session_apis.data_io import (
     BehaviorLimsApi, BehaviorOphysLimsApi)
-from allensdk.internal.api import PostgresQueryMixin
+from allensdk.internal.api import db_connection_creator
 from allensdk.brain_observatory.ecephys.ecephys_project_api.http_engine import (
     HttpEngine)
 from allensdk.core.typing import SupportsStr
-from allensdk.core.authentication import DbCredentials, credential_injector
+from allensdk.core.authentication import DbCredentials
 from allensdk.core.auth_config import (
     MTRAIN_DB_CREDENTIAL_MAP, LIMS_DB_CREDENTIAL_MAP)
 
@@ -96,26 +96,13 @@ def default(
         _app_kwargs = {"scheme": "http", "host": "lims2"}
         if app_kwargs:
             _app_kwargs.update(app_kwargs)
-        if lims_credentials:
-            lims_engine = PostgresQueryMixin(
-                dbname=lims_credentials.dbname, user=lims_credentials.user,
-                host=lims_credentials.host, password=lims_credentials.password,
-                port=lims_credentials.port)
-        else:
-            # Currying is equivalent to decorator syntactic sugar
-            lims_engine = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
-                           (PostgresQueryMixin)())
-
-        if mtrain_credentials:
-            mtrain_engine = PostgresQueryMixin(
-                dbname=mtrain_credentials.dbname, user=mtrain_credentials.user,
-                host=mtrain_credentials.host, password=mtrain_credentials.password,
-                port=mtrain_credentials.port)
-        else:
-            # Currying is equivalent to decorator syntactic sugar
-            mtrain_engine = (
-                credential_injector(MTRAIN_DB_CREDENTIAL_MAP)
-                (PostgresQueryMixin)())
+
+        lims_engine = db_connection_creator(
+            credentials=lims_credentials,
+            fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
+        mtrain_engine = db_connection_creator(
+            credentials=mtrain_credentials,
+            fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP)
 
         app_engine = HttpEngine(**_app_kwargs)
         return cls(lims_engine, mtrain_engine, app_engine)
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index 912ebadd0..b02c7a913 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -29,11 +29,11 @@ def __init__(self, behavior_session_id: int,
 
         self.mtrain_db = db_connection_creator(
             credentials=mtrain_credentials,
-            default_credentials=MTRAIN_DB_CREDENTIAL_MAP)
+            fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP)
 
         self.lims_db = db_connection_creator(
             credentials=lims_credentials,
-            default_credentials=LIMS_DB_CREDENTIAL_MAP)
+            fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
 
         self.behavior_session_id = behavior_session_id
         ids = self._get_ids()
@@ -59,7 +59,7 @@ def from_foraging_id(cls,
 
         lims_db = db_connection_creator(
             credentials=lims_credentials,
-            default_credentials=LIMS_DB_CREDENTIAL_MAP)
+            fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
 
         if isinstance(foraging_id, uuid.UUID):
             foraging_id = str(foraging_id)
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index 76f1c8427..e021b050b 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -27,11 +27,11 @@ def __init__(self, ophys_experiment_id: int,
 
         self.lims_db = db_connection_creator(
             credentials=lims_credentials,
-            default_credentials=LIMS_DB_CREDENTIAL_MAP)
+            fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
 
         self.mtrain_db = db_connection_creator(
             credentials=mtrain_credentials,
-            default_credentials=MTRAIN_DB_CREDENTIAL_MAP)
+            fallback_credentials=MTRAIN_DB_CREDENTIAL_MAP)
 
         self.ophys_experiment_id = ophys_experiment_id
         self.behavior_session_id = self.get_behavior_session_id()
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
index 030ef2783..a9b102fc5 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
@@ -1,13 +1,14 @@
 import pytz
+from datetime import datetime
 import pandas as pd
 from typing import Optional
 
 from allensdk.internal.api import (
-    PostgresQueryMixin, OneOrMoreResultExpectedError, db_connection_creator)
+    OneOrMoreResultExpectedError, db_connection_creator)
 from allensdk.api.cache import memoize
 from allensdk.internal.core.lims_utilities import safe_system_path
 from allensdk.core.cache_method_utilities import CachedInstanceMethodMixin
-from allensdk.core.authentication import credential_injector, DbCredentials
+from allensdk.core.authentication import DbCredentials
 from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
 
 
@@ -19,7 +20,7 @@ def __init__(self, ophys_experiment_id: int,
 
         self.lims_db = db_connection_creator(
             credentials=lims_credentials,
-            default_credentials=LIMS_DB_CREDENTIAL_MAP)
+            fallback_credentials=LIMS_DB_CREDENTIAL_MAP)
 
     def get_ophys_experiment_id(self):
         return self.ophys_experiment_id
diff --git a/allensdk/internal/api/__init__.py b/allensdk/internal/api/__init__.py
index bb2fbd7bd..ae7af12ba 100644
--- a/allensdk/internal/api/__init__.py
+++ b/allensdk/internal/api/__init__.py
@@ -75,33 +75,52 @@ def select_one(self, query):
         return {}
 
 
-def db_connection_creator(default_credentials: dict,
-                          credentials: Optional[DbCredentials] = None,
+def db_connection_creator(credentials: Optional[DbCredentials] = None,
+                          fallback_credentials: Optional[dict] = None,
                           ) -> PostgresQueryMixin:
-    """Create a db connection using either default credentials or
-    custom credentials.
+    """Create a db connection using credentials. If credentials are not
+    provided then use fallback credentials (which attempt to read from
+    shell environment variables).
+
+    Note: Must provide one of either 'credentials' or 'fallback_credentials'.
+    If both are provided, 'credentials' will take precedence.
 
     Parameters
     ----------
-    default_credentials : dict
-        Default credentials to use for creating the DB connection.
-        Some options can be found in allensdk.core.auth_config.
     credentials : Optional[DbCredentials], optional
         User specified credentials, by default None
+    fallback_credentials : dict
+        Fallback credentials to use for creating the DB connection in the
+        case that no 'credentials' are provided, by default None.
+
+        Fallback credentials will attempt to get db connection info from
+        shell environment variables.
+
+        Some examples of environment variables that fallback credentials
+        will try to read from can be found in allensdk.core.auth_config.
 
     Returns
     -------
     PostgresQueryMixin
         A DB connection instance which can execute queries to the DB
-        specified by credentials or default_credentials.
+        specified by credentials or fallback_credentials.
+
+    Raises
+    ------
+    RuntimeError
+        If neither 'credentials' nor 'fallback_credentials' were provided.
     """
     if credentials:
         db_conn = PostgresQueryMixin(
             dbname=credentials.dbname, user=credentials.user,
             host=credentials.host, port=credentials.port,
             password=credentials.password)
-    else:
-        db_conn = (credential_injector(default_credentials)
+    elif fallback_credentials:
+        db_conn = (credential_injector(fallback_credentials)
                    (PostgresQueryMixin)())
+    else:
+        raise RuntimeError(
+            "Must provide either credentials or fallback credentials in "
+            "order to create a db connection!")
 
     return db_conn

From f9533bddd8fbbbc831b1cb179afc02f30be01455 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 8 Dec 2020 16:53:17 -0800
Subject: [PATCH 47/56] Standardize LIMS queries for behavior APIs

This commit standardizes the formatting of LIMS queries
for the various behavior APIs.

This commit also adds better documentation API methods
(including return type and better descriptions of
data retrieved by queries).
---
 .../session_apis/data_io/behavior_lims_api.py |   6 +-
 .../data_io/behavior_ophys_lims_api.py        | 151 +++---
 .../session_apis/data_io/ophys_lims_api.py    | 429 +++++++++++-------
 3 files changed, 341 insertions(+), 245 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index b02c7a913..793733a68 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -69,7 +69,7 @@ def from_foraging_id(cls,
         query = f"""
             SELECT id
             FROM behavior_sessions
-            WHERE foraging_id = '{foraging_id}'
+            WHERE foraging_id = '{foraging_id};'
         """
         session_id = lims_db.fetchone(query, strict=True)
         return cls(session_id, lims_credentials=lims_credentials)
@@ -163,7 +163,7 @@ def get_birth_date(self) -> datetime:
         SELECT d.date_of_birth
         FROM behavior_sessions bs
         JOIN donors d on d.id = bs.donor_id
-        WHERE bs.id = {self.behavior_session_id}
+        WHERE bs.id = {self.behavior_session_id};
         """
         return self.lims_db.fetchone(query, strict=True).date()
 
@@ -217,7 +217,7 @@ def get_stimulus_name(self) -> str:
             SELECT stages.name
             FROM behavior_sessions bs
             JOIN stages ON stages.id = bs.state_id
-            WHERE bs.id = '{self.foraging_id}'
+            WHERE bs.id = '{self.foraging_id};'
         """
         return self.mtrain_db.fetchone(query, strict=True)
 
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index e021b050b..d62d69f06 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -1,5 +1,5 @@
 import logging
-from typing import Optional
+from typing import List, Optional
 import pandas as pd
 
 from allensdk.api.cache import memoize
@@ -36,129 +36,132 @@ def __init__(self, ophys_experiment_id: int,
         self.ophys_experiment_id = ophys_experiment_id
         self.behavior_session_id = self.get_behavior_session_id()
 
-    def get_ophys_experiment_id(self):
+    def get_ophys_experiment_id(self) -> int:
         return self.ophys_experiment_id
 
     @memoize
-    def get_ophys_session_id(self):
-        query = '''
+    def get_ophys_session_id(self) -> int:
+        query = """
                 SELECT os.id FROM ophys_sessions os
                 JOIN ophys_experiment oe ON oe.ophys_session_id = os.id
                 WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_experiment_container_id(self):
-        query = '''
+    def get_experiment_container_id(self) -> int:
+        query = """
                 SELECT visual_behavior_experiment_container_id
                 FROM ophys_experiments_visual_behavior_experiment_containers
-                WHERE ophys_experiment_id= {};
-                '''.format(self.get_ophys_experiment_id())
+                WHERE ophys_experiment_id = {};
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=False)
 
     @memoize
-    def get_behavior_stimulus_file(self):
-        query = '''
-                SELECT stim.storage_directory || stim.filename AS stim_file
+    def get_behavior_stimulus_file(self) -> str:
+        query = """
+                SELECT wkf.storage_directory || wkf.filename AS stim_file
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                JOIN behavior_sessions bs ON bs.ophys_session_id=os.id
-                LEFT JOIN well_known_files stim ON stim.attachable_id=bs.id AND stim.attachable_type = 'BehaviorSession' AND stim.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'StimulusPickle')
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN behavior_sessions bs ON bs.ophys_session_id = os.id
+                LEFT JOIN well_known_files wkf ON wkf.attachable_id = bs.id
+                JOIN well_known_file_types wkft
+                ON wkf.well_known_file_type_id = wkft.id
+                WHERE wkf.attachable_type = 'BehaviorSession'
+                AND wkft.name = 'StimulusPickle'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_nwb_filepath(self):
-
-        query = '''
+    def get_nwb_filepath(self) -> str:
+        query = """
                 SELECT wkf.storage_directory || wkf.filename AS nwb_file
                 FROM ophys_experiments oe
-                LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.id AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'BehaviorOphysNwb')
-                WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN well_known_files wkf ON wkf.attachable_id = oe.id
+                JOIN well_known_file_types wkft
+                ON wkf.well_known_file_type_id = wkft.id
+                WHERE wkft.name ='BehaviorOphysNwb'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_eye_tracking_filepath(self):
-        query = '''SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file
-                   FROM ophys_experiments oe
-                   LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.ophys_session_id
-                   AND wkf.attachable_type = 'OphysSession'
-                   AND wkf.well_known_file_type_id=(SELECT id FROM well_known_file_types WHERE name = 'EyeTracking Ellipses')
-                   WHERE oe.id={};
-                   '''.format(self.get_ophys_experiment_id())
+    def get_eye_tracking_filepath(self) -> str:
+        query = """
+                SELECT wkf.storage_directory || wkf.filename
+                AS eye_tracking_file
+                FROM ophys_experiments oe
+                LEFT JOIN well_known_files wkf
+                ON wkf.attachable_id = oe.ophys_session_id
+                JOIN well_known_file_types wkft
+                ON wkf.well_known_file_type_id = wkft.id
+                WHERE wkf.attachable_type = 'OphysSession'
+                AND wkft.name = 'EyeTracking Ellipses'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @staticmethod
-    def get_ophys_experiment_df():
+    def get_ophys_experiment_df() -> pd.DataFrame:
 
         api = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
                (PostgresQueryMixin)())
-        query = '''
-                SELECT
-
-                oec.visual_behavior_experiment_container_id as container_id,
-                oec.ophys_experiment_id,
-                oe.workflow_state,
-                d.full_genotype as full_genotype,
-                id.depth as imaging_depth,
-                st.acronym as targeted_structure,
-                os.name as session_name,
-                equipment.name as equipment_name
-
-                FROM ophys_experiments_visual_behavior_experiment_containers oec
-                LEFT JOIN ophys_experiments oe ON oe.id = oec.ophys_experiment_id
-                LEFT JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                LEFT JOIN specimens sp ON sp.id=os.specimen_id
-                LEFT JOIN donors d ON d.id=sp.donor_id
-                LEFT JOIN imaging_depths id ON id.id=oe.imaging_depth_id
-                LEFT JOIN structures st ON st.id=oe.targeted_structure_id
-                LEFT JOIN equipment ON equipment.id=os.equipment_id
-                '''
+        query = """
+            SELECT
+
+            oec.visual_behavior_experiment_container_id as container_id,
+            oec.ophys_experiment_id,
+            oe.workflow_state,
+            d.full_genotype as full_genotype,
+            id.depth as imaging_depth,
+            st.acronym as targeted_structure,
+            os.name as session_name,
+            equipment.name as equipment_name
+
+            FROM ophys_experiments_visual_behavior_experiment_containers oec
+            LEFT JOIN ophys_experiments oe ON oe.id = oec.ophys_experiment_id
+            LEFT JOIN ophys_sessions os ON oe.ophys_session_id = os.id
+            LEFT JOIN specimens sp ON sp.id = os.specimen_id
+            LEFT JOIN donors d ON d.id = sp.donor_id
+            LEFT JOIN imaging_depths id ON id.id = oe.imaging_depth_id
+            LEFT JOIN structures st ON st.id = oe.targeted_structure_id
+            LEFT JOIN equipment ON equipment.id = os.equipment_id;
+            """
 
         return pd.read_sql(query, api.get_connection())
 
     @staticmethod
-    def get_containers_df(only_passed=True):
+    def get_containers_df(only_passed=True) -> pd.DataFrame:
 
         api = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
                (PostgresQueryMixin)())
         if only_passed is True:
-            query = '''
+            query = """
                     SELECT *
                     FROM visual_behavior_experiment_containers vbc
                     WHERE workflow_state IN ('container_qc','publish');
-                    '''
+                    """
         else:
-            query = '''
+            query = """
                     SELECT *
-                    FROM visual_behavior_experiment_containers vbc
-                    '''
+                    FROM visual_behavior_experiment_containers vbc;
+                    """
 
-        return pd.read_sql(query, api.get_connection()).rename(columns={'id': 'container_id'})[['container_id', 'specimen_id', 'workflow_state']]
+        return pd.read_sql(query, api.get_connection()).rename(
+            columns={'id': 'container_id'})[['container_id',
+                                             'specimen_id',
+                                             'workflow_state']]
 
     @classmethod
-    def get_api_list_by_container_id(cls, container_id):
-
+    def get_api_list_by_container_id(cls, container_id
+                                     ) -> List["BehaviorOphysLimsApi"]:
         df = cls.get_ophys_experiment_df()
-        oeid_list = df[df['container_id'] == container_id]['ophys_experiment_id'].values
+        container_selector = df['container_id'] == container_id
+        oeid_list = df[container_selector]['ophys_experiment_id'].values
         return [cls(oeid) for oeid in oeid_list]
 
 
 if __name__ == "__main__":
 
     print(BehaviorOphysLimsApi.get_ophys_experiment_df())
-    # print(BehaviorOphysLimsApi.get_containers_df(only_passed=False))
-
-    # print(BehaviorOphysLimsApi.get_api_by_container(838105949))
-
-    # ophys_experiment_id = df['ophys_experiment_id'].iloc[0]
-    # print(ophys_experiment_id)
-    # BehaviorOphysLimsApi
-    # print(L)
-    # for c in sorted(L.columns):
-    #     print(c)
-    # for x in [791352433, 814796698, 814796612, 814796558, 814797528]:
-    #     print(x in L)
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
index a9b102fc5..38799e6b4 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
@@ -93,301 +93,393 @@ def get_behavior_session_id(self) -> Optional[int]:
             return response[0]
 
     @memoize
-    def get_ophys_experiment_dir(self):
-        query = '''
+    def get_ophys_experiment_dir(self) -> str:
+        """Get the storage directory associated with the ophys experiment"""
+        query = """
                 SELECT oe.storage_directory
                 FROM ophys_experiments oe
                 WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_nwb_filepath(self):
-        query = '''
+    def get_nwb_filepath(self) -> str:
+        """Get the filepath of the nwb file associated with the ophys
+        experiment"""
+        query = """
                 SELECT wkf.storage_directory || wkf.filename AS nwb_file
                 FROM ophys_experiments oe
-                LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.id AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'NWBOphys')
-                WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN well_known_files wkf ON wkf.attachable_id = oe.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE wkft.name = 'NWBOphys'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_sync_file(self, ophys_experiment_id=None):
-        query = '''
-                SELECT sync.storage_directory || sync.filename AS sync_file
+    def get_sync_file(self, ophys_experiment_id=None) -> str:
+        """Get the filepath of the sync timing file associated with the
+        ophys experiment"""
+        query = """
+                SELECT wkf.storage_directory || wkf.filename AS sync_file
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                LEFT JOIN well_known_files sync ON sync.attachable_id=os.id AND sync.attachable_type = 'OphysSession' AND sync.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysRigSync')
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN well_known_files wkf ON wkf.attachable_id = os.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE wkf.attachable_type = 'OphysSession'
+                AND wkft.name = 'OphysRigSync'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_max_projection_file(self):
-        query = '''
+    def get_max_projection_file(self) -> str:
+        """Get the filepath of the max projection image associated with the
+        ophys experiment"""
+        query = """
                 SELECT wkf.storage_directory || wkf.filename AS maxint_file
                 FROM ophys_experiments oe
-                LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't'
-                LEFT JOIN well_known_files wkf ON wkf.attachable_id=ocsr.id AND wkf.attachable_type = 'OphysCellSegmentationRun' AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysMaxIntImage')
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN ophys_cell_segmentation_runs ocsr
+                ON ocsr.ophys_experiment_id = oe.id
+                JOIN well_known_files wkf ON wkf.attachable_id = ocsr.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE ocsr.current = 't'
+                AND wkf.attachable_type = 'OphysCellSegmentationRun'
+                AND wkft.name = 'OphysMaxIntImage'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_targeted_structure(self):
-        query = '''
+    def get_targeted_structure(self) -> str:
+        """Get the targeted structure (acronym) for an ophys experiment
+        (ex: "Visp")"""
+        query = """
                 SELECT st.acronym
                 FROM ophys_experiments oe
-                LEFT JOIN structures st ON st.id=oe.targeted_structure_id
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                LEFT JOIN structures st ON st.id = oe.targeted_structure_id
+                WHERE oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_imaging_depth(self):
-        query = '''
+    def get_imaging_depth(self) -> int:
+        """Get the imaging depth for an ophys experiment
+        (ex: 400, 500, etc.)"""
+        query = """
                 SELECT id.depth
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                LEFT JOIN imaging_depths id ON id.id=oe.imaging_depth_id
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                LEFT JOIN imaging_depths id ON id.id = oe.imaging_depth_id
+                WHERE oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_stimulus_name(self):
-        query = '''
+    def get_stimulus_name(self) -> str:
+        """Get the name of the stimulus presented for an ophys experiment"""
+        query = """
                 SELECT os.stimulus_name
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                WHERE oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         stimulus_name = self.lims_db.fetchone(query, strict=False)
         stimulus_name = 'Unknown' if stimulus_name is None else stimulus_name
         return stimulus_name
 
-
     @memoize
-    def get_experiment_date(self):
-        query = '''
+    def get_experiment_date(self) -> datetime:
+        """Get the acquisition date of an ophys experiment"""
+        query = """
                 SELECT os.date_of_acquisition
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                WHERE oe.id = {};
+                """.format(self.get_ophys_experiment_id())
 
         experiment_date = self.lims_db.fetchone(query, strict=True)
         return pytz.utc.localize(experiment_date)
 
     @memoize
-    def get_reporter_line(self):
-        query = '''
+    def get_reporter_line(self) -> str:
+        """Get the (gene) reporter line for the subject associated with an
+        ophys experiment
+        """
+        query = """
                 SELECT g.name as reporter_line
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                JOIN specimens sp ON sp.id=os.specimen_id
-                JOIN donors d ON d.id=sp.donor_id
-                JOIN donors_genotypes dg ON dg.donor_id=d.id
-                JOIN genotypes g ON g.id=dg.genotype_id
-                JOIN genotype_types gt ON gt.id=g.genotype_type_id AND gt.name = 'reporter'
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN specimens sp ON sp.id = os.specimen_id
+                JOIN donors d ON d.id = sp.donor_id
+                JOIN donors_genotypes dg ON dg.donor_id = d.id
+                JOIN genotypes g ON g.id = dg.genotype_id
+                JOIN genotype_types gt ON gt.id = g.genotype_type_id
+                WHERE gt.name = 'reporter'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         result = self.lims_db.fetchall(query)
         if result is None or len(result) < 1:
-            raise OneOrMoreResultExpectedError('Expected one or more, but received: {} from query'.format(result))
+            raise OneOrMoreResultExpectedError(
+                f"Expected one or more, but received: '{result}' from query")
         return result
 
     @memoize
-    def get_driver_line(self):
-        query = '''
+    def get_driver_line(self) -> str:
+        """Get the (gene) driver line for the subject associated with an ophys
+        experiment"""
+        query = """
                 SELECT g.name as driver_line
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                JOIN specimens sp ON sp.id=os.specimen_id
-                JOIN donors d ON d.id=sp.donor_id
-                JOIN donors_genotypes dg ON dg.donor_id=d.id
-                JOIN genotypes g ON g.id=dg.genotype_id
-                JOIN genotype_types gt ON gt.id=g.genotype_type_id AND gt.name = 'driver'
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN specimens sp ON sp.id = os.specimen_id
+                JOIN donors d ON d.id = sp.donor_id
+                JOIN donors_genotypes dg ON dg.donor_id = d.id
+                JOIN genotypes g ON g.id = dg.genotype_id
+                JOIN genotype_types gt ON gt.id = g.genotype_type_id
+                WHERE gt.name = 'driver'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         result = self.lims_db.fetchall(query)
         if result is None or len(result) < 1:
-            raise OneOrMoreResultExpectedError('Expected one or more, but received: {} from query'.format(result))
+            raise OneOrMoreResultExpectedError(
+                f"Expected one or more, but received: '{result}' from query")
         return result
 
     @memoize
-    def get_external_specimen_name(self, ophys_experiment_id=None):
-        query = '''
+    def get_external_specimen_name(self) -> int:
+        """Get the external specimen id for the subject associated with an
+        ophys experiment"""
+        query = """
                 SELECT sp.external_specimen_name
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                JOIN specimens sp ON sp.id=os.specimen_id
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN specimens sp ON sp.id = os.specimen_id
+                WHERE oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return int(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_full_genotype(self):
-        query = '''
+    def get_full_genotype(self) -> str:
+        """Get the full genotype of the subject associated with an ophys
+        experiment"""
+        query = """
                 SELECT d.full_genotype
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                JOIN specimens sp ON sp.id=os.specimen_id
-                JOIN donors d ON d.id=sp.donor_id
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
-        return self.lims_db.fetchone(query, strict=True)
-
-    @memoize
-    def get_equipment_id(self):
-        query = '''
-                SELECT e.name
-                FROM ophys_experiments oe
-                JOIN ophys_sessions os ON oe.ophys_session_id = os.id
-                JOIN equipment e ON e.id=os.equipment_id
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN specimens sp ON sp.id = os.specimen_id
+                JOIN donors d ON d.id = sp.donor_id
+                WHERE oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_dff_file(self):
-        query = '''
-                SELECT dff.storage_directory || dff.filename AS dff_file
+    def get_dff_file(self) -> str:
+        """Get the filepath of the dff trace file associated with an ophys
+        experiment"""
+        query = """
+                SELECT wkf.storage_directory || wkf.filename AS dff_file
                 FROM ophys_experiments oe
-                LEFT JOIN well_known_files dff ON dff.attachable_id=oe.id AND dff.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysDffTraceFile')
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN well_known_files wkf ON wkf.attachable_id = oe.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE wkft.name = 'OphysDffTraceFile'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_objectlist_file(self):
-        query = '''
-                SELECT obj.storage_directory || obj.filename AS obj_file
+    def get_objectlist_file(self) -> str:
+        """Get the objectlist.txt filepath associated with an ophys experiment
+
+        NOTE: Although this will be deprecated for visual behavior it will
+        still be valid for visual coding.
+        """
+        query = """
+                SELECT wkf.storage_directory || wkf.filename AS obj_file
                 FROM ophys_experiments oe
-                LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't'
-                LEFT JOIN well_known_files obj ON obj.attachable_id=ocsr.id AND obj.attachable_type = 'OphysCellSegmentationRun' AND obj.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysSegmentationObjects')
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                LEFT JOIN ophys_cell_segmentation_runs ocsr
+                ON ocsr.ophys_experiment_id = oe.id
+                JOIN well_known_files wkf ON wkf.attachable_id = ocsr.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE wkft.name = 'OphysSegmentationObjects'
+                AND wkf.attachable_type = 'OphysCellSegmentationRun'
+                AND ocsr.current = 't'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_demix_file(self):
-        query = '''
+    def get_demix_file(self) -> str:
+        """Get the filepath of the demixed traces file associated with an
+        ophys experiment"""
+        query = """
                 SELECT wkf.storage_directory || wkf.filename AS demix_file
                 FROM ophys_experiments oe
-                LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.id AND wkf.attachable_type = 'OphysExperiment' AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'DemixedTracesFile')
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN well_known_files wkf ON wkf.attachable_id = oe.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE wkf.attachable_type = 'OphysExperiment'
+                AND wkft.name = 'DemixedTracesFile'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_average_intensity_projection_image_file(self):
-        query = '''
-                SELECT avg.storage_directory || avg.filename AS avgint_file
+    def get_average_intensity_projection_image_file(self) -> str:
+        """Get the avg intensity project image filepath associated with an
+        ophys experiment"""
+        query = """
+                SELECT wkf.storage_directory || wkf.filename AS avgint_file
                 FROM ophys_experiments oe
-                LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't'
-                LEFT JOIN well_known_files avg ON avg.attachable_id=ocsr.id AND avg.attachable_type = 'OphysCellSegmentationRun' AND avg.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysAverageIntensityProjectionImage')
-                WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN ophys_cell_segmentation_runs ocsr
+                ON ocsr.ophys_experiment_id = oe.id
+                JOIN well_known_files wkf ON wkf.attachable_id=ocsr.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE ocsr.current = 't'
+                AND wkf.attachable_type = 'OphysCellSegmentationRun'
+                AND wkft.name = 'OphysAverageIntensityProjectionImage'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_rigid_motion_transform_file(self):
-        query = '''
-                SELECT tra.storage_directory || tra.filename AS transform_file
+    def get_rigid_motion_transform_file(self) -> str:
+        """Get the filepath for the motion transform file (.csv) associated
+        with an ophys experiment"""
+        query = """
+                SELECT wkf.storage_directory || wkf.filename AS transform_file
                 FROM ophys_experiments oe
-                LEFT JOIN well_known_files tra ON tra.attachable_id=oe.id AND tra.attachable_type = 'OphysExperiment' AND tra.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysMotionXyOffsetData')
-                WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                JOIN well_known_files wkf ON wkf.attachable_id = oe.id
+                JOIN well_known_file_types wkft
+                ON wkft.id = wkf.well_known_file_type_id
+                WHERE wkf.attachable_type = 'OphysExperiment'
+                AND wkft.name = 'OphysMotionXyOffsetData'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_motion_corrected_image_stack_file(self):
-        query = f"""
-            select wkf.storage_directory || wkf.filename
-            from well_known_files wkf
-            join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
-            where wkft.name = 'MotionCorrectedImageStack'
-            and wkf.attachable_id = {self.get_ophys_experiment_id()}
-        """
+    def get_motion_corrected_image_stack_file(self) -> str:
+        """Get the filepath for the motion corrected image stack associated
+        with a an ophys experiment"""
+        query = """
+            SELECT wkf.storage_directory || wkf.filename
+            FROM well_known_files wkf
+            JOIN well_known_file_types wkft
+            ON wkft.id = wkf.well_known_file_type_id
+            WHERE wkft.name = 'MotionCorrectedImageStack'
+            AND wkf.attachable_id = {};
+            """.format(self.get_ophys_experiment_id())
+
         return safe_system_path(self.lims_db.fetchone(query, strict=True))
 
     @memoize
-    def get_foraging_id(self):
-        query = '''
+    def get_foraging_id(self) -> str:
+        """Get the foraging id associated with an ophys experiment. This
+        id is obtained in str format but can be interpreted as a UUID.
+        (ex: 6448125b-5d18-4bda-94b6-fb4eb6613979)"""
+        query = """
                 SELECT os.foraging_id
                 FROM ophys_experiments oe
                 LEFT JOIN ophys_sessions os ON oe.ophys_session_id = os.id
                 WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())        
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_rig_name(self):
-        query = '''
-                select e.name as device_name
-                from ophys_experiments oe
-                join ophys_sessions os on os.id = oe.ophys_session_id
-                join equipment e on e.id = os.equipment_id
-                where oe.id = {}
-                '''.format(self.get_ophys_experiment_id())
+    def get_rig_name(self) -> str:
+        """Get the name of the experiment rig (ex: CAM2P.3)"""
+        query = """
+                SELECT e.name AS device_name
+                FROM ophys_experiments oe
+                JOIN ophys_sessions os ON os.id = oe.ophys_session_id
+                JOIN equipment e ON e.id = os.equipment_id
+                WHERE oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_field_of_view_shape(self):
-        query = '''
-                select {}
-                from ophys_experiments oe
-                where oe.id = {}
-                '''
-        X = {c: self.lims_db.fetchone(query.format('oe.movie_{}'.format(c), self.get_ophys_experiment_id()), strict=True) for c in ['width', 'height']}
-        return X
+    def get_field_of_view_shape(self) -> dict:
+        """Get a field of view dictionary for a given ophys experiment.
+           ex: {"width": int, "height": int}
+        """
+        query = """
+                SELECT {}
+                FROM ophys_experiments oe
+                WHERE oe.id = {};
+                """
+
+        fov_shape = dict()
+        ophys_expt_id = self.get_ophys_experiment_id()
+        for dim in ['width', 'height']:
+            select_col = f'oe.movie_{dim}'
+            formatted_query = query.format(select_col, ophys_expt_id)
+            fov_shape[dim] = self.lims_db.fetchone(formatted_query,
+                                                   strict=True)
+        return fov_shape
 
     @memoize
-    def get_ophys_cell_segmentation_run_id(self):
-        query = '''
-                select oseg.id
-                from ophys_experiments oe
-                join ophys_cell_segmentation_runs oseg on oe.id = oseg.ophys_experiment_id
-                where oe.id = {} and oseg.current = 't'
-                '''.format(self.get_ophys_experiment_id())
+    def get_ophys_cell_segmentation_run_id(self) -> int:
+        """Get the ophys cell segmentation run id associated with an
+        ophys experiment id"""
+        query = """
+                SELECT oseg.id
+                FROM ophys_experiments oe
+                JOIN ophys_cell_segmentation_runs oseg
+                ON oe.id = oseg.ophys_experiment_id
+                WHERE oseg.current = 't'
+                AND oe.id = {};
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_raw_cell_specimen_table_dict(self):
-        ophys_cell_segmentation_run_id = self.get_ophys_cell_segmentation_run_id()
-        query = '''
-                select *
-                from cell_rois cr
-                where cr.ophys_cell_segmentation_run_id = {}
-                '''.format(ophys_cell_segmentation_run_id)
-        cell_specimen_table = pd.read_sql(query, self.lims_db.get_connection()).rename(columns={'id': 'cell_roi_id', 'mask_matrix': 'image_mask'})
-        cell_specimen_table.drop(['ophys_experiment_id', 'ophys_cell_segmentation_run_id'], inplace=True, axis=1)
+    def get_raw_cell_specimen_table_dict(self) -> dict:
+        """Get the cell_rois table from LIMS in dictionary form"""
+        ophys_cell_seg_run_id = self.get_ophys_cell_segmentation_run_id()
+        query = """
+                SELECT *
+                FROM cell_rois cr
+                WHERE cr.ophys_cell_segmentation_run_id = {}
+                """.format(ophys_cell_seg_run_id)
+        initial_cs_table = pd.read_sql(query, self.lims_db.get_connection())
+        cell_specimen_table = initial_cs_table.rename(
+            columns={'id': 'cell_roi_id', 'mask_matrix': 'image_mask'})
+        cell_specimen_table.drop(['ophys_experiment_id',
+                                  'ophys_cell_segmentation_run_id'],
+                                 inplace=True, axis=1)
         return cell_specimen_table.to_dict()
 
     @memoize
-    def get_surface_2p_pixel_size_um(self):
-        query = '''
+    def get_surface_2p_pixel_size_um(self) -> float:
+        """Get the pixel size for 2-photon movies in micrometers"""
+        query = """
                 SELECT sc.resolution
                 FROM ophys_experiments oe
                 JOIN scans sc ON sc.image_id=oe.ophys_primary_image_id
                 WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
-
     @memoize
-    def get_workflow_state(self):
-        query = '''
+    def get_workflow_state(self) -> str:
+        """Get the workflow state of an ophys experiment (ex: 'failed')"""
+        query = """
                 SELECT oe.workflow_state
                 FROM ophys_experiments oe
                 WHERE oe.id = {};
-                '''.format(self.get_ophys_experiment_id())
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_sex(self):
-        query = '''
+    def get_sex(self) -> str:
+        """Get the sex of the subject (ex: 'M', 'F', or 'unknown')"""
+        query = """
                 SELECT g.name as sex
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
@@ -395,12 +487,13 @@ def get_sex(self):
                 JOIN donors d ON d.id=sp.donor_id
                 JOIN genders g ON g.id=d.gender_id
                 WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
     @memoize
-    def get_age(self):
-        query = '''
+    def get_age(self) -> str:
+        """Get the age of the subject (ex: 'P15', 'Adult', etc...)"""
+        query = """
                 SELECT a.name as age
                 FROM ophys_experiments oe
                 JOIN ophys_sessions os ON oe.ophys_session_id = os.id
@@ -408,7 +501,7 @@ def get_age(self):
                 JOIN donors d ON d.id=sp.donor_id
                 JOIN ages a ON a.id=d.age_id
                 WHERE oe.id= {};
-                '''.format(self.get_ophys_experiment_id())
+                """.format(self.get_ophys_experiment_id())
         return self.lims_db.fetchone(query, strict=True)
 
 

From 45bd848de617ed5b38e336b4064595de4aa2656e Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 8 Dec 2020 17:03:55 -0800
Subject: [PATCH 48/56] Add better docstrings for behavior_ophys_json API
 methods

---
 .../data_io/behavior_ophys_json_api.py        | 104 +++++++++++++-----
 1 file changed, 76 insertions(+), 28 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
index d4371266e..afb90e770 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
@@ -1,6 +1,7 @@
 import logging
 from datetime import datetime
 import pytz
+from typing import Optional
 
 from allensdk.brain_observatory.behavior.session_apis.data_transforms import (
     BehaviorOphysDataXforms)
@@ -15,7 +16,7 @@ def __init__(self, data):
         self.data = data
         self.logger = logging.getLogger(self.__class__.__name__)
 
-    def get_ophys_experiment_id(self):
+    def get_ophys_experiment_id(self) -> int:
         return self.data['ophys_experiment_id']
 
     # TODO: This should be replaced with a dict lookup after the
@@ -28,82 +29,129 @@ def get_behavior_session_id(self):
     def get_ophys_session_id(self):
         NotImplementedError()
 
-    def get_surface_2p_pixel_size_um(self):
+    def get_surface_2p_pixel_size_um(self) -> float:
+        """Get the pixel size for 2-photon movies in micrometers"""
         return self.data['surface_2p_pixel_size_um']
 
-    def get_max_projection_file(self):
+    def get_max_projection_file(self) -> str:
+        """Get the filepath of the max projection image associated with the
+        ophys experiment"""
         return self.data['max_projection_file']
 
-    def get_sync_file(self):
+    def get_sync_file(self) -> str:
+        """Get the filepath of the sync timing file associated with the
+        ophys experiment"""
         return self.data['sync_file']
 
-    def get_rig_name(self):
+    def get_rig_name(self) -> str:
+        """Get the name of the experiment rig (ex: CAM2P.3)"""
         return self.data['rig_name']
 
-    def get_sex(self):
+    def get_sex(self) -> str:
+        """Get the sex of the subject (ex: 'M', 'F', or 'unknown')"""
         return self.data['sex']
 
-    def get_age(self):
+    def get_age(self) -> str:
+        """Get the age of the subject (ex: 'P15', 'Adult', etc...)"""
         return self.data['age']
 
-    def get_field_of_view_shape(self):
+    def get_field_of_view_shape(self) -> dict:
+        """Get a field of view dictionary for a given ophys experiment.
+           ex: {"width": int, "height": int}
+        """
         return {'height': self.data['movie_height'],
                 'width': self.data['movie_width']}
 
-    def get_experiment_container_id(self):
+    def get_experiment_container_id(self) -> int:
+        """Get the experiment container id associated with an ophys
+        experiment"""
         return self.data['container_id']
 
-    def get_targeted_structure(self):
+    def get_targeted_structure(self) -> str:
+        """Get the targeted structure (acronym) for an ophys experiment
+        (ex: "Visp")"""
         return self.data['targeted_structure']
 
-    def get_imaging_depth(self):
+    def get_imaging_depth(self) -> int:
+        """Get the imaging depth for an ophys experiment
+        (ex: 400, 500, etc.)"""
         return self.data['targeted_depth']
 
-    def get_stimulus_name(self):
+    def get_stimulus_name(self) -> str:
+        """Get the name of the stimulus presented for an ophys experiment"""
         return self.data['stimulus_name']
 
-    def get_experiment_date(self):
+    def get_experiment_date(self) -> datetime:
+        """Get the acquisition date of an ophys experiment"""
         return pytz.utc.localize(
             datetime.strptime(self.data['date_of_acquisition'],
                               "%Y-%m-%d %H:%M:%S"))
 
-    def get_reporter_line(self):
+    def get_reporter_line(self) -> str:
+        """Get the (gene) reporter line for the subject associated with an
+        ophys experiment
+        """
         return self.data['reporter_line']
 
-    def get_driver_line(self):
+    def get_driver_line(self) -> str:
+        """Get the (gene) driver line for the subject associated with an ophys
+        experiment"""
         return self.data['driver_line']
 
-    def external_specimen_name(self):
+    def external_specimen_name(self) -> int:
+        """Get the external specimen id for the subject associated with an
+        ophys experiment"""
         return self.data['external_specimen_name']
 
-    def get_full_genotype(self):
+    def get_full_genotype(self) -> str:
+        """Get the full genotype of the subject associated with an ophys
+        experiment"""
         return self.data['full_genotype']
 
-    def get_behavior_stimulus_file(self):
+    def get_behavior_stimulus_file(self) -> str:
         return self.data['behavior_stimulus_file']
 
-    def get_dff_file(self):
+    def get_dff_file(self) -> str:
+        """Get the filepath of the dff trace file associated with an ophys
+        experiment."""
         return self.data['dff_file']
 
-    def get_ophys_cell_segmentation_run_id(self):
+    def get_ophys_cell_segmentation_run_id(self) -> int:
+        """Get the ophys cell segmentation run id associated with an
+        ophys experiment id"""
         return self.data['ophys_cell_segmentation_run_id']
 
-    def get_raw_cell_specimen_table_dict(self):
+    def get_raw_cell_specimen_table_dict(self) -> dict:
+        """Get the cell_rois table from LIMS in dictionary form"""
         return self.data['cell_specimen_table_dict']
 
-    def get_demix_file(self):
+    def get_demix_file(self) -> str:
+        """Get the filepath of the demixed traces file associated with an
+        ophys experiment"""
         return self.data['demix_file']
 
-    def get_average_intensity_projection_image_file(self):
+    def get_average_intensity_projection_image_file(self) -> str:
+        """Get the avg intensity project image filepath associated with an
+        ophys experiment"""
         return self.data['average_intensity_projection_image_file']
 
-    def get_rigid_motion_transform_file(self):
+    def get_rigid_motion_transform_file(self) -> str:
+        """Get the filepath for the motion transform file (.csv) associated
+        with an ophys experiment"""
         return self.data['rigid_motion_transform_file']
 
-    def get_external_specimen_name(self):
-        return self.data['external_specimen_name']
-
-    def get_imaging_plane_group(self):
+    def get_external_specimen_name(self) -> int:
+        """Get the external specimen id for the subject associated with an
+        ophys experiment"""
+        return int(self.data['external_specimen_name'])
+
+    def get_imaging_plane_group(self) -> Optional[int]:
+        """Get the imaging plane group number. This is a numeric index
+        that indicates the order that the frames were acquired when
+        there is more than one frame acquired concurrently. Relevant for
+        mesoscope data timestamps, as the laser jumps between plane
+        groups during the scan. Will be None for non-mesoscope data.
+        """
         try:
             # Will only contain the "imaging_plane_group" key if we are
             # dealing with Mesoscope data

From 6cd9bb860c52e93fed89a7173f76233e7472f760 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 8 Dec 2020 17:08:24 -0800
Subject: [PATCH 49/56] Resolve some linter issues

---
 .../data_io/behavior_ophys_nwb_api.py         | 56 ++++++++++++-------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
index 72e42208e..746d2143e 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
@@ -51,11 +51,15 @@ def save(self, session_object):
         )
 
         # Add stimulus_timestamps to NWB in-memory object:
-        nwb.add_stimulus_timestamps(nwbfile, session_object.stimulus_timestamps)
+        nwb.add_stimulus_timestamps(nwbfile,
+                                    session_object.stimulus_timestamps)
 
         # Add running data to NWB in-memory object:
-        unit_dict = {'v_sig': 'V', 'v_in': 'V', 'speed': 'cm/s', 'timestamps': 's', 'dx': 'cm'}
-        nwb.add_running_data_df_to_nwbfile(nwbfile, session_object.running_data_df, unit_dict)
+        unit_dict = {'v_sig': 'V', 'v_in': 'V',
+                     'speed': 'cm/s', 'timestamps': 's', 'dx': 'cm'}
+        nwb.add_running_data_df_to_nwbfile(nwbfile,
+                                           session_object.running_data_df,
+                                           unit_dict)
 
         # Add stimulus template data to NWB in-memory object:
         for name, image_data in session_object.stimulus_templates.items():
@@ -67,13 +71,16 @@ def save(self, session_object):
             nwb.add_stimulus_index(nwbfile, stimulus_index, nwb_template)
 
         # search for omitted rows and add stop_time before writing to NWB file
-        set_omitted_stop_time(stimulus_table=session_object.stimulus_presentations)
+        set_omitted_stop_time(
+            stimulus_table=session_object.stimulus_presentations)
 
         # Add stimulus presentations data to NWB in-memory object:
-        nwb.add_stimulus_presentations(nwbfile, session_object.stimulus_presentations)
+        nwb.add_stimulus_presentations(nwbfile,
+                                       session_object.stimulus_presentations)
 
         # Add trials data to NWB in-memory object:
-        nwb.add_trials(nwbfile, session_object.trials, TRIAL_COLUMN_DESCRIPTION_DICT)
+        nwb.add_trials(nwbfile, session_object.trials,
+                       TRIAL_COLUMN_DESCRIPTION_DICT)
 
         # Add licks data to NWB in-memory object:
         if len(session_object.licks) > 0:
@@ -90,7 +97,8 @@ def save(self, session_object):
         nwb.add_average_image(nwbfile, session_object.average_projection)
 
         # Add segmentation_mask_image image data to NWB in-memory object:
-        nwb.add_segmentation_mask_image(nwbfile, session_object.segmentation_mask_image)
+        nwb.add_segmentation_mask_image(nwbfile,
+                                        session_object.segmentation_mask_image)
 
         # Add metadata to NWB in-memory object:
         nwb.add_metadata(nwbfile, session_object.metadata)
@@ -104,10 +112,13 @@ def save(self, session_object):
                                     session_object.metadata)
 
         # Add dff to NWB in-memory object:
-        nwb.add_dff_traces(nwbfile, session_object.dff_traces, session_object.ophys_timestamps)
+        nwb.add_dff_traces(nwbfile, session_object.dff_traces,
+                           session_object.ophys_timestamps)
 
         # Add corrected_fluorescence to NWB in-memory object:
-        nwb.add_corrected_fluorescence_traces(nwbfile, session_object.corrected_fluorescence_traces)
+        nwb.add_corrected_fluorescence_traces(
+            nwbfile,
+            session_object.corrected_fluorescence_traces)
 
         # Add motion correction to NWB in-memory object:
         nwb.add_motion_correction(nwbfile, session_object.motion_correction)
@@ -121,11 +132,11 @@ def save(self, session_object):
     def get_ophys_experiment_id(self) -> int:
         return int(self.nwbfile.identifier)
 
-    # TODO: Implement save and lod of behavior_session_id to/from NWB file
+    # TODO: Implement save and load of behavior_session_id to/from NWB file
     def get_behavior_session_id(self) -> int:
         raise NotImplementedError()
 
-    # TODO: Implement save and load of ophys_session_id to/from NWB file 
+    # TODO: Implement save and load of ophys_session_id to/from NWB file
     def get_ophys_session_id(self) -> int:
         raise NotImplementedError()
 
@@ -133,12 +144,13 @@ def get_ophys_session_id(self) -> int:
     def get_eye_tracking(self) -> int:
         raise NotImplementedError()
 
-    def get_running_data_df(self, **kwargs):
+    def get_running_data_df(self, **kwargs) -> pd.DataFrame:
 
         running_speed = self.get_running_speed()
 
         running_data_df = pd.DataFrame({'speed': running_speed.values},
-                                       index=pd.Index(running_speed.timestamps, name='timestamps'))
+                                       index=pd.Index(running_speed.timestamps,
+                                                      name='timestamps'))
 
         for key in ['v_in', 'v_sig']:
             if key in self.nwbfile.acquisition:
@@ -154,11 +166,8 @@ def get_ophys_timestamps(self) -> np.ndarray:
         return self.nwbfile.processing['ophys'].get_data_interface('dff').roi_response_series['traces'].timestamps[:]
 
     def get_stimulus_templates(self, **kwargs):
-        return {key: val.data[:] for key, val in self.nwbfile.stimulus_template.items()}
-
-    # TODO: Implement save and load of 'raw' unaligned stimulus timestamps to/from NWB file
-    def get_raw_stimulus_timestamps(self) -> np.ndarray:
-        raise NotImplementedError()
+        return {key: val.data[:]
+                for key, val in self.nwbfile.stimulus_template.items()}
 
     def get_stimulus_timestamps(self) -> np.ndarray:
         return self.nwbfile.processing['stimulus'].get_data_interface('timestamps').timestamps[:]
@@ -181,9 +190,13 @@ def get_rewards(self) -> np.ndarray:
             time = self.nwbfile.processing['rewards'].get_data_interface('autorewarded').timestamps[:]
             autorewarded = self.nwbfile.processing['rewards'].get_data_interface('autorewarded').data[:]
             volume = self.nwbfile.processing['rewards'].get_data_interface('volume').data[:]
-            return pd.DataFrame({'volume': volume, 'timestamps': time, 'autorewarded': autorewarded}).set_index('timestamps')
+            return pd.DataFrame({
+                'volume': volume, 'timestamps': time,
+                'autorewarded': autorewarded}).set_index('timestamps')
         else:
-            return pd.DataFrame({'volume': [], 'timestamps': [], 'autorewarded': []}).set_index('timestamps')
+            return pd.DataFrame({
+                'volume': [], 'timestamps': [], 
+                'autorewarded': []}).set_index('timestamps')
 
     def get_max_projection(self, image_api=None) -> sitk.Image:
         return self.get_image('max_projection', 'ophys', image_api=image_api)
@@ -192,7 +205,8 @@ def get_average_projection(self, image_api=None) -> sitk.Image:
         return self.get_image('average_image', 'ophys', image_api=image_api)
 
     def get_segmentation_mask_image(self, image_api=None) -> sitk.Image:
-        return self.get_image('segmentation_mask_image', 'ophys', image_api=image_api)
+        return self.get_image('segmentation_mask_image',
+                              'ophys', image_api=image_api)
 
     def get_metadata(self) -> dict:
 

From 81408eea51288684e5ce6e89b523fc09d87d8261 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Tue, 8 Dec 2020 17:11:28 -0800
Subject: [PATCH 50/56] Fix typo in API update notebook

---
 .../examples/nb/api_modernization/updated_vs_legacy_api.ipynb   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb b/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb
index 33fe74959..ae4748cfd 100644
--- a/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb
+++ b/doc_template/examples_root/examples/nb/api_modernization/updated_vs_legacy_api.ipynb
@@ -17,7 +17,7 @@
    "source": [
     "import matplotlib.pyplot as plt\n",
     "\n",
-    "from from allensdk.brain_observatory.behavior.session_apis.data_io import BehaviorOphysLimsApi\n",
+    "from allensdk.brain_observatory.behavior.session_apis.data_io import BehaviorOphysLimsApi\n",
     "from allensdk.brain_observatory.behavior.behavior_ophys_session import BehaviorOphysSession\n",
     "\n",
     "\n",

From 0374cb93fe9457d356cbf2640546c9ce59532d8e Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Wed, 9 Dec 2020 08:01:17 -0800
Subject: [PATCH 51/56] Add better docstrings for behavior API and Xform
 classes

---
 .../session_apis/data_io/behavior_lims_api.py         |  8 ++++++--
 .../session_apis/data_io/behavior_ophys_json_api.py   | 11 +++++++++--
 .../session_apis/data_io/behavior_ophys_lims_api.py   |  8 ++++++++
 .../session_apis/data_io/behavior_ophys_nwb_api.py    |  4 ++++
 .../behavior/session_apis/data_io/ophys_lims_api.py   |  9 +++++++++
 .../data_transforms/behavior_data_xforms.py           |  3 +++
 .../data_transforms/behavior_ophys_data_xforms.py     |  3 +++
 7 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index 793733a68..10c00f265 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -18,8 +18,12 @@
 
 
 class BehaviorLimsApi(BehaviorDataXforms, CachedInstanceMethodMixin):
-    """A data fetching class that serves as an API for filling
-    'BehaviorSession' attributes from LIMS.
+    """A data fetching class that serves as an API for fetching 'raw'
+    data from LIMS necessary (but not sufficient) for filling a
+    'BehaviorSession'.
+
+    Most 'raw' data provided by this API needs to be processed by
+    BehaviorDataXforms methods in order to usable by 'BehaviorSession's
     """
     def __init__(self, behavior_session_id: int,
                  lims_credentials: Optional[DbCredentials] = None,
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
index afb90e770..4d096cfcd 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
@@ -8,8 +8,15 @@
 
 
 class BehaviorOphysJsonApi(BehaviorOphysDataXforms):
-    """
-        This class is used by both Scientifica and Mesoscope ophys experiments.
+    """A data fetching class that serves as an API for fetching 'raw'
+    data from a json file necessary (but not sufficient) for filling
+    a 'BehaviorOphysSession'.
+
+    Most 'raw' data provided by this API needs to be processed by
+    BehaviorOphysDataXforms methods in order to usable by
+    'BehaviorOphysSession's.
+
+    This class is used by the write_nwb module for behavior ophys sessions.
     """
 
     def __init__(self, data):
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index d62d69f06..6749181e7 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -18,6 +18,14 @@
 
 class BehaviorOphysLimsApi(BehaviorOphysDataXforms,  OphysLimsApi,
                            BehaviorLimsApi):
+    """A data fetching class that serves as an API for fetching 'raw'
+    data from LIMS necessary (but not sufficient) for filling
+    a 'BehaviorOphysSession'.
+
+    Most 'raw' data provided by this API needs to be processed by
+    BehaviorOphysDataXforms methods in order to usable by
+    'BehaviorOphysSession's.
+    """
 
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None,
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
index 746d2143e..ec98f8105 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_nwb_api.py
@@ -30,6 +30,10 @@
 
 
 class BehaviorOphysNwbApi(NwbApi, BehaviorOphysBase):
+    """A data fetching class that serves as an API for fetching 'raw'
+    data from an NWB file that is both necessary and sufficient for filling
+    a 'BehaviorOphysSession'.
+    """
 
     def __init__(self, *args, **kwargs):
         self.filter_invalid_rois = kwargs.pop("filter_invalid_rois", False)
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
index 38799e6b4..171019084 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
@@ -13,6 +13,15 @@
 
 
 class OphysLimsApi(CachedInstanceMethodMixin):
+    """A data fetching class that serves as an API for fetching 'raw'
+    data from LIMS for filling optical physiology data. This data is
+    is necessary (but not sufficient) to fill the 'Ophys' portion of a
+    BehaviorOphysSession.
+
+    This class needs to be inherited by the BehaviorOphysLimsApi and also
+    have methods from BehaviorOphysDataXforms in order to be usable by a
+    BehaviorOphysSession.
+    """
 
     def __init__(self, ophys_experiment_id: int,
                  lims_credentials: Optional[DbCredentials] = None):
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
index 674fd2a9e..a7699b9fa 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_data_xforms.py
@@ -24,6 +24,9 @@
 
 
 class BehaviorDataXforms(BehaviorBase):
+    """This class provides methods that transform (xform) 'raw' data provided
+    by LIMS data APIs to fill a BehaviorSession.
+    """
 
     @memoize
     def _behavior_stimulus_file(self) -> pd.DataFrame:
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
index 8d293ca13..3d4ebde09 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_transforms/behavior_ophys_data_xforms.py
@@ -35,6 +35,9 @@
 
 
 class BehaviorOphysDataXforms(BehaviorOphysBase):
+    """This class provides methods that transform (xform) 'raw' data provided
+    by LIMS data APIs to fill a BehaviorOphysSession.
+    """
 
     @memoize
     def get_cell_specimen_table(self):

From 389806ffeb0bbfc37d4dec5e912fb96e253ea074 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Wed, 9 Dec 2020 14:49:20 -0800
Subject: [PATCH 52/56] Add more documentation; Fix LIMS query typo

This is a combination of the following two commits:

1) Add more documentation

   This commit adds more documentation to brain_observatory.behavior
   data api methods.

2) Fix a couple of LIMS query typos

   A set of typos where a semicolon was introduced within a single quote
   block resulted in a non-functional query. This commit fixes these
   typos.
---
 .../session_apis/data_io/behavior_lims_api.py        |  4 ++--
 .../session_apis/data_io/behavior_ophys_json_api.py  |  5 +++--
 .../session_apis/data_io/behavior_ophys_lims_api.py  | 12 +++++++++++-
 .../behavior/session_apis/data_io/ophys_lims_api.py  | 12 ++++++------
 4 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index 10c00f265..5fb2fbc8f 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -73,7 +73,7 @@ def from_foraging_id(cls,
         query = f"""
             SELECT id
             FROM behavior_sessions
-            WHERE foraging_id = '{foraging_id};'
+            WHERE foraging_id = '{foraging_id}';
         """
         session_id = lims_db.fetchone(query, strict=True)
         return cls(session_id, lims_credentials=lims_credentials)
@@ -221,7 +221,7 @@ def get_stimulus_name(self) -> str:
             SELECT stages.name
             FROM behavior_sessions bs
             JOIN stages ON stages.id = bs.state_id
-            WHERE bs.id = '{self.foraging_id};'
+            WHERE bs.id = '{self.foraging_id}';
         """
         return self.mtrain_db.fetchone(query, strict=True)
 
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
index 4d096cfcd..999b2f3de 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_json_api.py
@@ -106,8 +106,8 @@ def get_driver_line(self) -> str:
         return self.data['driver_line']
 
     def external_specimen_name(self) -> int:
-        """Get the external specimen id for the subject associated with an
-        ophys experiment"""
+        """Get the external specimen id (LabTracks ID) for the subject
+        associated with an ophys experiment"""
         return self.data['external_specimen_name']
 
     def get_full_genotype(self) -> str:
@@ -116,6 +116,7 @@ def get_full_genotype(self) -> str:
         return self.data['full_genotype']
 
     def get_behavior_stimulus_file(self) -> str:
+        """Get the filepath to the StimulusPickle file for the session"""
         return self.data['behavior_stimulus_file']
 
     def get_dff_file(self) -> str:
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index 6749181e7..22351c9dd 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -49,6 +49,8 @@ def get_ophys_experiment_id(self) -> int:
 
     @memoize
     def get_ophys_session_id(self) -> int:
+        """Get the ophys session id associated with the ophys experiment
+        id used to initialize the API"""
         query = """
                 SELECT os.id FROM ophys_sessions os
                 JOIN ophys_experiment oe ON oe.ophys_session_id = os.id
@@ -58,6 +60,8 @@ def get_ophys_session_id(self) -> int:
 
     @memoize
     def get_experiment_container_id(self) -> int:
+        """Get the experiment container id associated with the ophys
+        experiment id used to initialize the API"""
         query = """
                 SELECT visual_behavior_experiment_container_id
                 FROM ophys_experiments_visual_behavior_experiment_containers
@@ -67,6 +71,8 @@ def get_experiment_container_id(self) -> int:
 
     @memoize
     def get_behavior_stimulus_file(self) -> str:
+        """Get the filepath to the StimulusPickle file for the session
+        associated with the ophys experiment id used to initialize the API"""
         query = """
                 SELECT wkf.storage_directory || wkf.filename AS stim_file
                 FROM ophys_experiments oe
@@ -83,6 +89,8 @@ def get_behavior_stimulus_file(self) -> str:
 
     @memoize
     def get_nwb_filepath(self) -> str:
+        """Get the filepath of the nwb file associated with the ophys
+        experiment"""
         query = """
                 SELECT wkf.storage_directory || wkf.filename AS nwb_file
                 FROM ophys_experiments oe
@@ -96,6 +104,8 @@ def get_nwb_filepath(self) -> str:
 
     @memoize
     def get_eye_tracking_filepath(self) -> str:
+        """Get the filepath of the eye tracking file (*.h5) associated with the
+        ophys experiment"""
         query = """
                 SELECT wkf.storage_directory || wkf.filename
                 AS eye_tracking_file
@@ -112,7 +122,7 @@ def get_eye_tracking_filepath(self) -> str:
 
     @staticmethod
     def get_ophys_experiment_df() -> pd.DataFrame:
-
+        """Get a DataFrame of metadata for ophys experiments"""
         api = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
                (PostgresQueryMixin)())
         query = """
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
index 171019084..2c434a115 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/ophys_lims_api.py
@@ -55,7 +55,7 @@ def get_plane_group_count(self) -> int:
                 ON pg.id = oe.ophys_imaging_plane_group_id
             WHERE
                 -- only 1 session for an experiment
-                os.id = (SELECT id from sess limit 1)
+                os.id = (SELECT id from sess limit 1);
         """
         return self.lims_db.fetchone(query, strict=True)
 
@@ -72,7 +72,7 @@ def get_imaging_plane_group(self) -> Optional[int]:
             FROM ophys_experiments oe
             JOIN ophys_imaging_plane_groups pg
             ON pg.id = oe.ophys_imaging_plane_group_id
-            WHERE oe.id = {self.get_ophys_experiment_id()}
+            WHERE oe.id = {self.get_ophys_experiment_id()};
         """
         # Non-mesoscope data will not have results
         group_order = self.lims_db.fetchall(query)
@@ -93,7 +93,7 @@ def get_behavior_session_id(self) -> Optional[int]:
             JOIN ophys_sessions os ON oe.ophys_session_id = os.id
             -- but not every ophys_session has a behavior_session
             LEFT JOIN behavior_sessions bs ON os.id = bs.ophys_session_id
-            WHERE oe.id = {self.get_ophys_experiment_id()}
+            WHERE oe.id = {self.get_ophys_experiment_id()};
         """
         response = self.lims_db.fetchall(query)     # Can be null
         if not len(response):
@@ -260,8 +260,8 @@ def get_driver_line(self) -> str:
 
     @memoize
     def get_external_specimen_name(self) -> int:
-        """Get the external specimen id for the subject associated with an
-        ophys experiment"""
+        """Get the external specimen id (LabTracks ID) for the subject
+        associated with an ophys experiment"""
         query = """
                 SELECT sp.external_specimen_name
                 FROM ophys_experiments oe
@@ -454,7 +454,7 @@ def get_raw_cell_specimen_table_dict(self) -> dict:
         query = """
                 SELECT *
                 FROM cell_rois cr
-                WHERE cr.ophys_cell_segmentation_run_id = {}
+                WHERE cr.ophys_cell_segmentation_run_id = {};
                 """.format(ophys_cell_seg_run_id)
         initial_cs_table = pd.read_sql(query, self.lims_db.get_connection())
         cell_specimen_table = initial_cs_table.rename(

From a6a9b2a3337bbe63b9a6e17fbe6c78c099fb1fb5 Mon Sep 17 00:00:00 2001
From: Nicholas Mei <nicholas.mei@alleninstitute.org>
Date: Thu, 10 Dec 2020 16:26:16 -0800
Subject: [PATCH 53/56] Add more detailed error message for an exception

---
 .../behavior/session_apis/data_io/behavior_lims_api.py        | 4 +++-
 .../behavior/session_apis/data_io/behavior_ophys_lims_api.py  | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
index 5fb2fbc8f..70f1f247a 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_lims_api.py
@@ -99,7 +99,9 @@ def _get_ids(self) -> Dict[str, Optional[Union[int, List[int]]]]:
         """
         ids_response = self.lims_db.select(query)
         if len(ids_response) > 1 or len(ids_response) < 1:
-            raise OneResultExpectedError
+            raise OneResultExpectedError(
+                f"Expected length one result, received: "
+                f"{ids_response} results from query")
         ids_dict = ids_response.iloc[0].to_dict()
 
         #  Get additional ids if also an ophys session
diff --git a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
index 22351c9dd..299e679bd 100644
--- a/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
+++ b/allensdk/brain_observatory/behavior/session_apis/data_io/behavior_ophys_lims_api.py
@@ -151,6 +151,7 @@ def get_ophys_experiment_df() -> pd.DataFrame:
 
     @staticmethod
     def get_containers_df(only_passed=True) -> pd.DataFrame:
+        """Get a DataFrame of experiment containers"""
 
         api = (credential_injector(LIMS_DB_CREDENTIAL_MAP)
                (PostgresQueryMixin)())
@@ -174,6 +175,8 @@ def get_containers_df(only_passed=True) -> pd.DataFrame:
     @classmethod
     def get_api_list_by_container_id(cls, container_id
                                      ) -> List["BehaviorOphysLimsApi"]:
+        """Return a list of BehaviorOphysLimsApi instances for all
+        ophys experiments"""
         df = cls.get_ophys_experiment_df()
         container_selector = df['container_id'] == container_id
         oeid_list = df[container_selector]['ophys_experiment_id'].values

From ec65f34ce299f96c17e52a6915e568fde98d0ff3 Mon Sep 17 00:00:00 2001
From: Doug Ollerenshaw <dougo@alleninstitute.org>
Date: Thu, 17 Dec 2020 16:25:35 -0800
Subject: [PATCH 54/56] change if statement to try/except

---
 allensdk/core/typing.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/allensdk/core/typing.py b/allensdk/core/typing.py
index 2429575bd..2a885e486 100644
--- a/allensdk/core/typing.py
+++ b/allensdk/core/typing.py
@@ -1,8 +1,12 @@
 import sys
-if sys.version_info.minor <= 7:  
+try:
+    # for Python 3.8 and greater
+    from typing import Protocol
+except ImportError:
+    # for Python 3.7 and before
     from typing import _Protocol as Protocol
 else:
-    from typing import Protocol
+    
 from abc import abstractmethod
 
 

From fbc73e5a40c525c80b5404732b3e9886c30c2feb Mon Sep 17 00:00:00 2001
From: Doug Ollerenshaw <dougo@alleninstitute.org>
Date: Thu, 17 Dec 2020 16:36:13 -0800
Subject: [PATCH 55/56] remove errant "else"

---
 allensdk/core/typing.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/allensdk/core/typing.py b/allensdk/core/typing.py
index 2a885e486..2a747557d 100644
--- a/allensdk/core/typing.py
+++ b/allensdk/core/typing.py
@@ -5,7 +5,6 @@
 except ImportError:
     # for Python 3.7 and before
     from typing import _Protocol as Protocol
-else:
     
 from abc import abstractmethod
 

From 478126e2137d976bce3342c120037caecc4baa68 Mon Sep 17 00:00:00 2001
From: Dan <danielk@alleninstitute.org>
Date: Mon, 21 Dec 2020 14:55:38 -0800
Subject: [PATCH 56/56] update changelog and index.rst

---
 CHANGELOG.md           |  7 +++++++
 doc_template/index.rst | 10 ++++++++++
 2 files changed, 17 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 871e64507..32d8bed27 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.4.0] = 2020-12-21
+- When running raster_plot on a spike_times dataframe, the spike times from each unit are plotted twice. (thank you @dgmurx)
+- improvements and fixes to behavior ophys NWB files.
+- improvements and fixes to BehaviorProjectCache tables including new column "donor_id"
+- implemented a timeout to obtaining an ecephys session. (thank you @wesley-jones)
+- big overhaul of how Behavior and BehaviorOphys classes are structured for the visual behavior project. See https://github.com/AllenInstitute/AllenSDK/pull/1789
+
 ## [2.3.3] = 2020-11-12
 ### Bug Fixes
 - (Internal) Fixed a bug in mesoscope processing where the ophys acquisition frames were being truncated
diff --git a/doc_template/index.rst b/doc_template/index.rst
index 7196fc07f..c3f6e27ab 100644
--- a/doc_template/index.rst
+++ b/doc_template/index.rst
@@ -91,6 +91,16 @@ The Allen SDK provides Python code for accessing experimental metadata along wit
 See the `mouse connectivity section <connectivity.html>`_ for more details.
 
 
+What's New - 2.3.2 (December 21, 2020)
+-----------------------------------------------------------------------
+As of the 2.4.0 release:
+- When running raster_plot on a spike_times dataframe, the spike times from each unit are plotted twice. (thank you @dgmurx)
+- improvements and fixes to behavior ophys NWB files.
+- improvements and fixes to BehaviorProjectCache tables including new column "donor_id"
+- implemented a timeout to obtaining an ecephys session. (thank you @wesley-jones)
+- big overhaul of how Behavior and BehaviorOphys classes are structured for the visual behavior project. See https://github.com/AllenInstitute/AllenSDK/pull/1789
+
+
 What's New - 2.3.2 (October 19, 2020)
 -----------------------------------------------------------------------
 As of the 2.3.2 release: