Merge pull request #2032 from AllenInstitute/rc/2.10.2

rc/2.10.2

.github/workflows/github-actions-ci.yml

@@ -33,7 +33,7 @@ jobs:
           export PATH="/home/runner/.local/bin:${PATH}"
           git fetch origin master
           # `|| true` to force exit code 0 even if no files found
-          CHANGED_PYFILES=$(git diff --name-only --diff-filter AM origin/master | grep .py || true)
+          CHANGED_PYFILES=$(git diff --name-only --diff-filter AM origin/master allensdk | grep -e ".*.py$" || true)
           echo "List of changed files:"
           echo ${CHANGED_PYFILES}
           flake8 ${CHANGED_PYFILES} --count

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.10.2] = TBD
+- update documentation to support visual behavior data release
+- Fixes a bug with the dictionary returned by BehaviorSession get get_performance_metrics() method
+- Adds docstrings to the BehaviorSession get_performance_metrics(), get_rolling_performance_df(), and get_reward_rate() methods
+
 ## [2.10.1] = 2021-03-23
 - changes name of BehaviorProjectCache to VisualBehaviorOphysProjectCache
 - changes VisualBehaviorOphysProjectCache method get_session_table() to get_ophys_session_table()

allensdk/__init__.py

@@ -35,7 +35,7 @@
 #
 import logging
 
-__version__ = '2.10.1'
+__version__ = '2.10.2'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py

@@ -11,7 +11,7 @@
 from allensdk.brain_observatory.behavior.behavior_project_cache.tables \
     .sessions_table import \
     SessionsTable
-from allensdk.brain_observatory.behavior.project_apis.data_io import (
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import (  # noqa: E501
     BehaviorProjectLimsApi, BehaviorProjectCloudApi)
 from allensdk.api.warehouse_cache.caching_utilities import \
     one_file_call_caching, call_caching

allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/abcs/__init__.py

@@ -0,0 +1 @@
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.abcs.behavior_project_base import BehaviorProjectBase  # noqa: F401, E501

allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/__init__.py

@@ -0,0 +1,2 @@
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io.behavior_project_lims_api import BehaviorProjectLimsApi  # noqa: F401, E501
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io.behavior_project_cloud_api import BehaviorProjectCloudApi  # noqa: F401, E501

allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_cloud_api.py → allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py

@@ -1,84 +1,40 @@
 import pandas as pd
-from typing import Iterable, Union, Dict, List, Optional
+from typing import Iterable, Union, List, Optional
 from pathlib import Path
 import logging
 import ast
 import semver
 
-from allensdk.brain_observatory.behavior.project_apis.abcs import (
-    BehaviorProjectBase)
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.abcs import BehaviorProjectBase  # noqa: E501
 from allensdk.brain_observatory.behavior.behavior_session import (
     BehaviorSession)
 from allensdk.brain_observatory.behavior.behavior_ophys_experiment import (
     BehaviorOphysExperiment)
 from allensdk.api.cloud_cache.cloud_cache import S3CloudCache, LocalCache
-from allensdk import __version__ as sdk_version
 
 
 # [min inclusive, max exclusive)
-COMPATIBILITY = {
-    "pipeline_versions": {
-        "2.9.0": {"AllenSDK": ["2.9.0", "3.0.0"]},
-        "2.10.0": {"AllenSDK": ["2.10.0", "3.0.0"]}
-    }
-}
+MANIFEST_COMPATIBILITY = ["0.0.0", "1.0.0"]
 
 
 class BehaviorCloudCacheVersionException(Exception):
     pass
 
 
-def version_check(pipeline_versions: List[Dict[str, str]],
-                  sdk_version: str = sdk_version,
-                  compatibility: Dict[str, Dict] = COMPATIBILITY):
-    """given a pipeline_versions list (from manifest) determine
-    the pipeline version of AllenSDK used to write the data. Lookup
-    the compatibility limits, and check the the running version of
-    AllenSDK meets those limits.
-
-    Parameters
-    ----------
-    pipeline_versions: List[Dict[str, str]]:
-        each element has keys name, version, (and comment - not used here)
-    sdk_version: str
-        typically the current return value for allensdk.__version__
-    compatibility_dict: Dict
-        keys (under 'pipeline_versions' key) are specific version numbers to
-        match a pipeline version for AllenSDK from the manifest. values
-        specify the min (inclusive) and max (exclusive) limits for
-        interoperability
-
-    Raises
-    ------
-    BehaviorCloudCacheVersionException
-
-    """
-    pipeline_version = [i for i in pipeline_versions
-                        if "AllenSDK" == i["name"]]
-    if len(pipeline_version) != 1:
-        raise BehaviorCloudCacheVersionException(
-                "expected to find 1 and only 1 entry for `AllenSDK` "
-                "in the manifest.data_pipeline metadata. "
-                f"found {len(pipeline_version)}")
-    pipeline_version = pipeline_version[0]["version"]
-    if pipeline_version not in compatibility["pipeline_versions"]:
-        raise BehaviorCloudCacheVersionException(
-                f"no version compatibility listed for {pipeline_version}")
-    version_limits = compatibility["pipeline_versions"][pipeline_version]
-    smin = semver.VersionInfo.parse(version_limits["AllenSDK"][0])
-    smax = semver.VersionInfo.parse(version_limits["AllenSDK"][1])
-    if (sdk_version < smin) | (sdk_version >= smax):
-        raise BehaviorCloudCacheVersionException(
-            f"""
-            The version of the visual-behavior-ophys data files (specified
-            in path_to_users_current_release_manifest) requires that your
-            AllenSDK version be >={smin} and <{smax}.
-            Your version of AllenSDK is: {sdk_version}.
-            If you want to use the specified manifest to retrieve data, please
-            upgrade or downgrade AllenSDK to the range specified.
-            If you just want to get the latest version of visual-behavior-ophys
-            data please upgrade to the latest AllenSDK version and try this
-            process again.""")
+def version_check(manifest_version: str,
+                  data_pipeline_version: str,
+                  cmin: str = MANIFEST_COMPATIBILITY[0],
+                  cmax: str = MANIFEST_COMPATIBILITY[1]):
+    mver_parsed = semver.VersionInfo.parse(manifest_version)
+    cmin_parsed = semver.VersionInfo.parse(cmin)
+    cmax_parsed = semver.VersionInfo.parse(cmax)
+    if (mver_parsed < cmin_parsed) | (mver_parsed >= cmax_parsed):
+        estr = (f"the manifest has manifest_version {manifest_version} but "
+                "this version of AllenSDK is compatible only with manifest "
+                f"versions {cmin} <= X < {cmax}. \n"
+                "Consider using a version of AllenSDK closer to the version "
+                f"used to release the data: {data_pipeline_version}")
+        raise BehaviorCloudCacheVersionException(estr)
 
 
 def literal_col_eval(df: pd.DataFrame,
@@ -121,18 +77,25 @@ def __init__(self, cache: Union[S3CloudCache, LocalCache],
                                  "ophys_session_table",
                                  "ophys_experiment_table"])
         self.cache = cache
+
         if cache._manifest.metadata_file_names is None:
             raise RuntimeError("S3CloudCache object has no metadata "
                                "file names. BehaviorProjectCloudApi "
                                "expects a S3CloudCache passed which "
                                "has already run load_manifest()")
         cache_metadata = set(cache._manifest.metadata_file_names)
+
         if cache_metadata != expected_metadata:
             raise RuntimeError("expected S3CloudCache object to have "
                                f"metadata file names: {expected_metadata} "
                                f"but it has {cache_metadata}")
+
         if not skip_version_check:
-            version_check(self.cache._manifest._data_pipeline)
+            data_sdk_version = [i for i in cache._manifest._data_pipeline
+                                if i['name'] == "AllenSDK"][0]["version"]
+            version_check(cache._manifest.version, data_sdk_version)
+
+        #    version_check(self.cache._manifest._data_pipeline)
         self.logger = logging.getLogger("BehaviorProjectCloudApi")
         self._local = local
         self._get_ophys_session_table()

allensdk/brain_observatory/behavior/project_apis/data_io/behavior_project_lims_api.py → allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_lims_api.py

@@ -2,8 +2,7 @@
 from typing import Optional, List, Dict, Any, Iterable
 import logging
 
-from allensdk.brain_observatory.behavior.project_apis.abcs import (
-    BehaviorProjectBase)
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.abcs import BehaviorProjectBase  # noqa: E501
 from allensdk.brain_observatory.behavior.behavior_session import (
     BehaviorSession)
 from allensdk.brain_observatory.behavior.behavior_ophys_experiment import (
@@ -328,7 +327,8 @@ def get_behavior_ophys_experiment(self, ophys_experiment_id: int
         :type ophys_experiment_id: int
         :rtype: BehaviorOphysExperiment
         """
-        return BehaviorOphysExperiment(BehaviorOphysLimsApi(ophys_experiment_id))
+        return BehaviorOphysExperiment(
+                BehaviorOphysLimsApi(ophys_experiment_id))
 
     def _get_ophys_experiment_table(self) -> pd.DataFrame:
         """
@@ -343,12 +343,12 @@ def _get_ophys_experiment_table(self) -> pd.DataFrame:
 
         :rtype: pd.DataFrame
         """
-        query = f"""
+        query = """
             SELECT
                 oe.id as ophys_experiment_id,
                 os.id as ophys_session_id,
                 bs.id as behavior_session_id,
-                oec.visual_behavior_experiment_container_id as 
+                oec.visual_behavior_experiment_container_id as
                     ophys_container_id,
                 pr.code as project_code,
                 vbc.workflow_state as container_workflow_state,
@@ -458,7 +458,8 @@ def get_ophys_experiment_table(
             to include
         :rtype: pd.DataFrame
         """
-        return self._get_ophys_experiment_table().set_index("ophys_experiment_id")
+        df = self._get_ophys_experiment_table()
+        return df.set_index("ophys_experiment_id")
 
     def get_behavior_session_table(self) -> pd.DataFrame:
         """Returns a pd.DataFrame table with all behavior session_ids to the
@@ -490,39 +491,39 @@ def get_release_files(self, file_type='BehaviorNwb') -> pd.DataFrame:
             -columns file_id and isilon filepath
         """
         if self.data_release_date is None:
-            raise RuntimeError(f'data_release_date must be set in constructor')
+            raise RuntimeError('data_release_date must be set in constructor')
 
         if file_type not in ('BehaviorNwb', 'BehaviorOphysNwb'):
             raise ValueError(f'cannot retrieve file type {file_type}')
 
         if file_type == 'BehaviorNwb':
             attachable_id_alias = 'behavior_session_id'
             select_clause = f'''
-                SELECT attachable_id as {attachable_id_alias}, id as file_id, 
+                SELECT attachable_id as {attachable_id_alias}, id as file_id,
                     filename, storage_directory
             '''
             join_clause = ''
         else:
             attachable_id_alias = 'ophys_experiment_id'
             select_clause = f'''
-                SELECT attachable_id as {attachable_id_alias}, 
-                    bs.id as behavior_session_id, wkf.id as file_id, 
+                SELECT attachable_id as {attachable_id_alias},
+                    bs.id as behavior_session_id, wkf.id as file_id,
                     filename, wkf.storage_directory
             '''
-            join_clause = f'''
+            join_clause = """
                 JOIN ophys_experiments oe ON oe.id = attachable_id
                 JOIN ophys_sessions os ON os.id = oe.ophys_session_id
                 JOIN behavior_sessions bs on bs.ophys_session_id = os.id
-            '''
+            """
 
         query = f'''
             {select_clause}
             FROM well_known_files wkf
             {join_clause}
-            WHERE published_at = '{self.data_release_date}' AND 
+            WHERE published_at = '{self.data_release_date}' AND
                 well_known_file_type_id IN (
-                    SELECT id 
-                    FROM well_known_file_types 
+                    SELECT id
+                    FROM well_known_file_types
                     WHERE name = '{file_type}'
                 );
         '''

allensdk/brain_observatory/behavior/behavior_project_cache/tables/sessions_table.py

@@ -15,8 +15,7 @@
     ProjectTable
 from allensdk.brain_observatory.behavior.metadata.behavior_metadata import \
     BehaviorMetadata
-from allensdk.brain_observatory.behavior.project_apis.data_io import \
-    BehaviorProjectLimsApi
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import BehaviorProjectLimsApi  # noqa: E501
 
 
 class SessionsTable(ProjectTable):

allensdk/brain_observatory/behavior/behavior_project_cache/tables/util/prior_exposure_processing.py

@@ -3,8 +3,7 @@
 
 import pandas as pd
 
-from allensdk.brain_observatory.behavior.project_apis.data_io import \
-    BehaviorProjectLimsApi
+from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import BehaviorProjectLimsApi  # noqa: E501
 
 
 def get_prior_exposures_to_session_type(df: pd.DataFrame) -> pd.Series:
@@ -43,8 +42,7 @@ def get_prior_exposures_to_image_set(df: pd.DataFrame) -> pd.Series:
     """
 
     def __get_image_set_name(session_type: Optional[str]):
-        match = re.match(r'OPHYS_\d+_images_(?P<image_set>\w)',
-                         session_type)
+        match = re.match(r'.*images_(?P<image_set>\w)', session_type)
         if match is None:
             return None
         return match.group('image_set')