From d027aa4ce995dd799910f485fab859be8dfe5125 Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 12:45:43 +0800
Subject: [PATCH 01/11] add test list

---
 tests/data/frame_test_list_with_offset.txt | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 tests/data/frame_test_list_with_offset.txt

diff --git a/tests/data/frame_test_list_with_offset.txt b/tests/data/frame_test_list_with_offset.txt
new file mode 100644
index 0000000000..bd44d276e0
--- /dev/null
+++ b/tests/data/frame_test_list_with_offset.txt
@@ -0,0 +1,2 @@
+test_imgs 2 5 127
+test_imgs 2 5 127

From 16c438fa16fabe4e0d53562fa14d4c8701191acb Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 12:48:14 +0800
Subject: [PATCH 02/11] add code for with_offset

---
 mmaction/datasets/pipelines/loading.py |  5 +++
 mmaction/datasets/rawframe_dataset.py  | 43 +++++++++++++++++---------
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/mmaction/datasets/pipelines/loading.py b/mmaction/datasets/pipelines/loading.py
index 6b1dee1412..2b249a4b1f 100644
--- a/mmaction/datasets/pipelines/loading.py
+++ b/mmaction/datasets/pipelines/loading.py
@@ -591,7 +591,12 @@ def __call__(self, results):
         if results['frame_inds'].ndim != 1:
             results['frame_inds'] = np.squeeze(results['frame_inds'])
 
+        offset = 0
+        if 'offset' in results:
+            offset = results['offset']
+
         for frame_idx in results['frame_inds']:
+            frame_idx += offset
             if modality == 'RGB':
                 filepath = osp.join(directory, filename_tmpl.format(frame_idx))
                 img_bytes = self.file_client.get(filepath)
diff --git a/mmaction/datasets/rawframe_dataset.py b/mmaction/datasets/rawframe_dataset.py
index 6b1b581734..40ae9ffb9b 100644
--- a/mmaction/datasets/rawframe_dataset.py
+++ b/mmaction/datasets/rawframe_dataset.py
@@ -52,6 +52,8 @@ class RawframeDataset(BaseDataset):
             Default: False.
         filename_tmpl (str): Template for each filename.
             Default: 'img_{:05}.jpg'.
+        with_offset (bool): Determines whether the offset information is in
+            ann_file. If not, frame_idx starts from 1. Default: False.
         multi_class (bool): Determines whether it is a multi-class
             recognition dataset. Default: False.
         num_classes (int): Number of classes in the dataset. Default: None.
@@ -65,12 +67,14 @@ def __init__(self,
                  data_prefix=None,
                  test_mode=False,
                  filename_tmpl='img_{:05}.jpg',
+                 with_offset=False,
                  multi_class=False,
                  num_classes=None,
                  modality='RGB'):
         super().__init__(ann_file, pipeline, data_prefix, test_mode,
                          multi_class, num_classes, modality)
         self.filename_tmpl = filename_tmpl
+        self.with_offset = with_offset
 
     def load_annotations(self):
         """Load annotation file to get video information."""
@@ -78,23 +82,34 @@ def load_annotations(self):
         with open(self.ann_file, 'r') as fin:
             for line in fin:
                 line_split = line.strip().split()
-                if self.multi_class:
+                item = {}
+                idx = 0
+                frame_dir = line_split[idx]
+                if self.data_prefix is not None:
+                    frame_dir = osp.join(self.data_prefix, frame_dir)
+                item['frame_dir'] = frame_dir
+                idx += 1
+                if self.with_offset:
+                    item['offset'] = int(line_split[idx])
+                    item['total_frames'] = int(line_split[idx + 1])
+                    idx += 2
+                else:
+                    item['total_frames'] = int(line_split[idx])
+                    idx += 1
+                label = [int(x) for x in line_split[idx:]]
+                assert len(label), 'missing label in line: {}'.format(line)
+                if len(label) == 1:
+                    item['label'] = label[0]
+                else:
+                    assert self.multi_class, (
+                        'found multiple labels in line'
+                        '{}, however multi_class == False'.format(line))
                     assert self.num_classes is not None
-                    (frame_dir, total_frames,
-                     label) = (line_split[0], line_split[1], line_split[2:])
-                    label = list(map(int, label))
                     onehot = torch.zeros(self.num_classes)
                     onehot[label] = 1.0
-                else:
-                    frame_dir, total_frames, label = line_split
-                    label = int(label)
-                if self.data_prefix is not None:
-                    frame_dir = osp.join(self.data_prefix, frame_dir)
-                video_infos.append(
-                    dict(
-                        frame_dir=frame_dir,
-                        total_frames=int(total_frames),
-                        label=onehot if self.multi_class else label))
+                    item['label'] = onehot
+                video_infos.append(item)
+
         return video_infos
 
     def prepare_train_frames(self, idx):

From e5ce3c79d92766e6d9f9df5a3755272b7bc4786f Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 12:49:10 +0800
Subject: [PATCH 03/11] add test for with_offset

---
 tests/test_data/test_dataset.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/test_data/test_dataset.py b/tests/test_data/test_dataset.py
index 24dde773ce..e78e7a08ef 100644
--- a/tests/test_data/test_dataset.py
+++ b/tests/test_data/test_dataset.py
@@ -55,6 +55,18 @@ def test_rawframe_dataset(self):
             dict(frame_dir=frame_dir, total_frames=5, label=127)
         ] * 2
 
+    def test_rawframe_dataset_with_offset(self):
+        rawframe_dataset = RawframeDataset(
+            self.frame_ann_file,
+            self.frame_pipeline,
+            self.data_prefix,
+            with_offset=True)
+        rawframe_infos = rawframe_dataset.video_infos
+        frame_dir = osp.join(self.data_prefix, 'test_imgs')
+        assert rawframe_infos == [
+            dict(frame_dir=frame_dir, offset=2, total_frames=5, label=127)
+        ] * 2
+
     def test_dataset_realpath(self):
         dataset = RawframeDataset(self.frame_ann_file, self.frame_pipeline,
                                   '.')

From f22a498d4e63de91f18d12a0898eac21ab34e3d4 Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 12:59:27 +0800
Subject: [PATCH 04/11] reorder, load_anno need with_offset

---
 mmaction/datasets/rawframe_dataset.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mmaction/datasets/rawframe_dataset.py b/mmaction/datasets/rawframe_dataset.py
index 40ae9ffb9b..d2b035d48f 100644
--- a/mmaction/datasets/rawframe_dataset.py
+++ b/mmaction/datasets/rawframe_dataset.py
@@ -71,10 +71,10 @@ def __init__(self,
                  multi_class=False,
                  num_classes=None,
                  modality='RGB'):
-        super().__init__(ann_file, pipeline, data_prefix, test_mode,
-                         multi_class, num_classes, modality)
         self.filename_tmpl = filename_tmpl
         self.with_offset = with_offset
+        super().__init__(ann_file, pipeline, data_prefix, test_mode,
+                         multi_class, num_classes, modality)
 
     def load_annotations(self):
         """Load annotation file to get video information."""

From dfeb84abac78cca2e34a0bbbeafc1fc281695702 Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 13:08:45 +0800
Subject: [PATCH 05/11] fix bug in test_dataset

---
 tests/test_data/test_dataset.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_data/test_dataset.py b/tests/test_data/test_dataset.py
index e78e7a08ef..0bca3febb1 100644
--- a/tests/test_data/test_dataset.py
+++ b/tests/test_data/test_dataset.py
@@ -22,6 +22,8 @@ def check_keys_contain(result_keys, target_keys):
     def setup_class(cls):
         cls.data_prefix = osp.join(osp.dirname(osp.dirname(__file__)), 'data')
         cls.frame_ann_file = osp.join(cls.data_prefix, 'frame_test_list.txt')
+        cls.frame_ann_file_with_offset = osp.join(
+            cls.data_prefix, 'frame_test_list_with_offset.txt')
         cls.video_ann_file = osp.join(cls.data_prefix, 'video_test_list.txt')
         cls.action_ann_file = osp.join(cls.data_prefix,
                                        'action_test_anno.json')
@@ -57,7 +59,7 @@ def test_rawframe_dataset(self):
 
     def test_rawframe_dataset_with_offset(self):
         rawframe_dataset = RawframeDataset(
-            self.frame_ann_file,
+            self.frame_ann_file_with_offset,
             self.frame_pipeline,
             self.data_prefix,
             with_offset=True)

From a0a09feeacb17d71873705b6c66f34215f26a758 Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 14:50:44 +0800
Subject: [PATCH 06/11] add missing unittest for multilabel

---
 mmaction/datasets/rawframe_dataset.py      | 11 +++++------
 tests/data/frame_test_list_multi_label.txt |  2 ++
 tests/test_data/test_dataset.py            | 21 +++++++++++++++++++++
 tests/test_data/test_loading.py            |  2 ++
 4 files changed, 30 insertions(+), 6 deletions(-)
 create mode 100644 tests/data/frame_test_list_multi_label.txt

diff --git a/mmaction/datasets/rawframe_dataset.py b/mmaction/datasets/rawframe_dataset.py
index d2b035d48f..ada7b756a3 100644
--- a/mmaction/datasets/rawframe_dataset.py
+++ b/mmaction/datasets/rawframe_dataset.py
@@ -98,16 +98,15 @@ def load_annotations(self):
                     idx += 1
                 label = [int(x) for x in line_split[idx:]]
                 assert len(label), 'missing label in line: {}'.format(line)
-                if len(label) == 1:
-                    item['label'] = label[0]
-                else:
-                    assert self.multi_class, (
-                        'found multiple labels in line'
-                        '{}, however multi_class == False'.format(line))
+
+                if self.multi_class:
                     assert self.num_classes is not None
                     onehot = torch.zeros(self.num_classes)
                     onehot[label] = 1.0
                     item['label'] = onehot
+                else:
+                    assert len(label) == 1
+                    item['label'] = label[0]
                 video_infos.append(item)
 
         return video_infos
diff --git a/tests/data/frame_test_list_multi_label.txt b/tests/data/frame_test_list_multi_label.txt
new file mode 100644
index 0000000000..ddd9dffff2
--- /dev/null
+++ b/tests/data/frame_test_list_multi_label.txt
@@ -0,0 +1,2 @@
+test_imgs 5 1
+test_imgs 5 3 5
diff --git a/tests/test_data/test_dataset.py b/tests/test_data/test_dataset.py
index 0bca3febb1..dd7e1db013 100644
--- a/tests/test_data/test_dataset.py
+++ b/tests/test_data/test_dataset.py
@@ -5,6 +5,7 @@
 import mmcv
 import numpy as np
 import pytest
+import torch
 from numpy.testing import assert_array_equal
 
 from mmaction.datasets import (ActivityNetDataset, RawframeDataset,
@@ -24,6 +25,8 @@ def setup_class(cls):
         cls.frame_ann_file = osp.join(cls.data_prefix, 'frame_test_list.txt')
         cls.frame_ann_file_with_offset = osp.join(
             cls.data_prefix, 'frame_test_list_with_offset.txt')
+        cls.frame_ann_file_multi_label = osp.join(
+            cls.data_prefix, 'frame_test_list_multi_label.txt')
         cls.video_ann_file = osp.join(cls.data_prefix, 'video_test_list.txt')
         cls.action_ann_file = osp.join(cls.data_prefix,
                                        'action_test_anno.json')
@@ -69,6 +72,24 @@ def test_rawframe_dataset_with_offset(self):
             dict(frame_dir=frame_dir, offset=2, total_frames=5, label=127)
         ] * 2
 
+    def test_rawframe_dataset_multi_label(self):
+        rawframe_dataset = RawframeDataset(
+            self.frame_ann_file_multi_labels,
+            self.frame_pipeline,
+            self.data_prefix,
+            multi_class=True,
+            num_classes=100)
+        rawframe_infos = rawframe_dataset.video_infos
+        frame_dir = osp.join(self.data_prefix, 'test_imgs')
+        label0 = torch.zeros(100)
+        label0[[1]] = 1.0
+        label1 = torch.zeros(100)
+        label1[[3, 5]] = 1.0
+        assert rawframe_infos == [
+            dict(frame_dir=frame_dir, total_frames=5, label=label0),
+            dict(frame_dir=frame_dir, total_frames=5, label=label1)
+        ]
+
     def test_dataset_realpath(self):
         dataset = RawframeDataset(self.frame_ann_file, self.frame_pipeline,
                                   '.')
diff --git a/tests/test_data/test_loading.py b/tests/test_data/test_loading.py
index 8e3da593e2..466b280dc3 100644
--- a/tests/test_data/test_loading.py
+++ b/tests/test_data/test_loading.py
@@ -45,12 +45,14 @@ def setup_class(cls):
             total_frames=cls.total_frames,
             filename_tmpl=cls.filename_tmpl,
             modality='RGB',
+            offset=0,
             label=1)
         cls.flow_frame_results = dict(
             frame_dir=cls.img_dir,
             total_frames=cls.total_frames,
             filename_tmpl=cls.flow_filename_tmpl,
             modality='Flow',
+            offset=0,
             label=1)
         cls.action_results = dict(
             video_name='v_test1',

From b8bcaaaebe3fa0bf5faaa30552728a73c8f588ab Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 15:19:43 +0800
Subject: [PATCH 07/11] resolve

---
 mmaction/datasets/pipelines/loading.py |  4 +--
 mmaction/datasets/rawframe_dataset.py  | 34 ++++++++++++++++++--------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/mmaction/datasets/pipelines/loading.py b/mmaction/datasets/pipelines/loading.py
index 2b249a4b1f..86815a97e3 100644
--- a/mmaction/datasets/pipelines/loading.py
+++ b/mmaction/datasets/pipelines/loading.py
@@ -591,9 +591,7 @@ def __call__(self, results):
         if results['frame_inds'].ndim != 1:
             results['frame_inds'] = np.squeeze(results['frame_inds'])
 
-        offset = 0
-        if 'offset' in results:
-            offset = results['offset']
+        offset = results.get('offset', 0)
 
         for frame_idx in results['frame_inds']:
             frame_idx += offset
diff --git a/mmaction/datasets/rawframe_dataset.py b/mmaction/datasets/rawframe_dataset.py
index ada7b756a3..5e76e1ec21 100644
--- a/mmaction/datasets/rawframe_dataset.py
+++ b/mmaction/datasets/rawframe_dataset.py
@@ -42,6 +42,21 @@ class RawframeDataset(BaseDataset):
         some/directory-5 295 3
         some/directory-6 121 3
 
+    Example of a with_offset annotation file (clips from long videos), each
+    line indicates the directory to frames of a video, the index of the start
+    frame, total frames of the video clip and the label of a video clip, which
+    are split with a whitespace.
+
+
+    .. code-block:: txt
+
+        some/directory-1 12 163 3
+        some/directory-2 213 122 4
+        some/directory-3 100 258 5
+        some/directory-4 98 234 2
+        some/directory-5 0 295 3
+        some/directory-6 50 121 3
+
 
     Args:
         ann_file (str): Path to the annotation file.
@@ -82,32 +97,31 @@ def load_annotations(self):
         with open(self.ann_file, 'r') as fin:
             for line in fin:
                 line_split = line.strip().split()
-                item = {}
+                video_info = {}
                 idx = 0
                 frame_dir = line_split[idx]
                 if self.data_prefix is not None:
                     frame_dir = osp.join(self.data_prefix, frame_dir)
-                item['frame_dir'] = frame_dir
+                video_info['frame_dir'] = frame_dir
                 idx += 1
                 if self.with_offset:
-                    item['offset'] = int(line_split[idx])
-                    item['total_frames'] = int(line_split[idx + 1])
+                    video_info['offset'] = int(line_split[idx])
+                    video_info['total_frames'] = int(line_split[idx + 1])
                     idx += 2
                 else:
-                    item['total_frames'] = int(line_split[idx])
+                    video_info['total_frames'] = int(line_split[idx])
                     idx += 1
                 label = [int(x) for x in line_split[idx:]]
-                assert len(label), 'missing label in line: {}'.format(line)
-
+                assert len(label), f'missing label in line: {line}'
                 if self.multi_class:
                     assert self.num_classes is not None
                     onehot = torch.zeros(self.num_classes)
                     onehot[label] = 1.0
-                    item['label'] = onehot
+                    video_info['label'] = onehot
                 else:
                     assert len(label) == 1
-                    item['label'] = label[0]
-                video_infos.append(item)
+                    video_info['label'] = label[0]
+                video_infos.append(video_info)
 
         return video_infos
 

From 8b17284e559260b4cb4ee20842439ccf0f643a8c Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 15:31:03 +0800
Subject: [PATCH 08/11] fix bug

---
 tests/test_data/test_dataset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_data/test_dataset.py b/tests/test_data/test_dataset.py
index dd7e1db013..ee2f51a464 100644
--- a/tests/test_data/test_dataset.py
+++ b/tests/test_data/test_dataset.py
@@ -74,7 +74,7 @@ def test_rawframe_dataset_with_offset(self):
 
     def test_rawframe_dataset_multi_label(self):
         rawframe_dataset = RawframeDataset(
-            self.frame_ann_file_multi_labels,
+            self.frame_ann_file_multi_label,
             self.frame_pipeline,
             self.data_prefix,
             multi_class=True,

From 2f711be4c22cbf04b487f982c7da8bc538fa8105 Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Thu, 23 Jul 2020 16:38:30 +0800
Subject: [PATCH 09/11] fix test, add comment

---
 mmaction/datasets/rawframe_dataset.py | 4 ++++
 tests/test_data/test_dataset.py       | 9 +++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/mmaction/datasets/rawframe_dataset.py b/mmaction/datasets/rawframe_dataset.py
index 5e76e1ec21..0c56aa79a3 100644
--- a/mmaction/datasets/rawframe_dataset.py
+++ b/mmaction/datasets/rawframe_dataset.py
@@ -99,18 +99,22 @@ def load_annotations(self):
                 line_split = line.strip().split()
                 video_info = {}
                 idx = 0
+                # idx for frame_dir
                 frame_dir = line_split[idx]
                 if self.data_prefix is not None:
                     frame_dir = osp.join(self.data_prefix, frame_dir)
                 video_info['frame_dir'] = frame_dir
                 idx += 1
                 if self.with_offset:
+                    # idx for offset and total_frames
                     video_info['offset'] = int(line_split[idx])
                     video_info['total_frames'] = int(line_split[idx + 1])
                     idx += 2
                 else:
+                    # idx for total_frames
                     video_info['total_frames'] = int(line_split[idx])
                     idx += 1
+                # idx for label[s]
                 label = [int(x) for x in line_split[idx:]]
                 assert len(label), f'missing label in line: {line}'
                 if self.multi_class:
diff --git a/tests/test_data/test_dataset.py b/tests/test_data/test_dataset.py
index ee2f51a464..ae6d0b2a8e 100644
--- a/tests/test_data/test_dataset.py
+++ b/tests/test_data/test_dataset.py
@@ -85,10 +85,11 @@ def test_rawframe_dataset_multi_label(self):
         label0[[1]] = 1.0
         label1 = torch.zeros(100)
         label1[[3, 5]] = 1.0
-        assert rawframe_infos == [
-            dict(frame_dir=frame_dir, total_frames=5, label=label0),
-            dict(frame_dir=frame_dir, total_frames=5, label=label1)
-        ]
+        labels = [label0, label1]
+        for info, label in zip(rawframe_infos, labels):
+            assert info['frame_dir'] == frame_dir
+            assert info['total_frames'] == 5
+            assert info['label'] == label
 
     def test_dataset_realpath(self):
         dataset = RawframeDataset(self.frame_ann_file, self.frame_pipeline,

From 03a352c1d662d870f7e03c62efb6cea6f85fda2b Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Fri, 24 Jul 2020 09:07:09 +0800
Subject: [PATCH 10/11] fix test

---
 tests/test_data/test_dataset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_data/test_dataset.py b/tests/test_data/test_dataset.py
index ae6d0b2a8e..3f9d451e55 100644
--- a/tests/test_data/test_dataset.py
+++ b/tests/test_data/test_dataset.py
@@ -89,7 +89,7 @@ def test_rawframe_dataset_multi_label(self):
         for info, label in zip(rawframe_infos, labels):
             assert info['frame_dir'] == frame_dir
             assert info['total_frames'] == 5
-            assert info['label'] == label
+            assert torch.all(info['label'] == label)
 
     def test_dataset_realpath(self):
         dataset = RawframeDataset(self.frame_ann_file, self.frame_pipeline,

From cedfbd2f022080872716b2f8649886fd320cdfbe Mon Sep 17 00:00:00 2001
From: HaodongDuan <duanhaodong@sensetime.com>
Date: Wed, 29 Jul 2020 16:09:57 +0800
Subject: [PATCH 11/11] minor fix

---
 mmaction/datasets/rawframe_dataset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mmaction/datasets/rawframe_dataset.py b/mmaction/datasets/rawframe_dataset.py
index 0c56aa79a3..3fdae695ef 100644
--- a/mmaction/datasets/rawframe_dataset.py
+++ b/mmaction/datasets/rawframe_dataset.py
@@ -68,7 +68,7 @@ class RawframeDataset(BaseDataset):
         filename_tmpl (str): Template for each filename.
             Default: 'img_{:05}.jpg'.
         with_offset (bool): Determines whether the offset information is in
-            ann_file. If not, frame_idx starts from 1. Default: False.
+            ann_file. Default: False.
         multi_class (bool): Determines whether it is a multi-class
             recognition dataset. Default: False.
         num_classes (int): Number of classes in the dataset. Default: None.