diff --git a/docs/getting_started.md b/docs/getting_started.md
index 1c7f204c95..a31934aec7 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -248,7 +248,7 @@ in [TSM: Temporal Shift Module for Efficient Video Understanding](https://arxiv.
 1. create a new file in `mmaction/models/backbones/resnet_tsm.py`.
 
     ```python
-    from ..registry import BACKBONES
+    from ..builder import BACKBONES
     from .resnet import ResNet
 
     @BACKBONES.register_module()
diff --git a/docs/tutorials/3_new_dataset.md b/docs/tutorials/3_new_dataset.md
index 8c7d08c913..1b1d18d37e 100644
--- a/docs/tutorials/3_new_dataset.md
+++ b/docs/tutorials/3_new_dataset.md
@@ -170,7 +170,7 @@ import os.path as osp
 import mmcv
 
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/docs/tutorials/5_new_modules.md b/docs/tutorials/5_new_modules.md
index 0652c3f7af..940205b596 100644
--- a/docs/tutorials/5_new_modules.md
+++ b/docs/tutorials/5_new_modules.md
@@ -23,7 +23,7 @@ Assume you want to add an optimizer named as `MyOptimizer`, which has arguments
 You need to first implement the new optimizer in a file, e.g., in `mmaction/core/optimizer/my_optimizer.py`:
 
 ```python
-from .registry import OPTIMIZERS
+from mmcv.runner import OPTIMIZERS
 from torch.optim import Optimizer
 
 @OPTIMIZERS.register_module()
@@ -118,7 +118,7 @@ Here we show how to develop new components with an example of TSN.
     ```python
     import torch.nn as nn
 
-    from ..registry import BACKBONES
+    from ..builder import BACKBONES
 
     @BACKBONES.register_module()
     class ResNet(nn.Module):
@@ -161,7 +161,7 @@ Here we show how to develop a new head with the example of TSNHead as the follow
     and overwrite `init_weights(self)` and `forward(self, x)` method.
 
     ```python
-    from ..registry import HEADS
+    from ..builder import HEADS
     from .base import BaseHead
 
 
diff --git a/docs/tutorials/7_customize_runtime.md b/docs/tutorials/7_customize_runtime.md
index 50c77542ff..dffb2e9668 100644
--- a/docs/tutorials/7_customize_runtime.md
+++ b/docs/tutorials/7_customize_runtime.md
@@ -59,7 +59,7 @@ You need to create a new directory named `mmaction/core/optimizer`.
 And then implement the new optimizer in a file, e.g., in `mmaction/core/optimizer/my_optimizer.py`:
 
 ```python
-from .registry import OPTIMIZERS
+from mmcv.runner import OPTIMIZERS
 from torch.optim import Optimizer
 
 
@@ -113,11 +113,7 @@ Some models may have some parameter-specific settings for optimization, e.g. wei
 The users can do those fine-grained parameter tuning through customizing optimizer constructor.
 
 ```python
-from mmcv.utils import build_from_cfg
-
-from mmcv.runner.optimizer import OPTIMIZER_BUILDERS, OPTIMIZERS
-from mmaction.utils import get_root_logger
-from .my_optimizer import MyOptimizer
+from mmcv.runner.optimizer import OPTIMIZER_BUILDERS
 
 
 @OPTIMIZER_BUILDERS.register_module()
diff --git a/docs_zh_CN/getting_started.md b/docs_zh_CN/getting_started.md
index 7e3de13fc3..b0e7dde2fa 100644
--- a/docs_zh_CN/getting_started.md
+++ b/docs_zh_CN/getting_started.md
@@ -242,7 +242,7 @@ MMAction2 将模型组件分为 4 种基础模型：
 1. 创建 `mmaction/models/backbones/resnet_tsm.py` 文件
 
     ```python
-    from ..registry import BACKBONES
+    from ..builder import BACKBONES
     from .resnet import ResNet
 
     @BACKBONES.register_module()
diff --git a/docs_zh_CN/tutorials/3_new_dataset.md b/docs_zh_CN/tutorials/3_new_dataset.md
index b6d2faf9e6..19402cb41e 100644
--- a/docs_zh_CN/tutorials/3_new_dataset.md
+++ b/docs_zh_CN/tutorials/3_new_dataset.md
@@ -163,7 +163,7 @@ import os.path as osp
 import mmcv
 
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/docs_zh_CN/tutorials/5_new_modules.md b/docs_zh_CN/tutorials/5_new_modules.md
index f6f23d1fe7..ead61332bb 100644
--- a/docs_zh_CN/tutorials/5_new_modules.md
+++ b/docs_zh_CN/tutorials/5_new_modules.md
@@ -23,7 +23,7 @@
 用户需要首先实现一个新的优化器文件，如 `mmaction/core/optimizer/my_optimizer.py`：
 
 ```python
-from .registry import OPTIMIZERS
+from mmcv.runner import OPTIMIZERS
 from torch.optim import Optimizer
 
 @OPTIMIZERS.register_module()
@@ -105,7 +105,7 @@ MMAction2 将模型组件分为 4 种基础模型：
     ```python
     import torch.nn as nn
 
-    from ..registry import BACKBONES
+    from ..builder import BACKBONES
 
     @BACKBONES.register_module()
     class ResNet(nn.Module):
@@ -148,7 +148,7 @@ MMAction2 将模型组件分为 4 种基础模型：
     并重写 `init_weights(self)` 和 `forward(self, x)` 方法
 
     ```python
-    from ..registry import HEADS
+    from ..builder import HEADS
     from .base import BaseHead
 
 
diff --git a/docs_zh_CN/tutorials/7_customize_runtime.md b/docs_zh_CN/tutorials/7_customize_runtime.md
index d0e80a0ff0..bbf451a0db 100644
--- a/docs_zh_CN/tutorials/7_customize_runtime.md
+++ b/docs_zh_CN/tutorials/7_customize_runtime.md
@@ -58,7 +58,7 @@ optimizer = dict(type='Adam', lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_de
 可以创建一个名为 `mmaction/core/optimizer` 的文件夹，并在目录下的文件进行构建，如 `mmaction/core/optimizer/my_optimizer.py`：
 
 ```python
-from .registry import OPTIMIZERS
+from mmcv.runner import OPTIMIZERS
 from torch.optim import Optimizer
 
 
@@ -111,11 +111,7 @@ optimizer = dict(type='MyOptimizer', a=a_value, b=b_value, c=c_value)
 用户可以通过自定义优化器构造函数来进行那些细粒度的参数调整。
 
 ```python
-from mmcv.utils import build_from_cfg
-
-from mmcv.runner.optimizer import OPTIMIZER_BUILDERS, OPTIMIZERS
-from mmaction.utils import get_root_logger
-from .my_optimizer import MyOptimizer
+from mmcv.runner.optimizer import OPTIMIZER_BUILDERS
 
 
 @OPTIMIZER_BUILDERS.register_module()
diff --git a/mmaction/datasets/__init__.py b/mmaction/datasets/__init__.py
index 48341c9383..5575422b67 100644
--- a/mmaction/datasets/__init__.py
+++ b/mmaction/datasets/__init__.py
@@ -6,7 +6,8 @@
 from .base import BaseDataset
 from .blending_utils import (BaseMiniBatchBlending, CutmixBlending,
                              MixupBlending)
-from .builder import build_dataloader, build_dataset
+from .builder import (BLENDINGS, DATASETS, PIPELINES, build_dataloader,
+                      build_dataset)
 from .dataset_wrappers import RepeatDataset
 from .hvu_dataset import HVUDataset
 from .image_dataset import ImageDataset
@@ -20,5 +21,6 @@
     'RawframeDataset', 'BaseDataset', 'ActivityNetDataset', 'SSNDataset',
     'HVUDataset', 'AudioDataset', 'AudioFeatureDataset', 'ImageDataset',
     'RawVideoDataset', 'AVADataset', 'AudioVisualDataset',
-    'BaseMiniBatchBlending', 'CutmixBlending', 'MixupBlending'
+    'BaseMiniBatchBlending', 'CutmixBlending', 'MixupBlending', 'DATASETS',
+    'PIPELINES', 'BLENDINGS'
 ]
diff --git a/mmaction/datasets/activitynet_dataset.py b/mmaction/datasets/activitynet_dataset.py
index 018fd7b889..db04a8cfb2 100644
--- a/mmaction/datasets/activitynet_dataset.py
+++ b/mmaction/datasets/activitynet_dataset.py
@@ -9,7 +9,7 @@
 
 from ..core import average_recall_at_avg_proposals
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/audio_dataset.py b/mmaction/datasets/audio_dataset.py
index 4443402a71..041695e9b2 100644
--- a/mmaction/datasets/audio_dataset.py
+++ b/mmaction/datasets/audio_dataset.py
@@ -3,7 +3,7 @@
 import torch
 
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/audio_feature_dataset.py b/mmaction/datasets/audio_feature_dataset.py
index 15daa1182c..89e7c06e3d 100644
--- a/mmaction/datasets/audio_feature_dataset.py
+++ b/mmaction/datasets/audio_feature_dataset.py
@@ -3,7 +3,7 @@
 import torch
 
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/audio_visual_dataset.py b/mmaction/datasets/audio_visual_dataset.py
index 4a1b97f946..6e10b4b040 100644
--- a/mmaction/datasets/audio_visual_dataset.py
+++ b/mmaction/datasets/audio_visual_dataset.py
@@ -1,7 +1,7 @@
 import os.path as osp
 
+from .builder import DATASETS
 from .rawframe_dataset import RawframeDataset
-from .registry import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/ava_dataset.py b/mmaction/datasets/ava_dataset.py
index fb017c0358..111a5e146f 100644
--- a/mmaction/datasets/ava_dataset.py
+++ b/mmaction/datasets/ava_dataset.py
@@ -11,7 +11,7 @@
 from ..core.evaluation.ava_utils import ava_eval, read_labelmap, results2csv
 from ..utils import get_root_logger
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/blending_utils.py b/mmaction/datasets/blending_utils.py
index 88eb541f61..8ef35b0e73 100644
--- a/mmaction/datasets/blending_utils.py
+++ b/mmaction/datasets/blending_utils.py
@@ -4,7 +4,7 @@
 import torch.nn.functional as F
 from torch.distributions.beta import Beta
 
-from .registry import BLENDINGS
+from .builder import BLENDINGS
 
 __all__ = ['BaseMiniBatchBlending', 'MixupBlending', 'CutmixBlending']
 
diff --git a/mmaction/datasets/builder.py b/mmaction/datasets/builder.py
index 33056bde7f..267c323b0f 100644
--- a/mmaction/datasets/builder.py
+++ b/mmaction/datasets/builder.py
@@ -5,11 +5,9 @@
 import numpy as np
 from mmcv.parallel import collate
 from mmcv.runner import get_dist_info
-from mmcv.utils import build_from_cfg
+from mmcv.utils import Registry, build_from_cfg
 from torch.utils.data import DataLoader
 
-from .dataset_wrappers import RepeatDataset
-from .registry import DATASETS
 from .samplers import ClassSpecificDistributedSampler, DistributedSampler
 
 if platform.system() != 'Windows':
@@ -20,6 +18,10 @@
     soft_limit = min(4096, hard_limit)
     resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
 
+DATASETS = Registry('dataset')
+PIPELINES = Registry('pipeline')
+BLENDINGS = Registry('blending')
+
 
 def build_dataset(cfg, default_args=None):
     """Build a dataset from config dict.
@@ -33,6 +35,7 @@ def build_dataset(cfg, default_args=None):
         Dataset: The constructed dataset.
     """
     if cfg['type'] == 'RepeatDataset':
+        from .dataset_wrappers import RepeatDataset
         dataset = RepeatDataset(
             build_dataset(cfg['dataset'], default_args), cfg['times'])
     else:
diff --git a/mmaction/datasets/dataset_wrappers.py b/mmaction/datasets/dataset_wrappers.py
index 3ec8af0941..ecb7609b24 100644
--- a/mmaction/datasets/dataset_wrappers.py
+++ b/mmaction/datasets/dataset_wrappers.py
@@ -1,4 +1,4 @@
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/hvu_dataset.py b/mmaction/datasets/hvu_dataset.py
index 12beeb64aa..59a4c855a8 100644
--- a/mmaction/datasets/hvu_dataset.py
+++ b/mmaction/datasets/hvu_dataset.py
@@ -8,7 +8,7 @@
 
 from ..core import mean_average_precision
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/image_dataset.py b/mmaction/datasets/image_dataset.py
index 59f859b83f..a359277264 100644
--- a/mmaction/datasets/image_dataset.py
+++ b/mmaction/datasets/image_dataset.py
@@ -1,4 +1,4 @@
-from .registry import DATASETS
+from .builder import DATASETS
 from .video_dataset import VideoDataset
 
 
diff --git a/mmaction/datasets/pipelines/augmentations.py b/mmaction/datasets/pipelines/augmentations.py
index ad4d44c85a..32a8f5c75c 100644
--- a/mmaction/datasets/pipelines/augmentations.py
+++ b/mmaction/datasets/pipelines/augmentations.py
@@ -6,7 +6,7 @@
 import numpy as np
 from torch.nn.modules.utils import _pair
 
-from ..registry import PIPELINES
+from ..builder import PIPELINES
 
 
 def _init_lazy_if_proper(results, lazy):
diff --git a/mmaction/datasets/pipelines/compose.py b/mmaction/datasets/pipelines/compose.py
index f6f557a8f3..c4d315aba5 100644
--- a/mmaction/datasets/pipelines/compose.py
+++ b/mmaction/datasets/pipelines/compose.py
@@ -2,7 +2,7 @@
 
 from mmcv.utils import build_from_cfg
 
-from ..registry import PIPELINES
+from ..builder import PIPELINES
 
 
 @PIPELINES.register_module()
diff --git a/mmaction/datasets/pipelines/formating.py b/mmaction/datasets/pipelines/formating.py
index df784796dc..045d54fbd0 100644
--- a/mmaction/datasets/pipelines/formating.py
+++ b/mmaction/datasets/pipelines/formating.py
@@ -5,7 +5,7 @@
 import torch
 from mmcv.parallel import DataContainer as DC
 
-from ..registry import PIPELINES
+from ..builder import PIPELINES
 
 
 def to_tensor(data):
diff --git a/mmaction/datasets/pipelines/loading.py b/mmaction/datasets/pipelines/loading.py
index 2fb6bf3365..102b221db8 100644
--- a/mmaction/datasets/pipelines/loading.py
+++ b/mmaction/datasets/pipelines/loading.py
@@ -11,7 +11,7 @@
 from torch.nn.modules.utils import _pair
 
 from ...utils import get_random_string, get_shm_dir, get_thread_id
-from ..registry import PIPELINES
+from ..builder import PIPELINES
 
 
 @PIPELINES.register_module()
diff --git a/mmaction/datasets/rawframe_dataset.py b/mmaction/datasets/rawframe_dataset.py
index 6a0883e18e..5bcf678cfe 100644
--- a/mmaction/datasets/rawframe_dataset.py
+++ b/mmaction/datasets/rawframe_dataset.py
@@ -4,7 +4,7 @@
 import torch
 
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/rawvideo_dataset.py b/mmaction/datasets/rawvideo_dataset.py
index 5ba5e612eb..ada7b4aa66 100644
--- a/mmaction/datasets/rawvideo_dataset.py
+++ b/mmaction/datasets/rawvideo_dataset.py
@@ -5,7 +5,7 @@
 import mmcv
 
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/datasets/registry.py b/mmaction/datasets/registry.py
deleted file mode 100644
index ade60f11b3..0000000000
--- a/mmaction/datasets/registry.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from mmcv.utils import Registry
-
-DATASETS = Registry('dataset')
-PIPELINES = Registry('pipeline')
-BLENDINGS = Registry('blending')
diff --git a/mmaction/datasets/ssn_dataset.py b/mmaction/datasets/ssn_dataset.py
index 374926d42a..76d24324df 100644
--- a/mmaction/datasets/ssn_dataset.py
+++ b/mmaction/datasets/ssn_dataset.py
@@ -12,7 +12,7 @@
                             perform_regression, temporal_iou, temporal_nms)
 from ..utils import get_root_logger
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 class SSNInstance:
diff --git a/mmaction/datasets/video_dataset.py b/mmaction/datasets/video_dataset.py
index 08b90862be..7c1b681b99 100644
--- a/mmaction/datasets/video_dataset.py
+++ b/mmaction/datasets/video_dataset.py
@@ -1,7 +1,7 @@
 import os.path as osp
 
 from .base import BaseDataset
-from .registry import DATASETS
+from .builder import DATASETS
 
 
 @DATASETS.register_module()
diff --git a/mmaction/models/__init__.py b/mmaction/models/__init__.py
index f73a26d83a..ac7b4ce27e 100644
--- a/mmaction/models/__init__.py
+++ b/mmaction/models/__init__.py
@@ -2,7 +2,8 @@
                         ResNet2Plus1d, ResNet3d, ResNet3dCSN, ResNet3dLayer,
                         ResNet3dSlowFast, ResNet3dSlowOnly, ResNetAudio,
                         ResNetTIN, ResNetTSM, TANet)
-from .builder import (DETECTORS, build_backbone, build_detector, build_head,
+from .builder import (BACKBONES, DETECTORS, HEADS, LOCALIZERS, LOSSES, NECKS,
+                      RECOGNIZERS, build_backbone, build_detector, build_head,
                       build_localizer, build_loss, build_model, build_neck,
                       build_recognizer)
 from .common import LFB, TAM, Conv2plus1d, ConvAudio
@@ -16,7 +17,6 @@
 from .necks import TPN
 from .recognizers import (AudioRecognizer, BaseRecognizer, recognizer2d,
                           recognizer3d)
-from .registry import BACKBONES, HEADS, LOCALIZERS, LOSSES, RECOGNIZERS
 from .roi_extractors import SingleRoIExtractor3D
 
 __all__ = [
@@ -32,5 +32,5 @@
     'AudioTSNHead', 'X3D', 'X3DHead', 'ResNet3dLayer', 'DETECTORS',
     'SingleRoIExtractor3D', 'BBoxHeadAVA', 'ResNetAudio', 'build_detector',
     'ConvAudio', 'AVARoIHead', 'MobileNetV2', 'MobileNetV2TSM', 'TANet', 'LFB',
-    'FBOHead', 'LFBInferHead', 'TRNHead'
+    'FBOHead', 'LFBInferHead', 'TRNHead', 'NECKS'
 ]
diff --git a/mmaction/models/backbones/c3d.py b/mmaction/models/backbones/c3d.py
index 847ff576d9..cfb203988a 100644
--- a/mmaction/models/backbones/c3d.py
+++ b/mmaction/models/backbones/c3d.py
@@ -4,7 +4,7 @@
 from mmcv.utils import _BatchNorm
 
 from ...utils import get_root_logger
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 
 
 @BACKBONES.register_module()
diff --git a/mmaction/models/backbones/mobilenet_v2_tsm.py b/mmaction/models/backbones/mobilenet_v2_tsm.py
index dd37bdbb7c..af9f9d5e18 100644
--- a/mmaction/models/backbones/mobilenet_v2_tsm.py
+++ b/mmaction/models/backbones/mobilenet_v2_tsm.py
@@ -1,4 +1,4 @@
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 from .mobilenet_v2 import InvertedResidual, MobileNetV2
 from .resnet_tsm import TemporalShift
 
diff --git a/mmaction/models/backbones/resnet.py b/mmaction/models/backbones/resnet.py
index abb9aff303..5004e0f4c1 100644
--- a/mmaction/models/backbones/resnet.py
+++ b/mmaction/models/backbones/resnet.py
@@ -5,7 +5,7 @@
 from torch.utils import checkpoint as cp
 
 from ...utils import get_root_logger
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 
 
 class BasicBlock(nn.Module):
diff --git a/mmaction/models/backbones/resnet2plus1d.py b/mmaction/models/backbones/resnet2plus1d.py
index ec408028eb..4329ba404c 100644
--- a/mmaction/models/backbones/resnet2plus1d.py
+++ b/mmaction/models/backbones/resnet2plus1d.py
@@ -1,4 +1,4 @@
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 from .resnet3d import ResNet3d
 
 
diff --git a/mmaction/models/backbones/resnet3d.py b/mmaction/models/backbones/resnet3d.py
index f1023342aa..ab1641e079 100644
--- a/mmaction/models/backbones/resnet3d.py
+++ b/mmaction/models/backbones/resnet3d.py
@@ -7,7 +7,7 @@
 from torch.nn.modules.utils import _ntuple, _triple
 
 from ...utils import get_root_logger
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 
 try:
     from mmdet.models.builder import SHARED_HEADS as MMDET_SHARED_HEADS
diff --git a/mmaction/models/backbones/resnet3d_csn.py b/mmaction/models/backbones/resnet3d_csn.py
index 97c3e420aa..4539dec01e 100644
--- a/mmaction/models/backbones/resnet3d_csn.py
+++ b/mmaction/models/backbones/resnet3d_csn.py
@@ -2,7 +2,7 @@
 from mmcv.cnn import ConvModule
 from mmcv.utils import _BatchNorm
 
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 from .resnet3d import Bottleneck3d, ResNet3d
 
 
diff --git a/mmaction/models/backbones/resnet3d_slowfast.py b/mmaction/models/backbones/resnet3d_slowfast.py
index 3db7794dd5..45e9d5a7da 100644
--- a/mmaction/models/backbones/resnet3d_slowfast.py
+++ b/mmaction/models/backbones/resnet3d_slowfast.py
@@ -5,7 +5,7 @@
 from mmcv.utils import print_log
 
 from ...utils import get_root_logger
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 from .resnet3d import ResNet3d
 
 try:
diff --git a/mmaction/models/backbones/resnet3d_slowonly.py b/mmaction/models/backbones/resnet3d_slowonly.py
index e50aae7834..89275809e2 100644
--- a/mmaction/models/backbones/resnet3d_slowonly.py
+++ b/mmaction/models/backbones/resnet3d_slowonly.py
@@ -1,4 +1,4 @@
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 from .resnet3d_slowfast import ResNet3dPathway
 
 try:
diff --git a/mmaction/models/backbones/resnet_audio.py b/mmaction/models/backbones/resnet_audio.py
index ea5792d874..d4fd9e1ece 100644
--- a/mmaction/models/backbones/resnet_audio.py
+++ b/mmaction/models/backbones/resnet_audio.py
@@ -5,8 +5,8 @@
 from torch.nn.modules.batchnorm import _BatchNorm
 from torch.nn.modules.utils import _ntuple
 
-from mmaction.models.registry import BACKBONES
-from mmaction.utils import get_root_logger
+from ...utils import get_root_logger
+from ..builder import BACKBONES
 
 
 class Bottleneck2dAudio(nn.Module):
diff --git a/mmaction/models/backbones/resnet_tin.py b/mmaction/models/backbones/resnet_tin.py
index 1c650d3731..229c387cc3 100644
--- a/mmaction/models/backbones/resnet_tin.py
+++ b/mmaction/models/backbones/resnet_tin.py
@@ -2,7 +2,7 @@
 import torch.nn as nn
 
 from mmaction.utils import import_module_error_func
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 from .resnet_tsm import ResNetTSM
 
 try:
diff --git a/mmaction/models/backbones/resnet_tsm.py b/mmaction/models/backbones/resnet_tsm.py
index 2c4f999b5c..d1a383b33d 100644
--- a/mmaction/models/backbones/resnet_tsm.py
+++ b/mmaction/models/backbones/resnet_tsm.py
@@ -3,7 +3,7 @@
 from mmcv.cnn import NonLocal3d
 from torch.nn.modules.utils import _ntuple
 
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 from .resnet import ResNet
 
 
diff --git a/mmaction/models/backbones/tanet.py b/mmaction/models/backbones/tanet.py
index d66233931d..15d3487d1a 100644
--- a/mmaction/models/backbones/tanet.py
+++ b/mmaction/models/backbones/tanet.py
@@ -3,8 +3,8 @@
 import torch.nn as nn
 from torch.utils import checkpoint as cp
 
+from ..builder import BACKBONES
 from ..common import TAM
-from ..registry import BACKBONES
 from .resnet import Bottleneck, ResNet
 
 
diff --git a/mmaction/models/backbones/x3d.py b/mmaction/models/backbones/x3d.py
index 4d6b85cff3..4d8e39b641 100644
--- a/mmaction/models/backbones/x3d.py
+++ b/mmaction/models/backbones/x3d.py
@@ -8,7 +8,7 @@
 from mmcv.utils import _BatchNorm
 
 from ...utils import get_root_logger
-from ..registry import BACKBONES
+from ..builder import BACKBONES
 
 
 class SEModule(nn.Module):
diff --git a/mmaction/models/builder.py b/mmaction/models/builder.py
index 7fed5524c2..ef6792fac2 100644
--- a/mmaction/models/builder.py
+++ b/mmaction/models/builder.py
@@ -1,53 +1,37 @@
 import warnings
 
-import torch.nn as nn
-from mmcv.utils import Registry, build_from_cfg
+from mmcv.cnn import MODELS as MMCV_MODELS
+from mmcv.utils import Registry
 
 from mmaction.utils import import_module_error_func
-from .registry import BACKBONES, HEADS, LOCALIZERS, LOSSES, NECKS, RECOGNIZERS
+
+MODELS = Registry('models', parent=MMCV_MODELS)
+BACKBONES = MODELS
+NECKS = MODELS
+HEADS = MODELS
+RECOGNIZERS = MODELS
+LOSSES = MODELS
+LOCALIZERS = MODELS
 
 try:
     from mmdet.models.builder import DETECTORS, build_detector
 except (ImportError, ModuleNotFoundError):
     # Define an empty registry and building func, so that can import
-    DETECTORS = Registry('detector')
+    DETECTORS = MODELS
 
     @import_module_error_func('mmdet')
     def build_detector(cfg, train_cfg, test_cfg):
         pass
 
 
-def build(cfg, registry, default_args=None):
-    """Build a module.
-
-    Args:
-        cfg (dict, list[dict]): The config of modules, it is either a dict
-            or a list of configs.
-        registry (:obj:`Registry`): A registry the module belongs to.
-        default_args (dict, optional): Default arguments to build the module.
-            Defaults to None.
-
-    Returns:
-        nn.Module: A built nn module.
-    """
-
-    if isinstance(cfg, list):
-        modules = [
-            build_from_cfg(cfg_, registry, default_args) for cfg_ in cfg
-        ]
-        return nn.Sequential(*modules)
-
-    return build_from_cfg(cfg, registry, default_args)
-
-
 def build_backbone(cfg):
     """Build backbone."""
-    return build(cfg, BACKBONES)
+    return BACKBONES.build(cfg)
 
 
 def build_head(cfg):
     """Build head."""
-    return build(cfg, HEADS)
+    return HEADS.build(cfg)
 
 
 def build_recognizer(cfg, train_cfg=None, test_cfg=None):
@@ -58,22 +42,24 @@ def build_recognizer(cfg, train_cfg=None, test_cfg=None):
             'please specify them in model. Details see this '
             'PR: https://github.com/open-mmlab/mmaction2/pull/629',
             UserWarning)
-    assert cfg.get('train_cfg') is None or train_cfg is None, \
-        'train_cfg specified in both outer field and model field '
-    assert cfg.get('test_cfg') is None or test_cfg is None, \
-        'test_cfg specified in both outer field and model field '
-    return build(cfg, RECOGNIZERS,
-                 dict(train_cfg=train_cfg, test_cfg=test_cfg))
+    assert cfg.get(
+        'train_cfg'
+    ) is None or train_cfg is None, 'train_cfg specified in both outer field and model field'  # noqa: E501
+    assert cfg.get(
+        'test_cfg'
+    ) is None or test_cfg is None, 'test_cfg specified in both outer field and model field '  # noqa: E501
+    return RECOGNIZERS.build(
+        cfg, default_args=dict(train_cfg=train_cfg, test_cfg=test_cfg))
 
 
 def build_loss(cfg):
     """Build loss."""
-    return build(cfg, LOSSES)
+    return LOSSES.build(cfg)
 
 
 def build_localizer(cfg):
     """Build localizer."""
-    return build(cfg, LOCALIZERS)
+    return LOCALIZERS.build(cfg)
 
 
 def build_model(cfg, train_cfg=None, test_cfg=None):
@@ -102,4 +88,4 @@ def build_model(cfg, train_cfg=None, test_cfg=None):
 
 def build_neck(cfg):
     """Build neck."""
-    return build(cfg, NECKS)
+    return NECKS.build(cfg)
diff --git a/mmaction/models/heads/audio_tsn_head.py b/mmaction/models/heads/audio_tsn_head.py
index b956e1f21b..4fc0359216 100644
--- a/mmaction/models/heads/audio_tsn_head.py
+++ b/mmaction/models/heads/audio_tsn_head.py
@@ -1,7 +1,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .base import BaseHead
 
 
diff --git a/mmaction/models/heads/i3d_head.py b/mmaction/models/heads/i3d_head.py
index bfe423dad4..f86b978661 100644
--- a/mmaction/models/heads/i3d_head.py
+++ b/mmaction/models/heads/i3d_head.py
@@ -1,7 +1,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .base import BaseHead
 
 
diff --git a/mmaction/models/heads/slowfast_head.py b/mmaction/models/heads/slowfast_head.py
index 5b195aa5fb..f8cb7d6964 100644
--- a/mmaction/models/heads/slowfast_head.py
+++ b/mmaction/models/heads/slowfast_head.py
@@ -2,7 +2,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .base import BaseHead
 
 
diff --git a/mmaction/models/heads/ssn_head.py b/mmaction/models/heads/ssn_head.py
index 3399d3dc1a..d51f921ce5 100644
--- a/mmaction/models/heads/ssn_head.py
+++ b/mmaction/models/heads/ssn_head.py
@@ -2,7 +2,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 
 
 def parse_stage_config(stage_cfg):
diff --git a/mmaction/models/heads/tpn_head.py b/mmaction/models/heads/tpn_head.py
index 1cd6501a9c..34d476c144 100644
--- a/mmaction/models/heads/tpn_head.py
+++ b/mmaction/models/heads/tpn_head.py
@@ -1,6 +1,6 @@
 import torch.nn as nn
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .tsn_head import TSNHead
 
 
diff --git a/mmaction/models/heads/trn_head.py b/mmaction/models/heads/trn_head.py
index f93818a4bc..dbc080f0d8 100644
--- a/mmaction/models/heads/trn_head.py
+++ b/mmaction/models/heads/trn_head.py
@@ -5,7 +5,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .base import BaseHead
 
 
diff --git a/mmaction/models/heads/tsm_head.py b/mmaction/models/heads/tsm_head.py
index e989be034a..3d6a5f6e00 100644
--- a/mmaction/models/heads/tsm_head.py
+++ b/mmaction/models/heads/tsm_head.py
@@ -2,7 +2,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .base import AvgConsensus, BaseHead
 
 
diff --git a/mmaction/models/heads/tsn_head.py b/mmaction/models/heads/tsn_head.py
index f36fb6a364..998e9b7e61 100644
--- a/mmaction/models/heads/tsn_head.py
+++ b/mmaction/models/heads/tsn_head.py
@@ -1,7 +1,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .base import AvgConsensus, BaseHead
 
 
diff --git a/mmaction/models/heads/x3d_head.py b/mmaction/models/heads/x3d_head.py
index 2452df027e..816c45a2b9 100644
--- a/mmaction/models/heads/x3d_head.py
+++ b/mmaction/models/heads/x3d_head.py
@@ -1,7 +1,7 @@
 import torch.nn as nn
 from mmcv.cnn import normal_init
 
-from ..registry import HEADS
+from ..builder import HEADS
 from .base import BaseHead
 
 
diff --git a/mmaction/models/localizers/bmn.py b/mmaction/models/localizers/bmn.py
index f8a63c9d12..a0bbece0cd 100644
--- a/mmaction/models/localizers/bmn.py
+++ b/mmaction/models/localizers/bmn.py
@@ -5,8 +5,7 @@
 import torch.nn as nn
 
 from ...localization import temporal_iop, temporal_iou
-from ..builder import build_loss
-from ..registry import LOCALIZERS
+from ..builder import LOCALIZERS, build_loss
 from .base import BaseLocalizer
 from .utils import post_processing
 
diff --git a/mmaction/models/localizers/bsn.py b/mmaction/models/localizers/bsn.py
index 23f3e7230a..83843002ff 100644
--- a/mmaction/models/localizers/bsn.py
+++ b/mmaction/models/localizers/bsn.py
@@ -4,8 +4,7 @@
 import torch.nn.functional as F
 
 from ...localization import temporal_iop
-from ..builder import build_loss
-from ..registry import LOCALIZERS
+from ..builder import LOCALIZERS, build_loss
 from .base import BaseLocalizer
 from .utils import post_processing
 
diff --git a/mmaction/models/localizers/ssn.py b/mmaction/models/localizers/ssn.py
index 2ab9973815..1284f694dd 100644
--- a/mmaction/models/localizers/ssn.py
+++ b/mmaction/models/localizers/ssn.py
@@ -2,7 +2,7 @@
 import torch.nn as nn
 
 from .. import builder
-from ..registry import LOCALIZERS
+from ..builder import LOCALIZERS
 from .base import BaseLocalizer
 
 
diff --git a/mmaction/models/losses/binary_logistic_regression_loss.py b/mmaction/models/losses/binary_logistic_regression_loss.py
index 343d1ca06e..ab23651cfa 100644
--- a/mmaction/models/losses/binary_logistic_regression_loss.py
+++ b/mmaction/models/losses/binary_logistic_regression_loss.py
@@ -1,7 +1,7 @@
 import torch
 import torch.nn as nn
 
-from ..registry import LOSSES
+from ..builder import LOSSES
 
 
 def binary_logistic_regression_loss(reg_score,
diff --git a/mmaction/models/losses/bmn_loss.py b/mmaction/models/losses/bmn_loss.py
index 50e49729b0..9ba312cad9 100644
--- a/mmaction/models/losses/bmn_loss.py
+++ b/mmaction/models/losses/bmn_loss.py
@@ -2,7 +2,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 
-from ..registry import LOSSES
+from ..builder import LOSSES
 from .binary_logistic_regression_loss import binary_logistic_regression_loss
 
 
diff --git a/mmaction/models/losses/cross_entropy_loss.py b/mmaction/models/losses/cross_entropy_loss.py
index bd192532a4..836b950c66 100644
--- a/mmaction/models/losses/cross_entropy_loss.py
+++ b/mmaction/models/losses/cross_entropy_loss.py
@@ -1,7 +1,7 @@
 import torch
 import torch.nn.functional as F
 
-from ..registry import LOSSES
+from ..builder import LOSSES
 from .base import BaseWeightedLoss
 
 
diff --git a/mmaction/models/losses/hvu_loss.py b/mmaction/models/losses/hvu_loss.py
index 9d9b00567d..9fdbbfd45b 100644
--- a/mmaction/models/losses/hvu_loss.py
+++ b/mmaction/models/losses/hvu_loss.py
@@ -1,7 +1,7 @@
 import torch
 import torch.nn.functional as F
 
-from ..registry import LOSSES
+from ..builder import LOSSES
 from .base import BaseWeightedLoss
 
 
diff --git a/mmaction/models/losses/nll_loss.py b/mmaction/models/losses/nll_loss.py
index 1c28537329..7bd57a50db 100644
--- a/mmaction/models/losses/nll_loss.py
+++ b/mmaction/models/losses/nll_loss.py
@@ -1,6 +1,6 @@
 import torch.nn.functional as F
 
-from ..registry import LOSSES
+from ..builder import LOSSES
 from .base import BaseWeightedLoss
 
 
diff --git a/mmaction/models/losses/ssn_loss.py b/mmaction/models/losses/ssn_loss.py
index 492dc8ddaa..030ab3cd0e 100644
--- a/mmaction/models/losses/ssn_loss.py
+++ b/mmaction/models/losses/ssn_loss.py
@@ -2,7 +2,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 
-from ..registry import LOSSES
+from ..builder import LOSSES
 from .ohem_hinge_loss import OHEMHingeLoss
 
 
diff --git a/mmaction/models/necks/tpn.py b/mmaction/models/necks/tpn.py
index 250357c675..7264b3c366 100644
--- a/mmaction/models/necks/tpn.py
+++ b/mmaction/models/necks/tpn.py
@@ -3,8 +3,7 @@
 import torch.nn as nn
 from mmcv.cnn import ConvModule, constant_init, normal_init, xavier_init
 
-from ..builder import build_loss
-from ..registry import NECKS
+from ..builder import NECKS, build_loss
 
 
 class Identity(nn.Module):
diff --git a/mmaction/models/recognizers/audio_recognizer.py b/mmaction/models/recognizers/audio_recognizer.py
index a9b431c06a..b17e44680a 100644
--- a/mmaction/models/recognizers/audio_recognizer.py
+++ b/mmaction/models/recognizers/audio_recognizer.py
@@ -1,4 +1,4 @@
-from ..registry import RECOGNIZERS
+from ..builder import RECOGNIZERS
 from .base import BaseRecognizer
 
 
diff --git a/mmaction/models/recognizers/base.py b/mmaction/models/recognizers/base.py
index 7732ed0766..281aa547e1 100644
--- a/mmaction/models/recognizers/base.py
+++ b/mmaction/models/recognizers/base.py
@@ -84,7 +84,7 @@ def __init__(self,
         self.blending = None
         if train_cfg is not None and 'blending' in train_cfg:
             from mmcv.utils import build_from_cfg
-            from ...datasets.registry import BLENDINGS
+            from mmaction.datasets.builder import BLENDINGS
             self.blending = build_from_cfg(train_cfg['blending'], BLENDINGS)
 
         self.init_weights()
diff --git a/mmaction/models/recognizers/recognizer2d.py b/mmaction/models/recognizers/recognizer2d.py
index bda7db2312..16f6349be8 100644
--- a/mmaction/models/recognizers/recognizer2d.py
+++ b/mmaction/models/recognizers/recognizer2d.py
@@ -1,7 +1,7 @@
 import torch
 from torch import nn
 
-from ..registry import RECOGNIZERS
+from ..builder import RECOGNIZERS
 from .base import BaseRecognizer
 
 
diff --git a/mmaction/models/recognizers/recognizer3d.py b/mmaction/models/recognizers/recognizer3d.py
index a4b420eee6..26e4668147 100644
--- a/mmaction/models/recognizers/recognizer3d.py
+++ b/mmaction/models/recognizers/recognizer3d.py
@@ -1,7 +1,7 @@
 import torch
 from torch import nn
 
-from ..registry import RECOGNIZERS
+from ..builder import RECOGNIZERS
 from .base import BaseRecognizer
 
 
diff --git a/mmaction/models/registry.py b/mmaction/models/registry.py
deleted file mode 100644
index 61dc40c331..0000000000
--- a/mmaction/models/registry.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from mmcv.utils import Registry
-
-BACKBONES = Registry('backbone')
-NECKS = Registry('neck')
-HEADS = Registry('head')
-RECOGNIZERS = Registry('recognizer')
-LOSSES = Registry('loss')
-LOCALIZERS = Registry('localizer')
diff --git a/tests/test_runtime/test_train.py b/tests/test_runtime/test_train.py
index 6490cd454d..28d16cc1ed 100644
--- a/tests/test_runtime/test_train.py
+++ b/tests/test_runtime/test_train.py
@@ -9,7 +9,7 @@
 from torch.utils.data import Dataset
 
 from mmaction.apis import train_model
-from mmaction.datasets.registry import DATASETS
+from mmaction.datasets import DATASETS
 
 
 @DATASETS.register_module()