From 8fdae9ac2f33f8051379e4e315b355789cc75530 Mon Sep 17 00:00:00 2001 From: fangyixiao18 Date: Thu, 8 Dec 2022 14:41:56 +0800 Subject: [PATCH 1/5] refactor projects folder --- projects/example_project/README.md | 121 ++++++++++++++++-- ...vit-base-p16_8xb512-amp-coslr-300e_in1k.py | 4 +- .../example_project/{ => models}/__init__.py | 0 .../example_project/{ => models}/dummy_mae.py | 0 4 files changed, 112 insertions(+), 13 deletions(-) rename projects/example_project/{ => models}/__init__.py (100%) rename projects/example_project/{ => models}/dummy_mae.py (100%) diff --git a/projects/example_project/README.md b/projects/example_project/README.md index e9a1c1095..61f9d6e10 100644 --- a/projects/example_project/README.md +++ b/projects/example_project/README.md @@ -5,8 +5,13 @@ This is an example README for community `projects/`. We have provided detailed e - [Dummy MAE Wrapper](#dummy-mae-wrapper) - [Description](#description) - [Usage](#usage) - - [Pre-training commands](#pre-training-commands) - - [Downstream tasks commands](#downstream-tasks-commands) + - [Setup Environment](#setup-environment) + - [Data Preparation](#data-preparation) + - [Pre-training Commands](#pre-training-commands) + - [On Local Single GPU](#on-local-single-gpu) + - [On Multiple GPUs](#on-multiple-gpus) + - [On Multiple GPUs with Slurm](#on-multiple-gpus-with-slurm) + - [Downstream Tasks Commands](#downstream-tasks-commands) - [Results](#results) - [Citation](#citation) - [Checklist](#checklist) @@ -21,27 +26,121 @@ This project implements a dummy MAE wrapper, which prints "Welcome to MMSelfSup" ## Usage - + -### Pre-training commands -In MMSelfSup's root directory, run the following command to train the model: +### Setup Environment + +It requires [PyTorch](https://pytorch.org/get-started/locally/) and the following OpenMMLab packages: + +- [MIM](https://github.com/open-mmlab/mim): A command-line tool to manage OpenMMLab packages and experiments. +- [MMEngine](https://github.com/open-mmlab/mmengine): OpenMMLab foundational library for training deep learning models. +- [MMCV](https://github.com/open-mmlab/mmcv): OpenMMLab foundational library for computer vision. +- [MMClassification](https://github.com/open-mmlab/mmclassification): OpenMMLab image classification toolbox and benchmark. Besides classification, it's also a repository to store various backbones. + +Assume you have prepared your Python and PyTorch environment, just use the following command to setup the environment. ```bash -python tools/train.py projects/example_project/configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py +pip install openmim +mim install mmengine "mmcv>=2.0.0rc1" +mim install mmselfsup ``` -### Downstream tasks commands +### Data Preparation -In MMSelfSup's root directory, run the following command to train the downstream model: +To show the dataset directory or provide the commands for dataset preparation if needed. + +For example: + +```text +data/ +└── imagenet + ├── train + ├── val + └── meta + ├── train.txt + └── val.txt +``` + +### Pre-training Commands + +At first, you need to add the current folder the the `PYTHONPATH`, so that Python can find your model files. In `example_project/` root directory, please ;run command below to add it. + +```shell +export PYTHONPATH=`pwd`:$PYTHONPATH +``` + +Then run the following commands to train the model: + +#### On Local Single GPU + +```bash +mim train mmselfsup $CONFIG --work-dir $WORK_DIR + +# a specific command example +mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py \ + --work-dir work_dirs/dummy_mae/ +``` + +#### On Multiple GPUs ```bash -sh tools/benchmarks/classification/mim_dist_train.sh ${CONFIGS} ${CHECKPOINT} [optional args] +# a specific command examples, 8 GPUs here +mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py \ + --work-dir work_dirs/dummy_mae/ \ + --launcher pytorch --gpus 8 +``` -# the example of custom command -GPUS=1 sh tools/benchmarks/classification/mim_dist_train.sh projects/example_projects/configs/xxx.py ${CHECKPOINT} --work-dir work_dirs/example_projects/classification/ +Note: +- CONFIG: the config files under the directory `configs/` +- WORK_DIR: the working directory to save configs, logs, and checkpoints + +#### On Multiple GPUs with Slurm + +```bash +# specific command examples +mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py \ + --work-dir work_dirs/dummy_mae/ \ + --launcher slurm --gpus 8 \ + --partition $PARTITION + +mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py \ + --work-dir work_dirs/dummy_mae/ \ + --launcher slurm --gpus 16 --gpus-per-node 8 \ + --partition $PARTITION ``` +Note: +- CONFIG: the config files under the directory `configs/` +- WORK_DIR: the working directory to save configs, logs, and checkpoints +- PARTITION: the slurm partition you are using + +### Downstream Tasks Commands + +In MMSelfSup's root directory, run the following command to train the downstream model: + +```bash +mim train mmcls $CONFIG \ + --work-dir $WORK_DIR \ + --launcher pytorch -gpus 8 \ + [optional args] + +# a specific command example +mim train mmcls configs/xxx.py \ + --work-dir work_dirs/dummy_mae/classification/ + --launcher pytorch -gpus 8 \ + --cfg-options model.backbone.init_cfg.type=Pretrained \ + model.backbone.init_cfg.checkpoint=$CHECKPOINT \ + model.backbone.init_cfg.prefix="backbone." \ + $PY_ARGS +``` + +Note: +- CONFIG: the config files under the directory `configs/` +- WORK_DIR: the working directory to save configs, logs, and checkpoints +- CHECKPOINT: the pretrained checkpoint of MMSelfSup saved in working directory, like `$WORK_DIR/epoch_300.pth` +- PY_ARGS: other optional args + ## Results - ### Setup Environment It requires [PyTorch](https://pytorch.org/get-started/locally/) and the following OpenMMLab packages: @@ -67,7 +66,7 @@ data/ At first, you need to add the current folder the the `PYTHONPATH`, so that Python can find your model files. In `example_project/` root directory, please ;run command below to add it. ```shell -export PYTHONPATH=`pwd`:$PYTHONPATH +export PYTHONPATH=`pwd`:$PYTHONPATH ``` Then run the following commands to train the model: @@ -92,6 +91,7 @@ mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py ``` Note: + - CONFIG: the config files under the directory `configs/` - WORK_DIR: the working directory to save configs, logs, and checkpoints @@ -111,6 +111,7 @@ mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py ``` Note: + - CONFIG: the config files under the directory `configs/` - WORK_DIR: the working directory to save configs, logs, and checkpoints - PARTITION: the slurm partition you are using @@ -136,6 +137,7 @@ mim train mmcls configs/xxx.py \ ``` Note: + - CONFIG: the config files under the directory `configs/` - WORK_DIR: the working directory to save configs, logs, and checkpoints - CHECKPOINT: the pretrained checkpoint of MMSelfSup saved in working directory, like `$WORK_DIR/epoch_300.pth` From 5cf66a0ad5f3eb7d2311ebac4c7bfed2f1474ef4 Mon Sep 17 00:00:00 2001 From: fangyixiao18 Date: Thu, 8 Dec 2022 17:05:39 +0800 Subject: [PATCH 3/5] fix typo --- projects/example_project/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/example_project/README.md b/projects/example_project/README.md index 4038e11ab..65bcb1f92 100644 --- a/projects/example_project/README.md +++ b/projects/example_project/README.md @@ -63,7 +63,7 @@ data/ ### Pre-training Commands -At first, you need to add the current folder the the `PYTHONPATH`, so that Python can find your model files. In `example_project/` root directory, please ;run command below to add it. +At first, you need to add the current folder to `PYTHONPATH`, so that Python can find your model files. In `example_project/` root directory, please run command below to add it. ```shell export PYTHONPATH=`pwd`:$PYTHONPATH From d72fb91aedf455dfdb943418cde7fe8961018cc8 Mon Sep 17 00:00:00 2001 From: fangyixiao18 Date: Thu, 8 Dec 2022 19:46:44 +0800 Subject: [PATCH 4/5] refine readme template --- projects/example_project/README.md | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/projects/example_project/README.md b/projects/example_project/README.md index 65bcb1f92..1069bbbd0 100644 --- a/projects/example_project/README.md +++ b/projects/example_project/README.md @@ -30,20 +30,7 @@ This project implements a dummy MAE wrapper, which prints "Welcome to MMSelfSup" ### Setup Environment -It requires [PyTorch](https://pytorch.org/get-started/locally/) and the following OpenMMLab packages: - -- [MIM](https://github.com/open-mmlab/mim): A command-line tool to manage OpenMMLab packages and experiments. -- [MMEngine](https://github.com/open-mmlab/mmengine): OpenMMLab foundational library for training deep learning models. -- [MMCV](https://github.com/open-mmlab/mmcv): OpenMMLab foundational library for computer vision. -- [MMClassification](https://github.com/open-mmlab/mmclassification): OpenMMLab image classification toolbox and benchmark. Besides classification, it's also a repository to store various backbones. - -Assume you have prepared your Python and PyTorch environment, just use the following command to setup the environment. - -```bash -pip install openmim -mim install mmengine "mmcv>=2.0.0rc1" -mim install mmselfsup -``` +Please refer to [Get Started](https://mmselfsup.readthedocs.io/en/1.x/get_started.html) documentation of MMSelfSup. ### Data Preparation @@ -98,12 +85,7 @@ Note: #### On Multiple GPUs with Slurm ```bash -# specific command examples -mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py \ - --work-dir work_dirs/dummy_mae/ \ - --launcher slurm --gpus 8 \ - --partition $PARTITION - +# a specific command example: 16 GPUs in 2 nodes mim train mmselfsup configs/dummy-mae_vit-base-p16_8xb512-amp-coslr-300e_in1k.py \ --work-dir work_dirs/dummy_mae/ \ --launcher slurm --gpus 16 --gpus-per-node 8 \ @@ -192,8 +174,12 @@ The Linear Eval and Fine-tuning results are based on ImageNet dataset. ## Checklist - @@ -239,4 +225,4 @@ A project does not necessarily have to be finished in a single PR, but it's esse - - [ ] Refactor and Move your modules into the core package following the codebase's file hierarchy structure. +- [ ] Refactor and Move your modules into the core package following the codebase's file hierarchy structure. From f658b4f640e205a0f8c0709560712e2e18bdd670 Mon Sep 17 00:00:00 2001 From: fangyixiao18 Date: Thu, 8 Dec 2022 20:34:39 +0800 Subject: [PATCH 5/5] update downsteam --- projects/example_project/README.md | 4 +- .../vit-base-p16_ft-8xb128-coslr-100e_in1k.py | 124 ++++++++++++++++++ 2 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 projects/example_project/configs/vit-base-p16_ft-8xb128-coslr-100e_in1k.py diff --git a/projects/example_project/README.md b/projects/example_project/README.md index 1069bbbd0..32d3b9008 100644 --- a/projects/example_project/README.md +++ b/projects/example_project/README.md @@ -109,11 +109,11 @@ mim train mmcls $CONFIG \ [optional args] # a specific command example -mim train mmcls configs/xxx.py \ +mim train mmcls configs/vit-base-p16_ft-8xb128-coslr-100e_in1k.py \ --work-dir work_dirs/dummy_mae/classification/ --launcher pytorch -gpus 8 \ --cfg-options model.backbone.init_cfg.type=Pretrained \ - model.backbone.init_cfg.checkpoint=$CHECKPOINT \ + model.backbone.init_cfg.checkpoint=https://download.openmmlab.com/mmselfsup/1.x/mae/mae_vit-base-p16_8xb512-fp16-coslr-300e_in1k/mae_vit-base-p16_8xb512-coslr-300e-fp16_in1k_20220829-c2cf66ba.pth \ model.backbone.init_cfg.prefix="backbone." \ $PY_ARGS ``` diff --git a/projects/example_project/configs/vit-base-p16_ft-8xb128-coslr-100e_in1k.py b/projects/example_project/configs/vit-base-p16_ft-8xb128-coslr-100e_in1k.py new file mode 100644 index 000000000..9b8d80620 --- /dev/null +++ b/projects/example_project/configs/vit-base-p16_ft-8xb128-coslr-100e_in1k.py @@ -0,0 +1,124 @@ +# mmcls:: means we use the default settings from MMClassification +_base_ = [ + 'mmcls::_base_/datasets/imagenet_bs64_swin_224.py', + 'mmcls::_base_/schedules/imagenet_bs1024_adamw_swin.py', + 'mmcls::_base_/default_runtime.py' +] +# MAE fine-tuning setting + +# model settings +model = dict( + type='ImageClassifier', + backbone=dict( + type='VisionTransformer', + arch='base', + img_size=224, + patch_size=16, + drop_path_rate=0.1, + avg_token=True, + output_cls_token=False, + final_norm=False, + init_cfg=dict(type='Pretrained', checkpoint='')), + neck=None, + head=dict( + type='LinearClsHead', + num_classes=1000, + in_channels=768, + loss=dict( + type='LabelSmoothLoss', label_smooth_val=0.1, mode='original'), + init_cfg=[dict(type='TruncNormal', layer='Linear', std=2e-5)]), + train_cfg=dict(augments=[ + dict(type='Mixup', alpha=0.8), + dict(type='CutMix', alpha=1.0) + ])) + +file_client_args = dict(backend='disk') + +train_pipeline = [ + dict(type='LoadImageFromFile', file_client_args=file_client_args), + dict( + type='RandomResizedCrop', + scale=224, + backend='pillow', + interpolation='bicubic'), + dict(type='RandomFlip', prob=0.5, direction='horizontal'), + dict( + type='RandAugment', + policies='timm_increasing', + num_policies=2, + total_level=10, + magnitude_level=9, + magnitude_std=0.5, + hparams=dict(pad_val=[104, 116, 124], interpolation='bicubic')), + dict( + type='RandomErasing', + erase_prob=0.25, + mode='rand', + min_area_ratio=0.02, + max_area_ratio=0.3333333333333333, + fill_color=[103.53, 116.28, 123.675], + fill_std=[57.375, 57.12, 58.395]), + dict(type='PackClsInputs') +] +test_pipeline = [ + dict(type='LoadImageFromFile', file_client_args=file_client_args), + dict( + type='ResizeEdge', + scale=256, + edge='short', + backend='pillow', + interpolation='bicubic'), + dict(type='CenterCrop', crop_size=224), + dict(type='PackClsInputs') +] + +train_dataloader = dict(batch_size=128, dataset=dict(pipeline=train_pipeline)) +val_dataloader = dict(batch_size=128, dataset=dict(pipeline=test_pipeline)) +test_dataloader = val_dataloader + +# optimizer wrapper +optim_wrapper = dict( + optimizer=dict( + type='AdamW', + lr=2e-3, + weight_decay=0.05, + eps=1e-8, + betas=(0.9, 0.999), + model_type='vit', # layer-wise lr decay type + layer_decay_rate=0.65), # layer-wise lr decay factor + constructor='mmselfsup.LearningRateDecayOptimWrapperConstructor', + paramwise_cfg=dict( + custom_keys={ + '.ln': dict(decay_mult=0.0), + '.bias': dict(decay_mult=0.0), + '.cls_token': dict(decay_mult=0.0), + '.pos_embed': dict(decay_mult=0.0) + })) + +# learning rate scheduler +param_scheduler = [ + dict( + type='LinearLR', + start_factor=1e-4, + by_epoch=True, + begin=0, + end=5, + convert_to_iter_based=True), + dict( + type='CosineAnnealingLR', + T_max=95, + by_epoch=True, + begin=5, + end=100, + eta_min=1e-6, + convert_to_iter_based=True) +] + +# runtime settings +default_hooks = dict( + # save checkpoint per epoch. + checkpoint=dict(type='CheckpointHook', interval=1, max_keep_ckpts=3)) + +train_cfg = dict(by_epoch=True, max_epochs=100) + +randomness = dict(seed=0, diff_rank_seed=True)