Skip to content

Commit

Permalink
Merge pull request #6 from Megvii-BaseDetection/add-exps
Browse files Browse the repository at this point in the history 
[Enhance]: Add exps
  • Loading branch information
@yinchimaoliang
yinchimaoliang authored Sep 5, 2022
2 parents 67b65d0 + a814e10 commit d6d6555
Show file tree
Hide file tree
Showing 7 changed files with 426 additions and 41 deletions.
11 changes: 8 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,14 @@ python [EXP_PATH] --amp_backend native -b 8 --gpus 8
python [EXP_PATH] --ckpt_path [CKPT_PATH] -e -b 8 --gpus 8
```
### Benchmark
|Backbone |mAP |mATE| mASE | mAOE |mAVE| mAAE | NDS | weights |
| ------ | :---: |:---: |:---: | :---: | :----: | :----: | :----: | :----: |
|[R50](exps/bev_stereo_lss_r50_256x704_128x128_24e_2key.py)| 0.3494 | 0.6671 |0.2785 | 0.5606 | 0.4686 | 0.2295 | 0.4543 | [github](https://github.com/Megvii-BaseDetection/BEVStereo/releases/download/v0.0.1/bev_stereo_lss_r50_256x704_128x128_24e_2key.pth)
|Exp| Frames |EMA| CBGS |mAP |mATE| mASE | mAOE |mAVE| mAAE | NDS | weights |
| ------ | :---: | :---: | :---: | :---: |:---: |:---: | :---: | :----: | :----: | :----: | :----: |
|[R50](exps/bev_stereo_lss_r50_256x704_128x128_24e_key4.py)|key + sweep4| | |0.3427| 0.6560 | 0.2784| 0.5982| 0.5347| 0.2228| 0.4423|[github](https://github.com/Megvii-BaseDetection/BEVStereo/releases/download/v0.0.2/bev_stereo_lss_r50_256x704_128x128_24e_key4.pth)|
|[R50](exps/bev_stereo_lss_r50_256x704_128x128_24e_key4.py)|key + sweep4|√| |0.3435|0.6585|0.2757|0.5792|0.5034|0.2163|0.4485|[github](https://github.com/Megvii-BaseDetection/BEVStereo/releases/download/v0.0.2/bev_stereo_lss_r50_256x704_128x128_24e_key4_ema.pth)|
|[R50](exps/bev_stereo_lss_r50_256x704_128x128_24e_2key.py)|key + key|| | 0.3456 | 0.6589 | 0.2774 | 0.5500 | 0.4980 | 0.2278 | 0.4516 | [github](https://github.com/Megvii-BaseDetection/BEVStereo/releases/download/v0.0.2/bev_stereo_lss_r50_256x704_128x128_24e_2key.pth)|
|[R50](exps/bev_stereo_lss_r50_256x704_128x128_24e_2key_ema.py)|key + key|√| | 0.3494 | 0.6671 |0.2785 | 0.5606 | 0.4686 | 0.2295 | 0.4543 | [github](https://github.com/Megvii-BaseDetection/BEVStereo/releases/download/v0.0.2/bev_stereo_lss_r50_256x704_128x128_24e_2key_ema.pth)|
|[R50](exps/bev_stereo_lss_r50_256x704_128x128_20e_cbgs_2key_da.py)|key + key| |√| 0.3576 | 0.6071 | 0.2684 | 0.4157 | 0.3928 | 0.2021 | 0.4902 | [github](https://github.com/Megvii-BaseDetection/BEVStereo/releases/download/v0.0.2/bev_stereo_lss_r50_256x704_128x128_20e_cbgs_2key_da.pth)|
|[R50](exps/bev_stereo_lss_r50_256x704_128x128_20e_cbgs_2key_da_ema.py)|key + key| |√| 0.3721 | 0.5980 | 0.2701 | 0.4381 | 0.3672 | 0.1898 | 0.4997 | [github](https://github.com/Megvii-BaseDetection/BEVStereo/releases/download/v0.0.2/bev_stereo_lss_r50_256x704_128x128_20e_cbgs_2key_da_ema.pth)|

## Acknowledgments
This project exists thanks to all the people who instruct.
Expand Down
41 changes: 32 additions & 9 deletions exps/bev_stereo_lss_r50_256x704_128x128_20e_cbgs_2key_da.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,37 @@
# Copyright (c) Megvii Inc. All rights reserved.
"""
mAP: 0.3576
mATE: 0.6071
mASE: 0.2684
mAOE: 0.4157
mAVE: 0.3928
mAAE: 0.2021
NDS: 0.4902
Eval time: 129.7s
Per-class results:
Object Class AP ATE ASE AOE AVE AAE
car 0.559 0.465 0.157 0.110 0.350 0.205
truck 0.285 0.633 0.205 0.101 0.304 0.209
bus 0.373 0.667 0.204 0.076 0.896 0.345
trailer 0.167 0.956 0.228 0.482 0.289 0.100
construction_vehicle 0.077 0.869 0.454 1.024 0.108 0.335
pedestrian 0.402 0.652 0.299 0.821 0.493 0.253
motorcycle 0.321 0.544 0.255 0.484 0.529 0.159
bicycle 0.276 0.466 0.272 0.522 0.173 0.011
traffic_cone 0.551 0.432 0.321 nan nan nan
barrier 0.565 0.386 0.287 0.121 nan nan
"""
from argparse import ArgumentParser, Namespace

import pytorch_lightning as pl
import torch
import torch.nn as nn
from torch.cuda.amp.autocast_mode import autocast
from torch.optim.lr_scheduler import MultiStepLR

from callbacks.ema import EMACallback
from exps.bev_stereo_lss_r50_256x704_128x128_24e_2key import \
BEVDepthLightningModel as BaseBEVDepthLightningModel
BEVStereoLightningModel as BEVStereoLightningModel
from layers.backbones.lss_fpn import LSSFPN as BaseLSSFPN
from layers.heads.bev_stereo_head import BEVStereoHead
from models.bev_stereo import BEVStereo as BaseBEVStereo
Expand Down Expand Up @@ -100,31 +123,31 @@ def __init__(self, backbone_conf, head_conf, is_train_depth=True):
self.is_train_depth = is_train_depth


class BEVStereoLightningModel(BaseBEVDepthLightningModel):
class BEVStereoLightningModel(BEVStereoLightningModel):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.model = BEVStereo(self.backbone_conf,
self.head_conf,
is_train_depth=True)
self.data_use_cbgs = True
self.basic_lr_per_img = 2e-4 / 32

def configure_optimizers(self):
lr = self.basic_lr_per_img * \
self.batch_size_per_device * self.gpus
optimizer = torch.optim.AdamW(self.model.parameters(),
lr=lr,
weight_decay=1e-2)
return [optimizer]
scheduler = MultiStepLR(optimizer, [16, 19])
return [[optimizer], [scheduler]]


def main(args: Namespace) -> None:
if args.seed is not None:
pl.seed_everything(args.seed)

model = BEVStereoLightningModel(**vars(args))
train_dataloader = model.train_dataloader()
ema_callback = EMACallback(len(train_dataloader.dataset) * args.max_epochs)
trainer = pl.Trainer.from_argparse_args(args, callbacks=[ema_callback])
trainer = pl.Trainer.from_argparse_args(args)
if args.evaluate:
trainer.test(model, ckpt_path=args.ckpt_path)
else:
Expand Down Expand Up @@ -153,9 +176,9 @@ def run_cli():
num_sanity_val_steps=0,
gradient_clip_val=5,
limit_val_batches=0,
enable_checkpointing=False,
enable_checkpointing=True,
precision=16,
default_root_dir='./outputs/bev_depth_lss_r50_'
default_root_dir='./outputs/bev_stereo_lss_r50_'
'256x704_128x128_20e_cbgs_2key_da')
args = parser.parse_args()
main(args)
Expand Down
90 changes: 90 additions & 0 deletions exps/bev_stereo_lss_r50_256x704_128x128_20e_cbgs_2key_da_ema.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Copyright (c) Megvii Inc. All rights reserved.
"""
mAP: 0.3721
mATE: 0.5980
mASE: 0.2701
mAOE: 0.4381
mAVE: 0.3672
mAAE: 0.1898
NDS: 0.4997
Eval time: 138.0s
Per-class results:
Object Class AP ATE ASE AOE AVE AAE
car 0.567 0.457 0.156 0.104 0.343 0.204
truck 0.299 0.650 0.205 0.103 0.321 0.197
bus 0.394 0.613 0.203 0.106 0.643 0.252
trailer 0.178 0.991 0.239 0.433 0.345 0.070
construction_vehicle 0.102 0.826 0.458 1.055 0.114 0.372
pedestrian 0.402 0.653 0.297 0.803 0.479 0.249
motorcycle 0.356 0.553 0.251 0.450 0.512 0.168
bicycle 0.311 0.440 0.265 0.779 0.180 0.006
traffic_cone 0.552 0.420 0.336 nan nan nan
barrier 0.561 0.377 0.291 0.111 nan nan
"""
from argparse import ArgumentParser, Namespace

import pytorch_lightning as pl
import torch

from callbacks.ema import EMACallback
from exps.bev_stereo_lss_r50_256x704_128x128_20e_cbgs_2key_da import \
BEVStereoLightningModel as BaseBEVStereoLightningModel


class BEVStereoLightningModel(BaseBEVStereoLightningModel):
def configure_optimizers(self):
lr = self.basic_lr_per_img * \
self.batch_size_per_device * self.gpus
optimizer = torch.optim.AdamW(self.model.parameters(),
lr=lr,
weight_decay=1e-2)
return [optimizer]


def main(args: Namespace) -> None:
if args.seed is not None:
pl.seed_everything(args.seed)

model = BEVStereoLightningModel(**vars(args))
train_dataloader = model.train_dataloader()
ema_callback = EMACallback(len(train_dataloader.dataset) * args.max_epochs)
trainer = pl.Trainer.from_argparse_args(args, callbacks=[ema_callback])
if args.evaluate:
trainer.test(model, ckpt_path=args.ckpt_path)
else:
trainer.fit(model)


def run_cli():
parent_parser = ArgumentParser(add_help=False)
parent_parser = pl.Trainer.add_argparse_args(parent_parser)
parent_parser.add_argument('-e',
'--evaluate',
dest='evaluate',
action='store_true',
help='evaluate model on validation set')
parent_parser.add_argument('-b', '--batch_size_per_device', type=int)
parent_parser.add_argument('--seed',
type=int,
default=0,
help='seed for initializing training.')
parent_parser.add_argument('--ckpt_path', type=str)
parser = BEVStereoLightningModel.add_model_specific_args(parent_parser)
parser.set_defaults(profiler='simple',
deterministic=False,
max_epochs=20,
accelerator='ddp',
num_sanity_val_steps=0,
gradient_clip_val=5,
limit_val_batches=0,
enable_checkpointing=False,
precision=16,
default_root_dir='./outputs/bev_stereo_lss_r50_256x704'
'_128x128_20e_cbgs_2key_da_ema')
args = parser.parse_args()
main(args)


if __name__ == '__main__':
run_cli()
57 changes: 28 additions & 29 deletions exps/bev_stereo_lss_r50_256x704_128x128_24e_2key.py
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
# Copyright (c) Megvii Inc. All rights reserved.
"""
mAP: 0.3494
mATE: 0.6671
mASE: 0.2785
mAOE: 0.5606
mAVE: 0.4686
mAAE: 0.2295
NDS: 0.4543
Eval time: 204.2s
mAP: 0.3456
mATE: 0.6589
mASE: 0.2774
mAOE: 0.5500
mAVE: 0.4980
mAAE: 0.2278
NDS: 0.4516
Eval time: 158.2s
Per-class results:
Object Class AP ATE ASE AOE AVE AAE
car 0.509 0.522 0.163 0.187 0.507 0.228
truck 0.287 0.694 0.213 0.202 0.449 0.229
bus 0.390 0.682 0.207 0.152 0.903 0.260
trailer 0.168 0.943 0.248 0.491 0.340 0.186
construction_vehicle 0.087 1.054 0.515 1.200 0.104 0.377
pedestrian 0.351 0.730 0.299 0.987 0.575 0.320
motorcycle 0.368 0.583 0.262 0.722 0.661 0.227
bicycle 0.338 0.494 0.258 0.920 0.209 0.008
traffic_cone 0.494 0.502 0.341 nan nan nan
barrier 0.502 0.467 0.278 0.185 nan nan
car 0.510 0.525 0.165 0.188 0.510 0.226
truck 0.288 0.698 0.220 0.205 0.443 0.227
bus 0.378 0.622 0.210 0.135 0.896 0.289
trailer 0.156 1.003 0.219 0.482 0.609 0.179
construction_vehicle 0.094 0.929 0.502 1.209 0.108 0.365
pedestrian 0.356 0.728 0.297 1.005 0.579 0.319
motorcycle 0.361 0.571 0.258 0.734 0.631 0.211
bicycle 0.318 0.533 0.269 0.793 0.208 0.007
traffic_cone 0.488 0.501 0.355 nan nan nan
barrier 0.506 0.478 0.277 0.200 nan nan
"""
from argparse import ArgumentParser, Namespace

Expand All @@ -34,8 +34,8 @@
import torchvision.models as models
from pytorch_lightning.core import LightningModule
from torch.cuda.amp.autocast_mode import autocast
from torch.optim.lr_scheduler import MultiStepLR

from callbacks.ema import EMACallback
from dataset.nusc_mv_det_dataset import NuscMVDetDataset, collate_fn
from evaluators.det_mv_evaluators import DetMVNuscEvaluator
from models.bev_stereo import BEVStereo
Expand Down Expand Up @@ -206,7 +206,7 @@
}


class BEVDepthLightningModel(LightningModule):
class BEVStereoLightningModel(LightningModule):
MODEL_NAMES = sorted(name for name in models.__dict__
if name.islower() and not name.startswith('__')
and callable(models.__dict__[name]))
Expand Down Expand Up @@ -397,7 +397,8 @@ def configure_optimizers(self):
optimizer = torch.optim.AdamW(self.model.parameters(),
lr=lr,
weight_decay=1e-7)
return [optimizer]
scheduler = MultiStepLR(optimizer, [19, 23])
return [[optimizer], [scheduler]]

def train_dataloader(self):
train_dataset = NuscMVDetDataset(
Expand Down Expand Up @@ -467,10 +468,8 @@ def main(args: Namespace) -> None:
if args.seed is not None:
pl.seed_everything(args.seed)

model = BEVDepthLightningModel(**vars(args))
train_dataloader = model.train_dataloader()
ema_callback = EMACallback(len(train_dataloader.dataset) * args.max_epochs)
trainer = pl.Trainer.from_argparse_args(args, callbacks=[ema_callback])
model = BEVStereoLightningModel(**vars(args))
trainer = pl.Trainer.from_argparse_args(args)
if args.evaluate:
trainer.test(model, ckpt_path=args.ckpt_path)
else:
Expand All @@ -491,7 +490,7 @@ def run_cli():
default=0,
help='seed for initializing training.')
parent_parser.add_argument('--ckpt_path', type=str)
parser = BEVDepthLightningModel.add_model_specific_args(parent_parser)
parser = BEVStereoLightningModel.add_model_specific_args(parent_parser)
parser.set_defaults(
profiler='simple',
deterministic=False,
Expand All @@ -500,10 +499,10 @@ def run_cli():
num_sanity_val_steps=0,
gradient_clip_val=5,
limit_val_batches=0,
enable_checkpointing=False,
enable_checkpointing=True,
precision=16,
default_root_dir='./outputs/bev_depth_lss_r50_256x704_128x128_24e_2key'
)
default_root_dir='./outputs/bev_stereo_lss_r50_256x704_'
'128x128_24e_2key')
args = parser.parse_args()
main(args)

Expand Down
Loading

0 comments on commit d6d6555

Please sign in to comment.