ValueError: cost matrix is infeasible

[zzy-ucas]

When I train the code with my dataset. It occurs the following problem. So weird! I have check all data pipelines to ensure the accuracy. I infer that it may be a Nan error! So, how can I resolve it? Thanks!

Traceback (most recent call last):
  File "tools/train.py", line 293, in <module>
    main()
  File "tools/train.py", line 282, in main
    train_model(
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmdet3d/apis/train.py", line 344, in train_model
    train_detector(
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmdet3d/apis/train.py", line 319, in train_detector
    runner.run(data_loaders, cfg.workflow)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 136, in run
    epoch_runner(data_loaders[i], **kwargs)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 53, in train
    self.run_iter(data_batch, train_mode=True, **kwargs)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 31, in run_iter
    outputs = self.model.train_step(data_batch, self.optimizer,
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmcv/parallel/distributed.py", line 63, in train_step
    output = self.module.train_step(*inputs[0], **kwargs[0])
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmdet/models/detectors/base.py", line 248, in train_step
    losses = self(**data)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
    return forward_call(*input, **kwargs)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 146, in new_func
    output = old_func(*new_args, **new_kwargs)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmdet3d/models/detectors/base.py", line 60, in forward
    return self.forward_train(**kwargs)
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/models/detectors/cmt.py", line 155, in forward_train
    losses_pts = self.forward_pts_train(pts_feats, img_feats, gt_bboxes_3d,
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 233, in new_func
    output = old_func(*new_args, **new_kwargs)
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/models/detectors/cmt.py", line 190, in forward_pts_train
    losses = self.pts_bbox_head.loss(*loss_inputs)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 205, in new_func
    return old_func(*args, **kwargs)
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/models/dense_heads/jdx_head.py", line 169, in loss
    loss_cls, loss_bbox = multi_apply(
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmdet/core/utils/misc.py", line 30, in multi_apply
    return tuple(map(list, zip(*map_results)))
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/models/dense_heads/cmt_head.py", line 740, in loss_single
    cls_reg_targets = self.get_targets(
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/models/dense_heads/cmt_head.py", line 645, in get_targets
    bbox_weights_list, pos_inds_list, neg_inds_list) = multi_apply(
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmdet/core/utils/misc.py", line 30, in multi_apply
    return tuple(map(list, zip(*map_results)))
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/models/dense_heads/cmt_head.py", line 623, in _get_targets_single
    = multi_apply(task_assign, pred_bboxes, pred_logits, task_boxes, task_classes, self.num_classes)
  File "/cfs/personal_data/zhangzhengyuan/envs/cmt-a100/lib/python3.8/site-packages/mmdet/core/utils/misc.py", line 30, in multi_apply
    return tuple(map(list, zip(*map_results)))
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/models/dense_heads/cmt_head.py", line 603, in task_assign
    assign_results = self.assigner.assign(bbox_pred, logits_pred, gt_bboxes, gt_labels)
  File "/cfs/personal_data/zhangzhengyuan/JDXLargeModel/projects/mmdet3d_plugin/core/bbox/assigners/hungarian_assigner_3d.py", line 143, in assign
    matched_row_inds, matched_col_inds = linear_sum_assignment(cost)
ValueError: cost matrix is infeasible