AssertionError: Default process group is not initialized #61
Description
Traceback (most recent call last):
File "/apdcephfs/private_finechen/cbcode/moco/moco_test/run_moco.py", line 353, in
main()
File "/apdcephfs/private_finechen/cbcode/moco/moco_test/run_moco.py", line 134, in main
main_worker(args.gpu, ngpus_per_node, args)
File "/apdcephfs/private_finechen/cbcode/moco/moco_test/run_moco.py", line 226, in main_worker
train(train_loader, model, criterion, optimizer, epoch, args)
File "/apdcephfs/private_finechen/cbcode/moco/moco_test/run_moco.py", line 250, in train
output, target = model(im_q=images1, im_k=images2)
File "/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py", line 532, in call
result = self.forward(*input, **kwargs)
File "/apdcephfs/private_finechen/cbcode/moco/moco_test/moco/builder.py", line 133, in forward
im_k, idx_unshuffle = self._batch_shuffle_ddp(im_k)
File "/usr/local/lib64/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
return func(*args, **kwargs)
File "/apdcephfs/private_finechen/cbcode/moco/moco_test/moco/builder.py", line 76, in _batch_shuffle_ddp
x_gather = concat_all_gather(x)
File "/usr/local/lib64/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
return func(*args, **kwargs)
File "/apdcephfs/private_finechen/cbcode/moco/moco_test/moco/builder.py", line 170, in concat_all_gather
tensors_gather = [torch.ones_like(tensor) for _ in range(torch.distributed.get_world_size())]
File "/usr/local/lib64/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 586, in get_world_size
return _get_group_size(group)
File "/usr/local/lib64/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 202, in _get_group_size
_check_default_pg()
File "/usr/local/lib64/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 193, in _check_default_pg
"Default process group is not initialized"
AssertionError: Default process group is not initialized