o
    i                     @   s\  U d Z ddlmZ ddlmZ ddlmZ ddlmZ ddl	Z	ddl
mZmZmZmZmZ ddlmZ i Zeed	f ed
< g dZG dd	 d	eZdededdfddZdedededede	jdedefddZdedededede	jdedefddZede ede dededee	jB dedef
dd Zdefd!d"Zed#eded$ fd%d&Z dS )'z
This is an experimental new API for PyTorch Distributed. This is actively in development and subject to change or deletion entirely.

This is intended as a proving ground for more flexible and object oriented distributed APIs.
    )	Generator)contextmanager)	timedelta)ProtocolN)_current_process_group_set_process_groupProcessGroupReduceOpStore)
rendezvousProcessGroupFactory	_BACKENDS)r   r	   r   register_backend	new_groupcurrent_process_groupprocess_groupc                   @   s8   e Zd ZdZdededededejde	de
fd	d
ZdS )r   z%Protocol for process group factories.storerank
world_sizetimeoutdevicekwargsreturnc                 K   s   d S )N )selfr   r   r   r   r   r   r   r   Z/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/torch/distributed/_dist2.py__call__'   s   zProcessGroupFactory.__call__N)__name__
__module____qualname____doc__r
   intr   torchr   objectr   r   r   r   r   r   r   $   s"    namefuncr   c                 C   s$   | t v rtd|  d|t | < dS )z
    Register a new process group backend.

    Args:
        name: The name of the backend.
        func: The function to create the process group.
    Backend z already registeredN)r   
ValueError)r$   r%   r   r   r   r   2   s   r   r   r   r   r   r   r   c           	      K   s   ddl m} t|dkrtd|| |||}|  t| ||}|tjj |	|tjj| |	t
dtjj| t
j rN|	t
dtjj| |S )Nr   )ProcessGroupGlooz'Gloo backend received unexpected kwargscpucuda)torch.distributedr(   lenAssertionError_set_sequence_number_for_groupr   _set_default_backendBackendTypeGLOO_register_backendr"   r   r*   is_available)	r   r   r   r   r   r   r(   backend_classpgr   r   r   _gloo_factory@   s    
r6   c                 K   s   ddl m} | }||_| D ]\}}	t||s!td| t|||	 q|| |||}
|
  |
	| t
| ||}|t
jj ||t
jj|
 |S )Nr   )ProcessGroupNCCLzUnknown option )r+   r7   Options_timeoutitemshasattrKeyErrorsetattrr.   eager_connect_single_devicer   r/   r0   NCCLr2   )r   r   r   r   r   r   r7   optskvr4   r5   r   r   r   _nccl_factory_   s   

rC   glooncclbackendc                 K   s^   | t vrtd|  dt|}tttd\}}}|| t |  |||||fi |S )aF  
    Create a new process group with the given backend and options. This group is
    independent and will not be globally registered and thus not usable via the
    standard torch.distributed.* APIs.

    Args:
        backend: The backend to use for the process group.
        timeout: The timeout for collective operations.
        device: The device to use for the process group.
        **kwargs: All remaining arguments are passed to the backend constructor.
                  See the backend specific documentation for details.

    Returns:
        A new process group.
    r&   z not registeredzenv://)r   r'   r"   r   nextiterr   set_timeout)rF   r   r   r   r   r   r   r   r   r   r      s   

r   c                   C   s   t  S )zn
    Get the current process group. Thread local method.

    Returns:
        The current process group.
    )r   r   r   r   r   r      s   r   r5   )NNNc                 c   s0    t  }t|  z
dV  W t| dS t| w )zs
    Context manager for process groups. Thread local method.

    Args:
        pg: The process group to use.
    N)r   r   )r5   prev_pgr   r   r   r      s   r   )!r    collections.abcr   
contextlibr   datetimer   typingr   r"   torch._C._distributed_c10dr   r   r   r	   r
   torch.distributed.rendezvousr   r   dictstr__annotations____all__r   r   r!   r   r#   r6   rC   r   r   r   r   r   r   r   <module>   st    




 
