o
    ^i                     @   sd  U d Z ddlmZ ddlmZ ddlmZ ddlmZm	Z	 ddl
Z
ddlmZmZmZmZmZ ddlmZ i Zeed	f ed
< g dZG dd	 d	eZdededdfddZdedededede
jdedefddZdedededede
jdedefddZede ede dedede	ee
jf dedef
dd Zdefd!d"Z ed#eded$ fd%d&Z!dS )'z
This is an experimental new API for PyTorch Distributed. This is actively in development and subject to change or deletion entirely.

This is intended as a proving ground for more flexible and object oriented distributed APIs.
    )	Generator)contextmanager)	timedelta)ProtocolUnionN)_current_process_group_set_process_groupProcessGroupReduceOpStore)
rendezvousProcessGroupFactory	_BACKENDS)r	   r
   r   register_backend	new_groupcurrent_process_groupprocess_groupc                   @   s8   e Zd ZdZdededededejde	de
fd	d
ZdS )r   z%Protocol for process group factories.storerank
world_sizetimeoutdevicekwargsreturnc                 K   s   d S )N )selfr   r   r   r   r   r   r   r   S/var/www/html/RAG/RAG_venv/lib/python3.10/site-packages/torch/distributed/_dist2.py__call__'   s   zProcessGroupFactory.__call__N)__name__
__module____qualname____doc__r   intr   torchr   objectr	   r   r   r   r   r   r   $   s"    namefuncr   c                 C   s$   | t v rtd|  d|t | < dS )z
    Register a new process group backend.

    Args:
        name: The name of the backend.
        func: The function to create the process group.
    Backend z already registeredN)r   
ValueError)r%   r&   r   r   r   r   2   s   r   r   r   r   r   r   r   c           	      K   s   ddl m} t|dksJ d|| |||}|  t| ||}|tjj ||tjj| |t	
dtjj| t	j rN|t	
dtjj| |S )Nr   )ProcessGroupGlooz'Gloo backend received unexpected kwargscpucuda)torch.distributedr)   len_set_sequence_number_for_groupr	   _set_default_backendBackendTypeGLOO_register_backendr#   r   r+   is_available)	r   r   r   r   r   r   r)   backend_classpgr   r   r   _gloo_factory@   s   
r6   c                 K   s   ddl m} | }||_| D ]\}}	t||s!td| t|||	 q|| |||}
|
  |
	| t
| ||}|t
jj ||t
jj|
 |S )Nr   )ProcessGroupNCCLzUnknown option )r,   r7   Options_timeoutitemshasattrKeyErrorsetattrr.   eager_connect_single_devicer	   r/   r0   NCCLr2   )r   r   r   r   r   r   r7   optskvr4   r5   r   r   r   _nccl_factory^   s   

rC   glooncclbackendc                 K   s^   | t vrtd|  dt|}tttd\}}}|| t |  |||||fi |S )aF  
    Create a new process group with the given backend and options. This group is
    independent and will not be globally registered and thus not usable via the
    standard torch.distributed.* APIs.

    Args:
        backend: The backend to use for the process group.
        timeout: The timeout for collective operations.
        device: The device to use for the process group.
        **kwargs: All remaining arguments are passed to the backend constructor.
                  See the backend specific documentation for details.

    Returns:
        A new process group.
    r'   z not registeredzenv://)r   r(   r#   r   nextiterr   set_timeout)rF   r   r   r   r   r   r   r   r   r   r   ~   s   

r   c                   C   s   t  S )zn
    Get the current process group. Thread local method.

    Returns:
        The current process group.
    )r   r   r   r   r   r      s   r   r5   )NNNc                 c   s0    t  }t|  z
dV  W t| dS t| w )zs
    Context manager for process groups. Thread local method.

    Args:
        pg: The process group to use.
    N)r   r   )r5   prev_pgr   r   r   r      s   r   )"r!   collections.abcr   
contextlibr   datetimer   typingr   r   r#   torch._C._distributed_c10dr   r   r	   r
   r   torch.distributed.rendezvousr   r   dictstr__annotations____all__r   r   r"   r   r$   r6   rC   r   r   r   r   r   r   r   <module>   st    




 
