Ë
    ßGìj;;  ã                  ód  — d dl mZ d dlmZ d dlZd dlZd dlmZ d dlZd dl	m
Z
 d dl	mZ d dl	mZ d dlmZ d d	lmZ  ej                   e«      Z ej&                  d
dd¬«      Z ej*                  ddd¬«      Z G d„ d«      Z e«       Z	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zd„ Zy)é    )Úannotations)ÚSequenceN)ÚAny)Úclusters)Úconfig)Ú
xla_bridge)Ú_jax)Újaxlib_extension_versionÚjax_check_proxy_envsTz1Checks proxy vars in user envs and emit warnings.)ÚnameÚdefaultÚhelpÚjax_enable_recoverabilityFzYAllows a multi-controller JAX job to continue running, even after some tasks have failed.c                  óº   — e Zd ZU dZded<   dZded<   dZded<   dZd	ed
<   dZded<   dZ	ded<   dZ
ded<   	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zd„ Zd„ Zy)ÚStater   ÚintÚ
process_idé   Únum_processesNz+_jax.DistributedRuntimeService | Any | NoneÚservicez*_jax.DistributedRuntimeClient | Any | NoneÚclientz
Any | NoneÚpreemption_sync_managerú
str | NoneÚcoordinator_addressú
int | NoneÚpartition_indexc
                ó  — |xs t         j                  j                  d«      }t        |t        «      r|g}|€It         j                  j                  d«      x}
r(t        t        t        |
j                  d«      «      «      }|dk7  r1d ||||fv r)t        j                  j                  ||||||«      \  }}}}|€t        d«      ‚|€t        d«      ‚|€t        d«      ‚t        |t        «      st        d|› d	t        |«      › d
�«      ‚t        |t        «      st        d|› d	t        |«      › d
�«      ‚d|cxk  r|k  sn t        d|› d|› d
�«      ‚|| _        d|j                  dd«      d   z   }|xs  t         j                  j                  d|«      }|€t        d«      ‚|rZdj!                  d„ |D «       «      }t"        j%                  d|«       t'        j(                  d|«       t'        j(                  d|«       || _        g }t,        j.                  r=t         j                  j1                  «       D �cg c]  }d|j3                  «       v r|‘Œ }}t5        |«      dkD  r/dj!                  |«      dz   }d|› d�}t"        j7                  |«       i }t8        dk\  rt:        j.                  |d<   |dk(  rJ| j<                  �t?        d«      ‚t"        j%                  d |«       tA        jB                  |||¬!«      | _        || _"        | jF                  �t?        d«      ‚tA        jH                  ||f|d"|d#œ|¤Ž| _#        t"        j%                  d$|«       | jF                  jK                  «        | jM                  «        |	€zt         j                  j                  d%«      }t         j                  j                  d&«      }|�t	        |«      }	|	| _*        y |�%tO        jP                  d'tR        «       t	        |«      }	|	| _*        y c c}w )(NÚJAX_COORDINATOR_ADDRESSÚJAX_LOCAL_DEVICE_IDSÚ,Ú
deactivatez&coordinator_address should be defined.z$Number of processes must be defined.z6The process id of the current process must be defined.z5process_id must be a nonnegative int. Got process_id=z	 of type Ú.z8num_processes must be a positive int. Got num_processes=r   zbprocess_id and num_processes must be nonnegative, with process_id < num_processes. Got process_id=z, num_processes=z[::]:Ú:r   ÚJAX_COORDINATOR_BIND_ADDRESSz+coordinator_bind_address should be defined.c              3  ó2   K  — | ]  }t        |«      –— Œ y ­w)N)Ústr)Ú.0Úxs     úXC:\Users\Stabi\Downloads\virtual-tryon(1)\venv\Lib\site-packages\jax/_src/distributed.pyÚ	<genexpr>z#State.initialize.<locals>.<genexpr>t   s   è ø€ Ð BÑ1A¨A¤ Q§Ñ1Aùs   ‚z4JAX distributed initialized with visible devices: %sÚjax_cuda_visible_devicesÚjax_rocm_visible_devicesÚ_proxyÚ z. zHJAX detected proxy variable(s) in the environment as distributed setup: zpOn some systems, this may cause a hang of distributed.initialize and you may need to unset these ENV variable(s)ip  Úrecoverablez2distributed.initialize should only be called once.z&Starting JAX distributed service on %s)Úheartbeat_timeoutT)Úinit_timeoutÚuse_compressionr0   z+Connecting to JAX distributed service on %sÚJAX_PARTITION_INDEXÚJAX_SLICE_INDEXzLJAX_SLICE_INDEX has been deprecated. Please use JAX_PARTITION_INDEX instead.)+ÚosÚenvironÚgetÚ
isinstancer   ÚlistÚmapÚsplitr   Ú
ClusterEnvÚ$auto_detect_unset_distributed_paramsÚ
ValueErrorÚ	TypeErrorÚtyper   ÚrsplitÚjoinÚloggerÚinfor   Úupdater   Ú_CHECK_PROXY_ENVSÚvalueÚkeysÚlowerÚlenÚwarningr
   Ú_ENABLE_RECOVERABILITYr   ÚRuntimeErrorr	   Úget_distributed_runtime_servicer   r   Úget_distributed_runtime_clientÚconnectÚ"initialize_preemption_sync_managerÚwarningsÚwarnÚDeprecationWarningr   )Úselfr   r   r   Úlocal_device_idsÚcluster_detection_methodÚinitialization_timeoutÚcoordinator_bind_addressÚheartbeat_timeout_secondsr   Úenv_idsÚ default_coordinator_bind_addressÚvisible_devicesÚ
proxy_varsÚkeyÚvarsrK   Úrecoverable_kwargsÚjax_partition_indexÚjax_slice_indexs                       r)   Ú
initializezState.initialize9   s?  € ð /ò EÜŸ:™:Ÿ>™>Ð*CÓDð äÐ"¤CÔ(Ø*Ð+ÐàÐ´·
±
·±Ð?UÓ0VÐ%V WÐ%VÜœc¤# w§}¡}°SÓ'9Ó:Ó;Ðà  LÒ0ØÐ$ m°ZÐAQÐRÑRä
×
Ñ
×
BÑ
BØ!ØØØØ&Ø$óñ IÐ˜M¨:Ð7Gð Ð"ÜÐ?Ó@Ð@ØÐÜÐ=Ó>Ð>ØÐÜÐOÓPÐPÜ�j¤#Ô&Üð (Ø(2 |°9¼TÀ*Ó=MÐ<NÈaðQó Rð Rä�m¤SÔ)Üð +Ø+8¨/¸Ä4ÈÓCVÐBWÐWXðZó [ð [à�Ô+˜mÔ+Üð )Ø)3¨Ð4DÀ]ÀOÐSTðVó Wð Wð  3€DÔð (/Ð1D×1KÑ1KÈCÐQRÓ1SÐTUÑ1VÑ'VÐ$Ø 8ò !QÜ "§
¡
§¡Ð/MØ/Oó!Qð ð  Ð'ÜÐDÓEÐEáØŸ™Ñ BÑ1AÓ BÓB€oÜ‡k�kÐHÈ/ÔZÜ‡m�mÐ.°Ô@Ü‡m�mÐ.°Ô@à €D„Oà€JÜ×ÒÜ#%§:¡:§?¡?Ô#4ó 0Ñ#4˜CØ 3§9¡9£;Ñ.ò Ð#4€jð 0ô ˆ:ƒ˜ÒØ�X‰X�jÓ! DÑ(€dà
RÐSWÐRXð	6ð	6ð ô
 ‡n�n�WÔàÐÜ 3Ò&ä*@×*FÑ*FÐ˜Ñ'à�Q‚Ø	�‰Ð	!ÜÐOÓPÐPÜ‡k�kØ
2Ð4Lôô ×9Ñ9Ø
" MØ5ô7€d„lð '€DÔà‡{�{ÐÜÐMÓNÐNä×5Ñ5Ø˜ZðØ6LØÐ0Iñð ñ€D„Kô ‡K�KÐ=Ð?RÔSØ‡K�K×ÑÔà×+Ñ+Ô-àÐÜŸJ™JŸN™NÐ+@ÓAÐÜŸ
™
Ÿ™Ð'8Ó9€oØ	Ð	(ÜÐ1Ó2ˆð +€DÕð Ð&ä�‰ð,äô	
ô
 ˜oÓ.ˆØ*€DÕùòm0s   È9Pc                ó@  — | j                   r7t        | j                   d«      r| j                   j                  «        d | _         | j                  r!| j                  j                  «        d | _        | j                  r"| j                  j                  «        d | _        y y )NÚshutdown)r   Úhasattrrf   r   r   ©rU   s    r)   rf   zState.shutdownµ   sx   € Ø×#Ò#ô 
�×-Ñ-¨zÔ	:Ø×$Ñ$×-Ñ-Ô/Ø%)€dÔ"Ø‡{‚{Ø
‡k�k×ÑÔØ€d„kØ‡|‚|Ø
‡l�l×ÑÔØ€d…lð ó    c                ó®   — | j                   �t        d«      ‚t        j                  «       | _         | j                   j	                  | j
                  «       y )Nz8Preemption sync manager should only be initialized once.)r   rM   r	   Úcreate_preemption_sync_managerrd   r   rh   s    r)   rQ   z(State.initialize_preemption_sync_managerÄ   sN   € Ø×#Ñ#Ð/ÜØ
DóFð Fô 	×+Ñ+Ó-ð 	Ô à× Ñ ×+Ñ+¨D¯K©KÕ8ri   )	NNNNNé,  Néd   N)r   r   r   r   r   r   rV   úint | Sequence[int] | NonerW   r   rX   r   rY   r   rZ   r   r   r   )Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__r   r   r   r   r   r   rd   rf   rQ   © ri   r)   r   r   0   sÙ   … Ø€*ˆcÓØ€-�ÓØ9=€'Ð6Ó=Ø7;€&Ð
4Ó;Ø(,Ð˜:Ó,Ø$(Ð�zÓ(Ø $€/�:Ó$ð 48Ø-1Ø*.Ø@DØ8<Ø/2Ø8<Ø25Ø/3ðz+Ø&0ðz+à *ðz+ð (ðz+ð $>ð	z+ð
 ,6ðz+ð *-ðz+ð ,6ðz+ð -0ðz+ð #-óz+òxó9ri   r   c
                ó¾   — t        j                  «       rt        d«      ‚|	€|�t        j                  dt
        «       |}	t        j                  | ||||||||	¬«	       y)a  Initializes the JAX distributed system.

  Calling :func:`~jax.distributed.initialize` prepares JAX for execution on
  multi-host GPU and Cloud TPU. :func:`~jax.distributed.initialize` must be
  called before performing any JAX computations.

  The JAX distributed system serves a number of roles:

    * It allows JAX processes to discover each other and share topology information,
    * It performs health checking, ensuring that all processes shut down if any process dies, and
    * It is used for distributed checkpointing.

  If you are using TPU, Slurm, or Open MPI, all arguments are optional: if omitted, they
  will be chosen automatically.

  The ``cluster_detection_method`` may be used to choose a specific method for detecting those
  distributed arguments. You may pass any of the automatic ``spec_detect_methods`` to this
  argument though it is not necessary in the TPU, Slurm, or Open MPI cases.  For other MPI
  installations, if you have a functional ``mpi4py`` installed, you may pass
  ``cluster_detection_method="mpi4py"`` to bootstrap the required arguments.

  Otherwise, you must provide the ``coordinator_address``,
  ``num_processes``, ``process_id``, and ``local_device_ids`` arguments
  to :func:`~jax.distributed.initialize`. When all four arguments are provided, cluster
  environment auto detection will be skipped.

  Please note: on some systems, particularly HPC clusters that only access external networks
  through proxy variables such as HTTP_PROXY, HTTPS_PROXY, etc., the call to
  :func:`~jax.distributed.initialize` may timeout.  You may need to unset these variables
  prior to application launch.

  Args:
    coordinator_address: the IP address of process `0` and a port on which that
      process should launch a coordinator service. The choice of
      port does not matter, so long as the port is available on the coordinator
      and all processes agree on the port.
      May be ``None`` only on supported environments, in which case it will be chosen automatically.
      Note that special addresses like ``localhost`` or ``127.0.0.1`` usually mean that the program
      will bind to a local interface and are not suitable when running in a multi-host environment.
    num_processes: Number of processes. May be ``None`` only on supported environments, in
      which case it will be chosen automatically.
    process_id: The ID number of the current process. The ``process_id`` values across
      the cluster must be a dense range ``0``, ``1``, ..., ``num_processes - 1``.
      May be ``None`` only on supported environments; if ``None`` it will be chosen automatically.
    local_device_ids: Restricts the visible devices of the current process to ``local_device_ids``.
      If ``None``, defaults to all local devices being visible to the process except when processes
      are launched via Slurm and Open MPI on GPUs. In that case, it will default to a single device per process.
    cluster_detection_method: An optional string to attempt to autodetect the configuration of the distributed
      run.  Note that "mpi4py" method requires you to have a working ``mpi4py`` install in your environment,
      and launch the applicatoin with an MPI-compatible job launcher such as ``mpiexec`` or ``mpirun``.
      Legacy auto-detect options "ompi" (OMPI) and "slurm" (Slurm) remain enabled. "deactivate" bypasses
      automatic cluster detection.
    initialization_timeout: Time period (in seconds) for which connection will
      be retried. If the initialization takes more than the timeout specified,
      the initialization will error. Defaults to 300 secs i.e. 5 mins.
    heartbeat_timeout_seconds: The time (in seconds) after which a process is
      considered dead if it hasn't successfully sent any heartbeats. Defaults
      to 100 seconds.
    coordinator_bind_address: the address and port to which the coordinator service
      on process `0` should bind. If this is not specified, the default is to bind to
      all available addresses on the same port as ``coordinator_address``. On systems
      that have multiple network interfaces per node it may be insufficient to only
      have the coordinator service listen on one address/interface.
    slice_index: DEPRECATED: Use ``partition_index`` instead.
    partition_index: The partition index assigned to this process' local devices. If any process sets ``partition_index``,
      then all processes must do so. If ``None`` the partition indices will be chosen automatically.

  Raises:
    RuntimeError: If :func:`~jax.distributed.initialize` is called more than once
      or if called after the backend is already initialized.

  Examples:

  Suppose there are two GPU processes, and process 0 is the designated coordinator
  with address ``10.0.0.1:1234``. To initialize the GPU cluster, run the
  following commands before anything else.

  On process 0:

  >>> jax.distributed.initialize(coordinator_address='10.0.0.1:1234', num_processes=2, process_id=0)  # doctest: +SKIP

  On process 1:

  >>> jax.distributed.initialize(coordinator_address='10.0.0.1:1234', num_processes=2, process_id=1)  # doctest: +SKIP
  zÁjax.distributed.initialize() must be called before any JAX calls that might initialise the XLA backend. This includes any computation, but also calls to jax.devices, jax.device_put, and others.NzH`slice_index` has been deprecated. Please use `partition_index` instead.)rZ   r   )r   Úbackends_are_initializedrM   rR   rS   rT   Úglobal_staterd   )
r   r   r   rV   rW   rX   rZ   rY   Úslice_indexr   s
             r)   rd   rd   Î   sy   € ô~ ×(Ñ(Ô*Ü
ð tó uð uð ÐØÐä‡m�mØ
TÜ
ôð "€OÜ×ÑÐ-¨}¸jØ*Ð,DØ0Ð2JØ4MØ*9ð	 õ ;ri   c                 ó&   — t         j                  duS )z3Check if the JAX distributed system is initialized.N)rv   r   rs   ri   r)   Úis_initializedry   @  s   € ä	×	Ñ	 DÐ	(Ð(ri   c                 ó,   — t         j                  «        y)z_Shuts down the distributed system.

  Does nothing if the distributed system is not running.
  N)rv   rf   rs   ri   r)   rf   rf   D  s   € ô
 ×ÑÕri   )
NNNNNrl   rm   NNN)r   r   r   r   r   r   rV   rn   rW   r   rX   r   rZ   r   rY   r   rw   r   r   r   )ÚreturnÚbool)Ú
__future__r   Úcollections.abcr   Úloggingr5   Útypingr   rR   Újax._srcr   r   r   Újax._src.libr	   r
   Ú	getLoggerro   rC   Ú	bool_flagrF   Ú
bool_staterL   r   rv   rd   ry   rf   rs   ri   r)   Ú<module>r†      s  ðõ #å $Û Û 	Ý Û å Ý Ý Ý Ý 1à	ˆ×	Ñ	˜8Ó	$€ð %�F×$Ñ$Ø	ØØ	<ôÐ ð +˜×*Ñ*Ø	$Øð	ô	Ð ÷Z9ñ Z9ñx ‹w€à15Ø+/Ø(,Ø>BØ6:Ø-0Ø03Ø6:Ø)-Ø-1ðo;Ø(ðo;à%ðo;ð "<ðo;ð *4ð	o;ð
 (+ðo;ð +.ðo;ð *4ðo;ð 'ðo;ð !+óo;ód)óri   