o iq @s&UddlZddlZddlmZddlmZmZddlmZddl Z ddl m Z ddl mZddlmZgZeeed<dZefd ed ejfd d Zefd ed eejeffd dZead eeeffddZedZedZdeeefd eeeffddZ deeefd eeeffddZ!dS)N)Callable)AnyTypeVar) ParamSpec) _log_handlers) _WaitCounter__all__default destinationreturncCsPt|\}}td|}|tjtd}||d|_|||S)Nzc10d-za%(asctime)s %(filename)s:%(lineno)s %(levelname)s p:%(processName)s t:%(threadName)s: %(message)sF) _get_logging_handlerlogging getLoggersetLevelDEBUG Formatter setFormatter propagate addHandler)r logging_handlerlog_handler_namelogger formatterr_/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/torch/distributed/c10d_logger.py_get_or_create_loggers    rcCs$t|}t|jd|}||fS)N-)rtype__name__)r log_handlerrrrrr (sr c OstrL|dp |d}|t|dt|tt|tt|d}|ddkrJtjj }d dd |D|d <|Sd |i}|S) Ngroup process_grouppg) func_namepg_namebackend world_size group_size global_rank local_rankr%nccl.css|]}t|VqdS)N)str).0vrrr Csz _get_msg_dict.. nccl_versionr#) distis_initializedget_get_process_group_name get_backendget_world_sizeget_ranktorchcudar*versionjoin)r#argskwargsr msg_dictr0rrr _get_msg_dict5s"    r?_T_Pfuncc,tdtjdtjdtffdd }|S)Nr<r=r c sXz|i|WSty+}ztjg|Ri|}||d<t|d}~ww)Nerror) Exceptionr?r _c10d_loggerdebug)r<r=rDr>rBrrwrapperPs  z"_exception_logger..wrapper functoolswrapsrAr<r=r@rBrIrrHr_exception_loggerOs  rNcrC)Nr<r=r csHtdj|i|}Wd|S1swY|S)Nzpytorch.wait_counter.c10d.)rrguard)r<r= func_returnrHrrrI^s  z_time_logger..wrapperrJrMrrHr _time_logger]s rQ)"rKr collections.abcrtypingrrtyping_extensionsrr8torch.distributed distributedr1"torch.distributed.logging_handlersr torch.monitorrrlistr,__annotations___DEFAULT_DESTINATIONLoggerrtupleHandlerr rFdictr?r@rArNrQrrrrs0         "&