o iB @sddlZddlZddlZddlZddlmZmZddlmZddl m Z ddl m Z m Z e rGddlZddlmZdZejrFddlZd ZndZe eZd d Zed Zd edefddZddZ d.dedBdededBfddZd.dedBdededBfddZ dej!fddZ"ddZ#edZ$d d!Z%d"d#Z&dedBfd$d%Z' & d/ded'edefd(d)Z(e d*d+e  d0dedBd'edefd,d-Z)dS)1N)contextmanagerredirect_stdout)StringIO)logging)is_torch_availablerequires) save_fileFTcCs trtjs dStjdkS)z7Return True if rank=0 or we aren't running distributed.Tr)_torch_distributed_availabletorch distributedis_initializedget_rankrrd/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/transformers/model_debugging_utils.py _is_rank_zero,srzobject at 0x[0-9A-Fa-f]+x_strreturncCs td|S)z Replace memory addresses in an object's repr with a stable placeholder so that beautiful JSON diffs won't be ruined by ephemeral addresses. zobject at 0xXXXXXXXX)MEMORY_ADDRESS_REGEXsub)rrrr_sanitize_repr_for_diff6s rcCstr dt|jSdS)z@Return a stable string representation for a DTensor-like object.zDTensor (rank0) -> zDTensor(non-rank0))rrepr _local_tensor)xrrr _dtensor_repr>sr debug_pathuse_repr path_to_valuec Cstjdd|r t|}n4|r6|ds|d7}|r!tj||n|}td| i|d|}n t d|d|dt |j t |j|d }|jtjtjtjhvrz|tt |tt |tt |tt |d |S) a Converts Tensors and DTensors to a JSON-serializable dictionary representation. Args: value: Any Python object, often including torch Tensors, lists, dicts, etc. debug_path (`str`, *optional*, defaults to `None`): Directory to dump debug JSON and SafeTensors files. use_repr (bool, *optional*, defaults to `True`): Whether to save a `repr()`-ized version of the tensor as the `value` property in the asscoiated FULL_TENSORS.json file, or to store the full tensors in separate SafeTensors file and store the relative path to that file in the `value` property in the dictionary. path_to_value (`str`, *optional*, defaults to `None`): The file name for the SafeTensors file holding the full tensor value if `use_repr=False`. Returns: A nested Python structure (list, dict, or sanitized string) that is safe to json.dump. T)sci_modez .safetensorsdataz./z use_repr=z and path_to_value=z cannot both be falsy.)shapedtypevalue)meanstdminmax)r set_printoptions _repr_to_listendswithospathjoinr contiguousdetachcpu ValueErrorrr r!float16float32bfloat16updaterr#r$r%r&)r"rrr value_outfilepathoutrrr_serialize_tensor_like_ioEs.    r8cst|ttfrfddt|DSt|tr&fdd|DSt|dr4t|jdSt|t j rBt|dSt t |S)a Recursively build a JSON-serializable Python structure from `value`. Tensors and DTensors become either sanitized repr strings, or are saved to disk as SafeTensors files and their relative paths are recorded in the returned Python structure. Lists/tuples/dicts are recursed into. All memory addresses are replaced with a stable placeholder. Args: value: Any Python object, often including torch Tensors, lists, dicts, etc. debug_path (`str`, *optional*, defaults to `None`): Directory to dump debug JSON and SafeTensors files. use_repr (bool, *optional*, defaults to `True`): Whether to save a `repr()`-ized version of the tensors as the `value` property in the asscoiated FULL_TENSORS.json file, or to store full tensors in separate SafeTensors files and store the relative path to that file in the `value` property. path_to_value (`str`, *optional*, defaults to `None`): The file name for the SafeTensors file holding the full tensor value if `use_repr=False`. Returns: A nested Python structure (list, dict, or sanitized string) that is safe to json.dump. c s*g|]\}}t|d|dqS_rrr _serialize_io).0ivrrrrr sz!_serialize_io..c s,i|]\}}|t|d|dqSr9r<)r>kr@rArr sz!_serialize_io..rr;) isinstancelisttuple enumeratedictitemshasattrr8rr Tensorrr)r"rrrrrArr=vs     r=r"c Csxtjdddt$}t|t||}Wdn1s"wYWdn1s1wYt|S)z Converts a tensor into a sanitized multi-line string representation. Args: value (`torch.Tensor`): The tensor to represent. Returns: `list[str]`: List of string lines representing the tensor. Tx)r linewidthN)r r'rrprintgetvaluer splitlines)r"bufrawrrrr(s   r(cCs4|dr|dd|dD]}t|qdSdS)Nchildrenoutputs)getpopprune_outputs_if_childrennodechildrrrrXs    rXz (.*)\.(\d+)$csHt|dd}|r|dsdS|dtfdd|dDS)z Checks whether a node represents a layer block with submodules. Args: node (`dict`): A node from the call tree. Returns: `bool`: Whether the node is a layer block. module_pathrTFc3s(|]}dd|ddvVqdS).r\r]NrV)r>r[numberrr s&z!is_layer_block..)LAYER_SUFFIX_REmatchrVgroupany)rZrerraris_layer_blocks   rhcs~|dsdSddt|dD}t|dkr2dd|ddDfd dt|dD|d<|dD]}t|q6dS) z Recursively removes intermediate layers from the tree to improve readability. Keeps at least the first and last layers if many consecutive layers are present. Args: node (`dict`): The root or subnode to prune recursively. rTNcSs g|] \}}t|r||fqSr)rhr>r?r[rrrrBs z-prune_intermediate_layers..r^cSsg|]\}}|qSrr)r>r?r:rrrrBsrcsg|] \}}|vr|qSrrri to_removerrrBs)rVrHlenprune_intermediate_layers)rZ layer_blocksr[rrkrrns    rnc s.|r+ztj|ddtj||jd}Wnty*}z td|d|d}~ww|jd}td|d|d}|d }t |j t |d }t j |j |d d Wdn1s_wYfd dt t |j }|t |d }t j ||d d WddS1swYdS)NTexist_ok _debug_tree"Unexpected or existing debug_path=r_zWriting model trace at z.jsonz_FULL_TENSORS.jsonz _SUMMARY.jsonwr^)indentcsJfdd|di|di|dgD]}|qdS)NcsTt|tr|dd|D]}|qdSt|tr&|D]}|qdSdS)Nr")rErIrWvaluesrF)valr@itemcleanrrrzs      z:log_model_debug_trace..strip_values..cleaninputsrUrTr`rY strip_valuesryrr}s   z+log_model_debug_trace..strip_values)r*makedirsr+r,_debugger_module_dump_name Exceptionr0loggerinforX _call_treeopenjsondumploadsdumps)rmodelbasee full_path summary_pathf tree_copyrr|rlog_model_debug_traces,     "rr_do_prune_layersc sjjddgd_g__r3z tjddWnty2}z tdd|d}~wwfdd} D]\}}|d krHq?||d|q?j t fd d }|_ dS) a Attaches a debugging wrapper to every module in the model. This records structured inputs and outputs during the forward pass into a call tree. Args: model (`PreTrainedModel`, `nn.Module`): Model to wrap. debug_path (`str`): Optional directory to dump debug JSON files. do_prune_layers (`bool`, *optional*, defaults to `True`): Whether to prune intermediate layers. use_repr (bool, *optional*, defaults to `True`): Whether to save a `repr()`-ized version of the tensors as the `value` property in the associated FULL_TENSORS.json file, or to store full tensors in separate SafeTensors files and store the relative path to that file in the `value` property. Nr\r{rUrTTrprsr_cs0jtfdd}|_dS)Ncstr'||dfddDtdddgd}j|t|i|}Wdn1s=wYtrtddDd krWd|d <n t|d d|d <j}|d sr|d jrjd d ||S)Nargskwargscs&i|]}t|dkr||qS)r)rmr>rC dict_inputsrrrD5s&zY_attach_debugger_logic..wrap_forward..wrapped_forward.._inputsr;rcss|]}dVqdS)rNr)r>r:rrrrcFszX_attach_debugger_logic..wrap_forward..wrapped_forward..rrU_outputsrTrj) rr=_debugger_model_call_stackappendr no_gradsumnamed_childrenrW)inpskwsrZr7finished)rrrmodule orig_forwardrrrwrapped_forward1s@     zE_attach_debugger_logic..wrap_forward..wrapped_forward)forward functoolswraps)rrr)rrr)rrrr wrap_forward.s 'z,_attach_debugger_logic..wrap_forwardr]cstrdt||ddddgd}j||i|}trnjrnt|dd|d<j}|djd<|djd<|d jd <fd d tjDrhtjt d |S) Nz (top-level)rrr;rrrUr{rTcs$g|]}j|sj|dqS)N)rrWr)rrrrBs$zG_attach_debugger_logic..top_wrapped_forward..)rr) rr=rrrWrrFkeysrnr)rrtop_noder7r class_namerrrreal_top_forwardrrrtop_wrapped_forwardds:     z3_attach_debugger_logic..top_wrapped_forward) __class____name__rrrr*r~rr0 named_modulesrrr) rrrrrrname submodulerrrr_attach_debugger_logics(. %r)r )backendsc csldd|D}|j||<t||||z|VW|D]\}}||_qdS|D]\}}||_q-w)a # Model addition debugger - context manager for model adders This context manager is a power user tool intended for model adders. It tracks all forward calls within a model forward and logs a slice of each input and output on a nested JSON file. If `use_repr=True` (the default), the JSON file will record a `repr()`-ized version of the tensors as a list of strings. If `use_repr=False`, the full tensors will be stored in separate SafeTensors files and the JSON file will provide a relative path to that file. To note, this context manager enforces `torch.no_grad()`. ## Usage add the context manager to a model to debug ```python import torch from PIL import Image from transformers import LlavaProcessor, LlavaForConditionalGeneration, model_addition_debugger_context torch.random.manual_seed(673) # load pretrained model and processor model_id = "llava-hf/llava-1.5-7b-hf" processor = LlavaProcessor.from_pretrained(model_id) model = LlavaForConditionalGeneration.from_pretrained(model_id) # create random image input random_image = Image.fromarray(torch.randint(0, 256, (224, 224, 3), dtype=torch.uint8).numpy()) # prompt prompt = "Describe this image." # process inputs inputs = processor(text=prompt, images=random_image, return_tensors="pt") # call forward method (not .generate!) with model_addition_debugger_context(model, debug_path="Your_debug_path", do_prune_layers=False): output = model.forward(**inputs) ``` cSsi|]\}}||jqSr)r)r>r:mrrrrDsz3model_addition_debugger_context..N)rrrrJ)rrrr orig_forwardsmodule_instanceforward_methodrrrmodel_addition_debugger_contexts3 r)NTN)r_TT)NTT)*rrr*re contextlibrriorutilsrutils.import_utilsrrr safetensors.torchr r r is_availabletorch.distributed.tensor get_loggerrrrcompilerstrrrboolr8r=rLr(rXrdrhrnrrrrrrrsv       1+ -