o 0i@sddlZddlZddlmZddlmZmZddlZddl Z ddl m Z dddZ ddZ d d Zd eeejfd eee jffd dZGdddeZGdddeZd eee jfdeefddZdS)N)Mapping)OptionalUnion) safe_opencCsd}t|jdkr|tj}d}|}tj ||d}|dur8|dur.t|j}|t |j d||<|j dkrA|d}t j||jd|j d}|dd|dd<||S)Nztorch.bfloat16bfloat16.dat)dtypeshaperzw+)rmoder )strrviewtorchint16cpunumpyospathjoinlistr ndimnpmemmapflush)weight weight_nameoffload_folderindexrarray tensor_file file_arrayr Z/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/accelerate/utils/offload.pyoffload_weights     r"cCs|t|d}|dkr d}|d}|dkrd}tj|||dd}t|dd kr+|d }t|}|ddkr<|tj}|S) Nr r )rrrr)rr r r)tuplerrlenr tensorr r) weight_file weight_infor rrr r r!load_offloaded_weight.s    r*cCs|dus t|dkr dStj|d}tj|r5t|dd }t|}Wdn1s/wYni}||t|ddd}tj ||ddWddS1sWwYdS)Nr index.jsonzutf-8)encodingw)indent) r&rrrisfileopenjsonloadupdatedump)rroffload_index_filef current_indexr r r!save_offload_indexDs   "r9save_dir state_dictcCsBtj|ddi}|D] \}}t||||d}q t||dS)a Offload a state dict in a given folder. Args: save_dir (`str` or `os.PathLike`): The directory in which to offload the state dict. state_dict (`Dict[str, torch.Tensor]`): The dictionary of tensors to offload. T)exist_ok)rN)rmakedirsitemsr"r9)r:r;rname parameterr r r!offload_state_dictUs  rAc@s:eZdZdZdedefddZddZdd Zd d Z d S) PrefixedDatasetz Will access keys in a given dataset by adding a prefix. Args: dataset (`Mapping`): Any map with string keys. prefix (`str`): A prefix to add when trying to access any element in the underlying dataset. datasetprefixcCs||_||_dSNrCrD)selfrCrDr r r!__init__qs zPrefixedDataset.__init__cCs|j|j|SrErF)rGkeyr r r! __getitem__uszPrefixedDataset.__getitem__cstfddjDS)Ncsg|] }|jr|qSr ) startswithrD.0rIrGr r! ysz,PrefixedDataset.__iter__..)iterrCrNr rNr!__iter__xszPrefixedDataset.__iter__cC t|jSrE)r&rCrNr r r!__len__{ zPrefixedDataset.__len__N) __name__ __module__ __qualname____doc__rr rHrJrQrSr r r r!rBhs  rBc@sneZdZdZ    ddeeeejfdee ee j fdee fddZ defd d Zd d Zd dZdS)OffloadedWeightsLoaderaC A collection that loads weights stored in a given state dict or memory-mapped on disk. Args: state_dict (`Dict[str, torch.Tensor]`, *optional*): A dictionary parameter name to tensor. save_folder (`str` or `os.PathLike`, *optional*): The directory in which the weights are stored (by `offload_state_dict` for instance). index (`Dict`, *optional*): A dictionary from weight name to their information (`dtype`/ `shape` or safetensors filename). Will default to the index saved in `save_folder`. Nr; save_folderrcs|dur|dur|durtd|durin|_|_|durB|durBttj|d }t|}Wdn1s=wY|durHin|_ t j _ j fddj D|_dS)NzWNeed either a `state_dict`, a `save_folder` or an `index` containing offloaded weights.r+csg|] }|jvr|qSr )all_keysrLrNr r!rOsz3OffloadedWeightsLoader.__init__..) ValueErrorr;rZr1rrrr2r3rrkeysr[extenddevice)rGr;rZrr_r7r rNr!rHs  zOffloadedWeightsLoader.__init__rIc Cs>||jvr |j|S|j|}|ddur|jdurdn|j}d}z$t|dd|d}||d|}Wdn1s@wYWn+tyqt|dddd}||d|}Wdn1sjwYYnwd|vr|tt |d}|jt |kr||}|St j |j |d}t||S)Nsafetensors_filerpt) frameworkr_rrr)r;rgetr_r get_tensor TypeErrortogetattrr rrrrZr*)rGrIr)r_r'r7r(r r r!rJs0      z"OffloadedWeightsLoader.__getitem__cCrRrE)rPr[rNr r r!rQrTzOffloadedWeightsLoader.__iter__cCrRrE)r&r[rNr r r!rSrTzOffloadedWeightsLoader.__len__)NNNN)rUrVrWrXrdictr r TensorrrPathLikerrHrJrQrSr r r r!rYs   rYsubmodule_namescs.i}|D]|fdd|Dq|S)a Extract the sub state-dict corresponding to a list of given submodules. Args: state_dict (`Dict[str, torch.Tensor]`): The state dict to extract from. submodule_names (`List[str]`): The list of submodule names we want to extract. cs,i|]\}}|ks|dr||qS).)rK)rMrIparam module_namer r! s z1extract_submodules_state_dict..)r4r>)r;rkresultr rnr!extract_submodules_state_dicts rrrE)r2rcollections.abcrtypingrrrrr safetensorsrr"r*r9r rjrhrirArBrYrrrr r r r!s   &$C