o i7@sddlmZddlmZddlmZddlmZmZddl m Z m Z m Z m Z mZe r5ddlZddlmZe r>dd lmZe Ze rJesJddlZeeZGd d d eZGd d d ejjZGdddejZeddddZ ddee dBfddZ!dS)) lru_cache)ACT2FN) ConversionOps)get_module_from_nameshould_convert_module)is_accelerate_availableis_fbgemm_gpu_availableis_torch_availableis_torch_xpu_availableloggingN)nn)init_empty_weightsc @sTeZdZddZ d deeejeejBfdej j dBdeeejffddZ dS) FbgemmFp8QuantizecCs ||_dSN) hf_quantizer)selfrrf/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/transformers/integrations/fbgemm_fp8.py__init__-s zFbgemmFp8Quantize.__init__N input_dictmodelreturncKsHt|d\}}|d}ddlm}t||\}}t||r|dkrR|dd} | j} | d| d} t | \} } | | }|dd}| | dd| d}nE|dkr|dd} | j} | d| d} t | \} } | | }|dd}| | d| dd}nt |\}}t j | |jdd}|t j ||d|iS) Nrr)FbgemmFp8Llama4TextExperts gate_up_proj down_proj_scale)tupleitems integrationsrr isinstance transposeshapereshapequantize_fp8_per_rowtorchr Parameterview)rrrkwargs target_keyvaluermodule tensor_nametransposed_paramoriginal_shapeflattened_paramnew_value_flatweight_scale_flat new_value weight_scalerrrconvert0s2           zFbgemmFp8Quantize.convertr) __name__ __module__ __qualname__rdictstrr'Tensorlistr Moduler6rrrrr,s  rc*eZdZejffdd ZddZZS)FbgemmFp8Linearcst|||||_||_tjtj||f|d|_tjtj|dftj d|_ |j dtjdgtj ddd|rLtjtj|jtj d|_ dSd|_ dS)Ndtyperinput_scale_ubF persistent)superr in_features out_featuresr'r r(zerosweightfloat32r5register_bufferfloatbias)rrGrHrNrB __class__rrrfs  zFbgemmFp8Linear.__init__cCsg|jdddR}t|d|jd|jd\}}|jtj}t r=tj ||j | d| |j|jd}ntjjj||j ||dd}|jdurT||jn|}||j}||}~~|S)Nr)scale_ub)scale_ascale_b out_dtyperNTuse_fast_accum)r$r&r) contiguousrCr5tor'rK_is_torch_xpu_available _scaled_mmrJt unsqueezerBrNopsfbgemmf8f8bf16_rowwisedevicer%)rx output_shape x_quantizedx_scaleweight_scale_float32outputrrrforwardts($   zFbgemmFp8Linear.forward)r7r8r9r' float8_e4m3fnrrg __classcell__rrrOrr@esr@cr?)rcst|j|_|j|_|j|_|j|_t|j|_ t j t j |j|jd|jft jd|_t j t j |jd|jdft jd|_t j t j |j|j|jft jd|_t j t j |j|jdft jd|_|jdt j dgt jddddS)NrrArrCFrD)rFrnum_local_experts num_expertsintermediate_size hidden_size expert_dimr hidden_actact_fnr'r r(rIrhrrKgate_up_proj_scalerdown_proj_scalerLrM)rconfigrBrOrrrs&  "z#FbgemmFp8Llama4TextExperts.__init__c Cs||jd|j}d}t|}t|jD]-}||}|d|j}t|||j\}}|j j dd} |j tj } trtj||j |ddd| |d| |dd| dd|jd} tj||j |dd| d|d| |d| ddd|jd} nNtjjj||j |ddd| || |dd| dddd} tjjj||j |dd| d|| |d| ddddd} | || } t| ||j\}}|j tj }tr"tj||j|dd|d||dd|jd}ntjjj||j|dd|||dddd}|||<q| |j}|d|jS) z Args: hidden_states (torch.Tensor): (batch_size * token_num, hidden_size) Returns: torch.Tensor: (batch_size * token_num, hidden_size) rNrrr)rRrSrTTrU)r)rkrmr' empty_likeranger%r&rCrr$rqrXrKrYrZr#rWr[r\rBr]r^r_rprrrr`)r hidden_states num_tokens next_statesi expert_hiddenexpert_hidden_reshapedexpert_quantized expert_scalesharded_expert_dimgate_up_proj_scale_float32gateup activatedactivated_quantizedactivated_scaledown_proj_scale_float32 expert_outputrrrrgsz  " "  z"FbgemmFp8Llama4TextExperts.forward)r7r8r9r'rKrrgrirrrOrrsrr)maxsizecCs$tr ddlm}|djStjjjS)Nr get_kernelzkernels-community/fp8-fbgemm)rY hub_kernelsrr&r'r]r^rrrrget_quantize_fp8_per_rows   rFmodules_to_not_convertc Cstad}|r inddi}|D]^\}}t||sqd} tdd8|jjdkr9t|jd|j} t | p6|j} nt |t j rSt |j|j|jdufi|} | dWdn1s]wY| durgq||| d}q|swtd|S) a A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules. This will enable running your models using high performance fp8 kernel from FBGEMM library. Parameters: model (`torch.nn.Module`): Input model or `torch.nn.Module` as the function is run recursively. modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`): Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons. quantization_config (`FbgemmFp8Config`): The quantization config object that contains the quantization parameters. pre_quantized (`book`, defaults to `False`): Whether the model is pre-quantized or not FrBNT)include_buffersLlama4TextExperts text_configzYou are loading your model using FP8 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)rr& named_modulesrrrPr7getattrrsrr"r Linearr@rGrHrNrequires_grad_ set_submoduleloggerwarning) rrquantization_config pre_quantizedtp_planhas_been_replaced module_kwargs module_namer- new_modulerrrrreplace_with_fbgemm_fp8_linear s>      r)NNFN)" functoolsr activationsrcore_model_loadingrquantizers.quantizers_utilsrrutilsrr r r r r'r acceleraterrYfbgemm_gpu.experimental.gen_ai fbgemm_gpu get_loggerr7rrrr@r>rrr=r;rrrrrs.        9/l