o
    i                     @   sR  d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZmZmZmZmZmZmZmZ ddl
mZ dd	l
mZ dd
l
mZ ddl
mZ ddl
mZ ddl
mZ ddl
mZ ddl
mZ ddl
mZ ddl
mZ ddl
mZ ddlm Z  e!e"Z#i dg dg dddgdg dg ddg ddg dg d g d!g d"g d#g d$d%gd&d'd(gd)g d*g d+d,d-gi d.g d/g d0g d1d2g d3g d4d5d6gd7g d8g d9d:d;gd<g d=g d>d?g d@dAgdBg dCdDg dEdFdGgdHdIgg dJdKgdLgg dMg dNZ$z	e se W n ey;   ddOl
m%Z% dPdQ e&e%D e$dR< Y nw dSdTge$dU< ze rKe sNe W n eyj   ddVl
m'Z' dWdQ e&e'D e$dX< Y nw dYdZge$dZ< z	e sye W n ey   dd[l
m(Z( d\dQ e&e(D e$d]< Y nw d^ge$d_< z	e se W n ey   dd`l
m)Z) dadQ e&e)D e$db< Y nw dcge$dd< dege$df< dgge$dh< z	e se W n ey   ddil
m*Z* djdQ e&e*D e$dk< Y nw dlge$dm< dnge$do< z	e se W n ey"   ddpl
m+Z+ dqdQ e&e+D e$dr< Y nnw dsge$dt< g e$du< g dve$dw< g dxe$dy< e$d* ,g dz d{d|ge$d}< g d~e$d< g e$d< dge$d< g e$d< ddge$d< g de$d< ddge$d< dge$d< g de$d< ddge$d< g e$d< dge$d< dge$d< dge$d< e
r0ddl-m.Z.m/Z/ ddl0m1Z1 ddl0m2Z2 ddl0m3Z3 ddl0m4Z4 ddl0m5Z5 ddl0m6Z6 ddl0m7Z7 ddl0m8Z8 ddl0m9Z9 ddl0m:Z: ddl;m<Z< ddl;m=Z= ddl>m?Z? ddl>m>Z> ddl@mAZA ddl@mBZB ddl@mCZC ddl@mDZD ddl@mEZE ddl@mFZF ddl@mGZG ddlHmIZI ddlHmJZJ ddlHmKZK ddlHmLZL ddlHmMZM ddlHmNZN ddlHmOZO ddlHmPZP ddlHmQZQ ddlHmRZR ddlHmSZS ddlHmTZT ddlHmUZU ddlHmVZV ddlHmWZW ddlHmXZX ddlHmYZY ddlHmZZZ ddl[m\Z\ ddl[m]Z] ddl[m^Z^ ddl[m_Z_ ddl[m`Z` ddl[maZa ddl[mbZb ddl[mcZc ddl[mdZd ddl[meZe ddl[mfZf ddl[mgZg ddlhmiZi ddlhmjZj ddlhmkZk ddlhmlZl ddlmmnZn ddlompZp ddlomqZq ddlrmsZs ddlrmtZt ddlrmuZu ddlrmvZv ddlrmwZw ddlrmxZx ddlrmyZy ddlrmzZz ddlrm{Z{ ddlrm|Z| ddlrm}Z} ddlrm~Z~ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ dd lrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ ddlrmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dd lmZ dd!lmZ dd"lmZ dd#lT dd$lmZ dd%lmZ dd&lmZ dd'lmZ dd(lmZ dd)lmZ dd*lmZ dd+lmZ dd,lmZ dd-lmZ dd.lmZ dd/lmZ dd0lmZ dd1lmZ dd2lmZ dd3lmZ dd4lmZ dd5lmZ dd6lmZ dd7lmZ dd8lmZ dd9lmZ dd:lmZ dd;lmZ dd<lmZ dd=lmZ dd>lmZ dd?lmZ dd@lmZ ddAlmZ ddBlmZ ddClmZ ddDlmZ ddElmZ ddFlmZ ddGlmZ ddHlmZ ddIlmZ ddJlmZ ddKlmZ ddLlmZ ddMlmZ ddNlmZ ddOlmZ ddPlmZ ddQlmZ ddRlߐm Z  ddSlߐmZ ddTlmZ ddUlmZ ddVlmZ ddWlmZ ddXlm	Z	 ddYl
mZ ddZl
mZ dd[l
mZ dd\l
mZ dd]l
mZ dd^lmZ dd_lmZ dd`lmZ ddalmZ ddblmZ ddclmZ dddlmZ ddelmZ ddflmZ ddglmZ ddhlmZ ddilm Z  ddjl!m"Z" ddkl#m$Z$ ddll#m%Z% ddml#m&Z& ddnl#m'Z' ddol#m(Z( ddpl)m*Z* ddql+m,Z, ddrl
m-Z- ddsl
m.Z. ddtl
m/Z/ ddul
m0Z0 ddvl
m1Z1 ddwl
m2Z2 ddxl
m3Z3 ddyl
m4Z4 ddzl
m5Z5 dd{l
m6Z6 dd|l
m7Z7 dd}l
m8Z8 dd~l
m9Z9 ddl
m:Z: ddl
m;Z; ddl
m<Z< ddl
m=Z= ddl
m>Z> ddl
m?Z? ddl
m@Z@ ddl
mAZA ddl
mBZB ddl
mCZC ddl
mDZD ddl
mEZE ddlmFZF ddlGmHZH ddlImJZJ ddlImKZK ddlImLZL ddlImMZM ddlImNZN ddlImOZO ddlImPZP ddlImQZQ ddlImRZR ddlImSZS ddlImTZT ddlImUZU ddlImVZV ddlImWZW ddlImXZX ddlImYZY ddlImZZZ ddlIm[Z[ ddlIm\Z\ ddl]m^Z^ njdd e$_ D Z$e ee`jad ddZbebeci  de$ ee"ee d ebefde idejge"< dehdehddfddZieie" dd eie" dd e 
se#jd dS dS (  z5.2.0    N)Path)TYPE_CHECKING   )dependency_versions_check)	OptionalDependencyNotAvailable_LazyModuleis_essentia_availableis_g2p_en_availableis_librosa_availableis_mistral_common_availableis_mlx_availableis_numba_availableis_pretty_midi_available)is_bitsandbytes_available)is_scipy_available)is_sentencepiece_available)is_speech_available)is_timm_available)is_tokenizers_available)is_torch_available)is_torchaudio_available)is_torchvision_available)is_vision_available)logging)define_import_structureaudio_utilscliconfiguration_utilsPreTrainedConfigPretrainedConfig+convert_slow_tokenizers_checkpoints_to_fastdata)DataProcessorInputExampleInputFeatures%SingleSentenceClassificationProcessorSquadExampleSquadFeaturesSquadV1ProcessorSquadV2Processorglue_compute_metrics!glue_convert_examples_to_featuresglue_output_modesglue_processorsglue_tasks_num_labels"squad_convert_examples_to_featuresxnli_compute_metricsxnli_output_modesxnli_processorsxnli_tasks_num_labelszdata.data_collator)DataCollatorDataCollatorForLanguageModelingDataCollatorForMultipleChoice*DataCollatorForPermutationLanguageModelingDataCollatorForSeq2SeqDataCollatorForSOP"DataCollatorForTokenClassificationDataCollatorForWholeWordMaskDataCollatorWithFlatteningDataCollatorWithPaddingDefaultDataCollatordefault_data_collatorzdata.metricszdata.processorsdebug_utilsr   dependency_versions_tabledynamic_module_utils!feature_extraction_sequence_utilsSequenceFeatureExtractorfeature_extraction_utilsBatchFeatureFeatureExtractionMixin
file_utils
generation)AsyncTextIteratorStreamerCompileConfigGenerationConfigTextIteratorStreamerTextStreamerWatermarkingConfighf_argparserHfArgumentParserhyperparameter_searchimage_transformsintegrations)is_clearml_availableis_comet_availableis_dvclive_availableis_neptune_availableis_optuna_availableis_ray_availableis_ray_tune_availableis_swanlab_availableis_tensorboard_availableis_trackio_availableis_wandb_availableloss	pipelines)"AnyToAnyPipelineAudioClassificationPipeline"AutomaticSpeechRecognitionPipelineCsvPipelineDataFormatDepthEstimationPipeline!DocumentQuestionAnsweringPipelineFeatureExtractionPipelineFillMaskPipelineImageClassificationPipelineImageFeatureExtractionPipelineImageSegmentationPipelineImageTextToTextPipelineImageToImagePipelineJsonPipelineDataFormatKeypointMatchingPipelineMaskGenerationPipelineNerPipelineObjectDetectionPipelinePipedPipelineDataFormatPipelinePipelineDataFormatQuestionAnsweringPipelineTableQuestionAnsweringPipelineTextClassificationPipelineTextGenerationPipelineTextToAudioPipelineTokenClassificationPipelineVideoClassificationPipelineVisualQuestionAnsweringPipeline#ZeroShotAudioClassificationPipelineZeroShotClassificationPipeline#ZeroShotImageClassificationPipelineZeroShotObjectDetectionPipelinepipelineprocessing_utilsProcessorMixin
quantizerstesting_utilstokenization_pythonPreTrainedTokenizerPythonBackendtokenization_utilstokenization_utils_base)
AddedTokenBatchEncodingCharSpanPreTrainedTokenizerBase	TokenSpantokenization_utils_fast tokenization_utils_sentencepieceSentencePieceBackendtrainer_callback)DefaultFlowCallbackEarlyStoppingCallbackPrinterCallbackProgressCallbackTrainerCallbackTrainerControlTrainerStatetrainer_utils)EvalPredictionIntervalStrategySchedulerTypeenable_full_determinismset_seedtraining_argsTrainingArgumentstraining_args_seq2seqSeq2SeqTrainingArguments)$CONFIG_NAMEMODEL_CARD_NAMESPIECE_UNDERLINEWEIGHTS_NAME
TensorTypeadd_end_docstringsadd_start_docstringsis_apex_availableis_av_availabler   is_datasets_availableis_faiss_availableis_matplotlib_availabler   is_phonemizer_availableis_psutil_availableis_py3nvml_availableis_pyctcdecode_availableis_sacremoses_availabler   r   is_sklearn_availabler   r   r   r   is_torch_hpu_availableis_torch_mlu_availableis_torch_musa_availableis_torch_neuroncore_availableis_torch_npu_availabler   is_torch_xla_availableis_torch_xpu_availabler   r   requires_backendsKernelConfig)
AqlmConfigAutoRoundConfig	AwqConfigBitNetQuantConfigBitsAndBytesConfigCompressedTensorsConfig
EetqConfigFbgemmFp8ConfigFineGrainedFP8ConfigFPQuantConfig
GPTQConfigHiggsConfig	HqqConfigMxfp4ConfigQuantoConfigQuarkConfig
SinqConfig
SpQRConfigTorchAoConfig
VptqConfig)utilszutils.import_utilszutils.kernel_configzutils.quantization_configvideo_utils)dummy_tokenizers_objectsc                 C      g | ]	}| d s|qS _
startswith.0name r   W/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/transformers/__init__.py
<listcomp>  
    
r   zutils.dummy_tokenizers_objectsPreTrainedTokenizerFastTokenizersBackendtokenization_utils_tokenizers)*dummy_sentencepiece_and_tokenizers_objectsc                 C   r   r   r   r   r   r   r   r   %  r   z0utils.dummy_sentencepiece_and_tokenizers_objectsSLOW_TO_FAST_CONVERTERSconvert_slow_tokenizer)dummy_mistral_common_objectsc                 C   r   r   r   r   r   r   r   r   4  r   z"utils.dummy_mistral_common_objectsMistralCommonBackendtokenization_mistral_common)dummy_vision_objectsc                 C   r   r   r   r   r   r   r   r   A  r   zutils.dummy_vision_objectsImageProcessingMixinimage_processing_baseBaseImageProcessorimage_processing_utilsImageFeatureExtractionMixinimage_utils)dummy_torchvision_objectsc                 C   r   r   r   r   r   r   r   r   O  r   zutils.dummy_torchvision_objectsBaseImageProcessorFastimage_processing_utils_fastBaseVideoProcessorvideo_processing_utils)dummy_pt_objectsc                 C   r   r   r   r   r   r   r   r   ]      zutils.dummy_pt_objectsmodel_addition_debugger_contextmodel_debugging_utilsactivations)CacheLayerMixinDynamicLayerStaticLayerStaticSlidingWindowLayerQuantoQuantizedLayerHQQQuantizedLayerCacheDynamicCacheEncoderDecoderCacheQuantizedCacheStaticCachecache_utils)GlueDatasetGlueDataTrainingArgumentsSquadDatasetSquadDataTrainingArgumentszdata.datasets).#AlternatingCodebooksLogitsProcessorBayesianDetectorConfigBayesianDetectorModel%ClassifierFreeGuidanceLogitsProcessorContinuousBatchingManagerContinuousMixin#EncoderNoRepeatNGramLogitsProcessor'EncoderRepetitionPenaltyLogitsProcessorEosTokenCriteriaEpsilonLogitsWarperMinPLogitsWarperEtaLogitsWarperExponentialDecayLengthPenaltyForcedBOSTokenLogitsProcessorForcedEOSTokenLogitsProcessorGenerationMixinInfNanRemoveLogitsProcessorLogitNormalizationLogitsProcessorLogitsProcessorListMaxLengthCriteriaMaxTimeCriteriaMinLengthLogitsProcessor!MinNewTokensLengthLogitsProcessorNoBadWordsLogitsProcessorNoRepeatNGramLogitsProcessor PrefixConstrainedLogitsProcessor RepetitionPenaltyLogitsProcessorSequenceBiasLogitsProcessorStoppingCriteriaStoppingCriteriaListStopStringCriteria$SuppressTokensAtBeginLogitsProcessorSuppressTokensLogitsProcessorSynthIDTextWatermarkDetectorSynthIDTextWatermarkingConfig#SynthIDTextWatermarkLogitsProcessorTemperatureLogitsWarperTopHLogitsWarperTopKLogitsWarperTopPLogitsWarperTypicalLogitsWarper.UnbatchedClassifierFreeGuidanceLogitsProcessorWatermarkDetectorWatermarkLogitsProcessorWhisperTimeStampLogitsProcessor$TorchExportableModuleWithStaticCacheconvert_and_export_with_cachezintegrations.executorch)ChunkConcatenateConversionOpsMergeModulelistPermuteForRopeSplitModulelistWeightConvertercore_model_loadingmodeling_flash_attention_utilsGradientCheckpointingLayermodeling_layersmodeling_outputsBackboneConfigMixinBackboneMixinbackbone_utils)ROPE_INIT_FUNCTIONSdynamic_rope_updateRopeParametersmodeling_rope_utilsPreTrainedModelAttentionInterfacemodeling_utilsAttentionMaskInterfacemasking_utils)	Adafactorget_constant_schedule!get_constant_schedule_with_warmupget_cosine_schedule_with_warmup2get_cosine_with_hard_restarts_schedule_with_warmup+get_cosine_with_min_lr_schedule_with_warmup3get_cosine_with_min_lr_schedule_with_warmup_lr_rateget_inverse_sqrt_scheduleget_linear_schedule_with_warmup)get_polynomial_decay_schedule_with_warmupget_schedulerget_wsd_scheduleget_reduce_on_plateau_scheduleoptimizationConv1Dapply_chunking_to_forwardpytorch_utilstime_series_utilsTrainertrainertorch_distributed_zero_firsttrainer_pt_utilsSeq2SeqTrainertrainer_seq2seq)rK  rL  )r  )r  )r   )r  )r  )r  )r  )r	  )r  )r  )r   )r   )r   )r   )r?  )r@  )rA  )rB  )rC  )rD  )rE  )r"   )r#   )r$   )r%   )r&   )r'   )r(   )r)   )r*   )r+   )r,   )r-   )r.   )r/   )r0   )r1   )r2   )r3   )r4   )r5   )r6   )r7   )r8   )r9   )r:   )r;   )r<   )r=   )r>   )r?   )r  )r  )r  )r  )rD   )rF   )rG   )r  )rJ   )r  )r  )r  )rK   )r  )r  )r  )r  )r  )r  )r  )r  )r  )r  )rL   )r  )r  )r   )r!  )r"  )r#  )r$  )r%  )r&  )r  )r'  )r(  )r)  )r*  )r+  )r,  )r-  )r.  )r/  )r0  )r1  )r2  )r3  )r4  )rM   )rN   )r5  )r6  )r7  )r8  )r9  )r:  )rO   )r;  )r<  )rQ   )r   )r   )r   )r   )rU   )rV   )rW   )rX   )rY   )rZ   )r[   )r\   )r]   )r^   )r_   )r=  )r>  )rU  )r   )rH  )rN  )rP  )rO  )rS  )rR  )*)
MambaCache)TimmWrapperImageProcessor)rW  )rX  )rY  )rZ  )r[  )r\  )r]  )r^  )r_  )r`  )ra  )rb  )rb   )rc   )rd   )re   )rf   )rg   )rh   )ri   )rj   )rk   )rl   )rm   )rn   )ro   )rp   )rq   )rr   )rs   )rt   )ru   )rv   )rw   )rx   )ry   )rz   )r{   )r|   )r}   )r~   )r   )r   )r   )r   )r   )r   )re  )rf  )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )ri  )r   )r   )r   )r   )r   )r   )r   )rk  )rm  )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   )r   c                 C   s   i | ]	\}}|t |qS r   )set)r   kvr   r   r   
<dictcomp>  r   ru  models)prefix__file____version__)module_specextra_objectsaliastargetreturnc                    sl   t | }d d|_fdd  fdd|_ fdd|_|tj| < ttjt | 	dd	d
 | dS )zv
        Lazily redirect legacy tokenization module paths to their replacements without importing heavy deps.
        z.Alias module for backward compatibility with `z`.c                      s   t  tS N)	importlibimport_module__name__r   )r}  r   r   _get_target  s   z/_create_tokenization_alias.<locals>._get_targetc                    s   t   | S r  )getattr)r   r  r   r   <lambda>  s    z,_create_tokenization_alias.<locals>.<lambda>c                      s
   t   S r  )dirr   r  r   r   r    s   
 .r   N)
types
ModuleType__doc____getattr____dir__sysmodulessetattrr  rsplit)r|  r}  moduler   )r  r}  r   _create_tokenization_alias  s   

"r  z.tokenization_utils_fastz.tokenization_utils_tokenizersz.tokenization_utilsz!.tokenization_utils_sentencepiecezxPyTorch was not found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.(k  ry  r  r  r  pathlibr   typingr    r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utils.import_utilsr   
get_loggerr  logger_import_structurer   r  r   r   r   r   r   extendrM  rK  rL  r
  r  r  r   r  r  r  r  r	  r  r  r   r   r   r   r   rF  r?  r@  rA  rB  rC  rD  rE  r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   data.data_collatorr4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   data.datasetsr  r  r  r  rC   rD   rE   rF   rG   rI   r  rJ   r  r  r  rK   r  r  r  r  r  r  r  r  r  r  rL   r  r  r   r!  r"  r#  r$  r%  r&  r  r'  r(  r)  r*  r+  r,  r-  r.  r/  r0  r1  r2  r3  r4  rM   rN   r5  r6  r7  r8  r9  r:  rO   r;  r<  rP   rQ   r   r   r   r   r   r   r   r   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   integrations.executorchr=  r>  rV  rU  r   r   rI  rH  rQ  rN  rP  rO  rT  rS  rR  rv  models.mamba.modeling_mambarp  models.timm_wrapperrq  rd  rW  rX  rY  rZ  r[  r\  r]  r^  r_  r`  ra  rb  ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   r   r   r   rg  re  rf  r   r   r   r   r   r   r   r   r   r   r   r   r   r   rj  ri  r   r   r   r   r   r   r   r   rl  rk  rn  rm  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utils.kernel_configr   utils.quantization_configr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   itemsrx  parentimport_structure	frozensetupdateglobals__spec__r  strr  warning_advicer   r   r   r   <module>   s  ,
'()*+,-./089:;HImnopqryz{       &  P	





5	





"&