o iw`@s0ddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z ddl m Z mZddlZddlZddlZddlmZmZddlmZddlmZzddlZWn eyaYnwedkrlddlZerdd lmZdd lm Z dd l!m"Z"d d dZ#e$ej%ej&Z'e$ej%ej(ej&dZ)ddiddiddiddiddiddiddidZ*dZ+dd,e*-dZ.GdddZ/Gdd d Z0d!e1dBd"efd#d$Z2d%e3e1dBd"e4fd&d'Z5d5d(e1dBd"e3e4fd)d*Z6d+e1d,e3e4d-e4d"e1fd.d/Z7d"e1fd0d1Z8e9d2kre0d3d4dSdS)6N) AsyncIterator) AnnotatedAny)urljoinurlparse)AsyncInferenceClientChatCompletionStreamOutput)GenerationConfig)is_rich_availableWindows)Console)Live)Markdown localhosti@)hostnameportz .!\"#$%&'()*+,\-/:<=>?@[]^_`{|}~textz5There is a Llama in my lawn, how can I get rid of it?zyWrite a Python function that integrates any Python function f(x) numerically over an arbitrary interval [x_start, x_end].z4How many helicopters can a human eat in one sitting?z4Count to 10 but skip every number ending with an 'e'zWhy aren't birds real?z2Why is it important to eat socks after meditating?z$Which number is larger, 9.9 or 9.11?)llamacode helicopternumbersbirdssocksnumbers2a **TRANSFORMERS CHAT INTERFACE** Chat interface to try out a model. Besides chatting with the model, here are some basic commands: - **!help**: shows all available commands (set generation settings, save chat, etc.) - **!status**: shows the current status of the model and generation settings - **!clear**: clears the current conversation and starts a new one - **!exit**: closes the interface am **TRANSFORMERS CHAT INTERFACE HELP** Full command list: - **!help**: shows this help message - **!clear**: clears the current conversation and starts a new one - **!status**: shows the current status of the model and generation settings - **!example {NAME}**: loads example named `{NAME}` from the config and uses it as the user input. Available example names: `z`, `a#` - **!set {ARG_1}={VALUE_1} {ARG_2}={VALUE_2}** ...: changes the system prompt or generation settings (multiple settings are separated by a space). Accepts the same flags and format as the `generate_flags` CLI argument. If you're a new user, check this basic flag guide: https://huggingface.co/docs/transformers/llm_tutorial#common-options - **!save {SAVE_NAME} (optional)**: saves the current chat and settings to file by default to `./chat_history/{MODEL_ID}/chat_{DATETIME}.yaml` or `{SAVE_NAME}` if provided - **!exit**: closes the interface c@seZdZdedefddZdeedeeeeBdBffdd Z defd d Z d d Z defddZ dedefddZ ddededefddZddefddZdefddZdS) RichInterfacemodel_iduser_idcCst|_||_||_dSN)r _consolerr)selfrrr W/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/transformers/cli/chat.py__init__gs zRichInterface.__init__streamreturnNc s|jd|jdt|jddg}d}d}|IdH2zT3dHW}|jdjj}t|jdd|}|s6qt dd |}||7}g}| D]}| || d rY| d qG| d qGt d|d d} |j| ddq6Wdn1s}wY|j||fS)Nz [bold blue]:)consolerefresh_per_secondr finish_reasonz <(/*)(\w*)>z\<\1\2\>z``` z z github-dark) code_themeT)refresh)rprintrr choicesdeltacontentgetattrresub splitlinesappend startswithrjoinstripupdate) rr#liverr)tokenoutputslineslinemarkdownr r r! stream_outputls0      *zRichInterface.stream_outputcCs$|jd|jd}|j|S)z!Gets user input from the console. [bold red]: )rinputrr-)rrBr r r!rBs zRichInterface.inputcCs|jdS)zClears the console.N)rclear)rr r r!rCszRichInterface.clearrcCs(|jd|jd||jdS)z%Prints a user message to the console.rAz>:[/ bold red] N)rr-r)rrr r r!print_user_messagesz RichInterface.print_user_messagecolorcCs&|jd|d||jdS)z,Prints text in a given color to the console.z[bold ]N)rr-)rrrEr r r! print_colorzRichInterface.print_colorFmessagedefaultcCsL|rdnd}|jd|d|d}|j|}|s"|S|dvS)zFDisplays a yes/no prompt to the user, returning True for confirmation.zY/nzy/Nz [bold yellow]z (z): >yyes)rrBr-r8lower)rrIrJ default_hintresponser r r!confirms   zRichInterface.confirmminimalcCs&|jt|rtnt|jdS)z'Prints the help message to the console.N)rr-rHELP_STRING_MINIMAL HELP_STRING)rrQr r r! print_helprHzRichInterface.print_helpconfigcCs6|jd|jd|jd||jdS)zFPrints the status of the model and generation settings to the console.z[bold blue]Model: r*z [bold blue]N)rr-r)rrUr r r! print_statusszRichInterface.print_status)F)__name__ __module__ __qualname__strr"rrtuplerr@rBrCrDrGboolrPrTr rVr r r r!rfs&0 rc@sTeZdZdZdeddedddddddfdeeejd d fd eedBejd d fd ee edBejdd fdeedBej dd fdeedBej dd fdeeej dd fdeedBej dd fdeedBej dd fddfddZ e ddZ deded eeeeeffd!ed"e edee eeff d#d$Zd%d&ZdS)'Chat(Chat with a model from the command line.zhttp://r:rNz./chat_history/rz9ID of the model to use (e.g. 'HuggingFaceTB/SmolLM3-3B').)helpbase_urlz7Base url to connect to (e.g. http://localhost:8000/v1).generate_flagsaFlags to pass to `generate`, using a space as a separator between flags. Accepts booleans, numbers, and lists of integers, more advanced parameterization should be set through --generation-config. Example: `transformers chat max_new_tokens=100 do_sample=False eos_token_id=[1,2]`. If you're a new user, check this basic flag guide: https://huggingface.co/docs/transformers/llm_tutorial#common-optionsuserzKUsername to display in chat interface. Defaults to the current user's name. system_promptzSystem prompt. save_folderzFolder to save chat history. examples_pathz"Path to a yaml file with examples.generation_configzPath to a local generation config file or to a HuggingFace repo containing a `generation_config.json` file. Other generation settings passed as CLI arguments will be applied on top of this generation config.r$c Cs||_t|j} | jtdkr| jtdkr||j||_||_||_t |} | j ddd| j d it || |_ |||j d|_|durM|nt|_|rnt|} t| |_Wdn1shwYnt|_tsxtdt|dS) r^rrT) do_samplemax_new_tokens)rarrUNzHYou need to install rich to use the chat interface. (`pip install rich`)r )rarrDEFAULT_HTTP_ENDPOINTr check_healthrrdreload_generation_configr9parse_generate_flagsrUto_dictsettings get_usernamercopenyaml safe_loadexamplesDEFAULT_EXAMPLESr ImportErrorasynciorun _inner_run) rrrarbrcrdrerfrgparsedrUfr r r!r"s,#   z Chat.__init__cCsbt|dd}zt|}|jdkrtd|d|jdWd Stjy0td|dw) N/healthzThe server running on z returned status code z on health check (/health).zNo server currently running on z. To run a local server, please run `transformers serve` in aseparate shell. Find more information here: https://huggingface.co/docs/transformers/servingT)rhttpxget status_code ValueError ConnectError)url health_urloutputr r r!rls    zChat.check_health user_input interfacerurUchatc Csd}|dkrt|j}|n|dkr|n|drYt|dkrY|}t|dkr4|dntj |j |j dt dd }t|||jd |jd |d d dn|dr|dd} | } | D]} d| vr|jd| dddn ql|jd"it| nd|drt|dkr|d} | |vr|g}||| d|d|| ddn1d| dt|d} |j| ddn|dkr|j|dnd}|jd |d!dd||||fS)#z Handles all user commands except for `!exit`. May update the chat history (e.g. reset it) or the generation config (e.g. set a new flag). T!clear!help!savechat_%Y-%m-%d_%H-%M-%S.jsonfilenamerrpChat saved to !greenrrE!setr%N=(Invalid flag format, missing `=` after `;`. Please use the format `arg_1=value_1 arg_2=value_2 ...`.red!examplerrcroler0Example * not found in list of available examples: .!statusrUF'/' is not a valid command. Showing help message.r )new_chat_historyrdrCrTr6lensplitospathr7rertimestrftime save_chatrprGr8r9rnrDr5listkeysrV) rrrrurUr valid_command split_inputrnew_generate_flagsflag example_name example_errorr r r!handle_non_exit_user_commands$sV           z"Chat.handle_non_exit_user_commandsc sTt|j|jd}|t|j}|jdd|j}t|j d4IdHv}d} zZ|dur:|}d}| |n| }|dkrEWnL|dkrTt|j}|Wq)|dkr^|Wq)| drt |d kr|}t |d kry|d ntj|j|jd td d }t|||jd|jd|dddWq)| dr|dd} | } | D]} d| vr|jd| dddn q|jd,it| Wq)| dr!t |d kr!|d } | |jvr |g}| |j| d|d|j| ddnBd| dt|jd} |j| ddn,|d kr.|j|d!Wq)| drE|jd"|d#dd|Wq)|d|d|j |d|j|!|jd$d%} |"| IdH\}}|d&|d|d'kr|d(d)|#d*rd+}Wq)Wn t$yYnwq*WdIdHdS1IdHswYdS)-N)rrT)rQ)raz!exitrrrrrrrrrrrrrrr%rrrrrrrcrrrrrrrr)rgmodel)r#r extra_body assistantlengthz2Generation stopped after reaching the token limit.yellowzContinue generating?u'Please continue. Do not repeat text.”r )%rrrcrCrrdrTrUrrarDrBr6rrrrr7rerrrrprGr8r9rnrur5rrrVchat_completionto_json_stringr@rPKeyboardInterrupt)rrrrUclientpending_user_inputrrrrrrrr# model_outputr)r r r!rzns                  0zChat._inner_run)rWrXrY__doc__rkrrZtyperArgumentrOptionr" staticmethodrlrdictr r[rrzr r r r!r]s|    ! F  Jr]rgr$cCsD|durtSd|vrtj|}tj|}t||St|S)Nr)r rrdirnamebasenamefrom_pretrained)rgrrr r r!rms    rmrbcs|dus t|dkr iSdd|D}dd|D}dd|D}dtdtfd d fd d|D}d d d|D}d|d}|dd}|dd}|dd}|dd}|dd}|dd}zt|}W|Stjyt dw)zUParses the generate flags from the user input into a dictionary of `generate` kwargs.NrcSs.i|]}d|ddd|ddqS)"rrr)r).0rr r r! s.z(parse_generate_flags..cSs*i|]\}}||dvr|n|qS))truefalse)rMrkvr r r!rscSs"i|] \}}||dkr dn|qS)Nonenullr rr r r!rs"sr$cSs|d}|dddS)N-rr(r) removeprefixreplaceisdigit)rr r r! is_numbers z'parse_generate_flags..is_numbercs*i|]\}}||sd|dn|qS)rr rrr r!rs*z, cSsg|] \}}|d|qS)z: r rr r r! sz(parse_generate_flags..{}z"null"rz"true"rz"false"rz"[[z]"rFrr_zFailed to convert `generate_flags` into a valid JSON object. `generate_flags` = {generate_flags} Converted JSON string = {generate_flags_string}) ritemsrZr\r7rjsonloadsJSONDecodeErrorr)rbgenerate_flags_as_dictgenerate_flags_stringprocessed_generate_flagsr rr!rns2        rnrdcCs|rd|dgSgS)z Returns a new chat conversation.systemrr )rdr r r!r$srrrrpcCsbtjtj|ddt|d}tj||d|ddWdn1s&wYtj|S)z!Saves the chat history to a file.T)exist_okw)rp chat_historyr%)indentN)rmakedirsrrrrrdumpabspath)rrrpr|r r r!r)s   rcCs$tdkr tSttjS)z)Returns the username of the current user.r )platformrrgetloginpwdgetpwuidgetuidpw_namer r r r!rq1s rq__main__z meta-llama/Llama-3.2-3b-Instruct)rr):rxrrrr2stringrcollections.abcrtypingrr urllib.parserrrrrshuggingface_hubrr transformersr transformers.utilsr readlinerwrr rich.consoler rich.liver rich.markdownrrkset ascii_letters whitespaceALLOWED_KEY_CHARSdigitsALLOWED_VALUE_CHARSrvrRr7rrSrr]rZrmrrrnrrrqrWr r r r!st            b 6