o iJ@sxddlZddlZddlmZmZmZddlZddlmZ dej fddZ dej fddZ dej fd d Z d eddfd d Zdej fddZdej fddZdej fddZdeefddZ    d,dededeedeedeededeeeffddZGdd d ZGd!d"d"Z d-d#eeefd$eeedeeeed"fffd%d&Z d.d'ed(ed)edefd*d+ZdS)/N)AnyOptionalUnion)_get_device_indexreturncCs\tjdkrtdtjjdd}ntd}|j|_|j |_ |j |_ |j |_|j|_|S)Nwin32 amdhip64_r.dllzlibamdhip64.so)sysplatformctypesCDLLtorchversionhiphipGetErrorStringcuGetErrorStringhipModuleLoadDatacuModuleLoadDatahipModuleGetFunctioncuModuleGetFunctionhipModuleLaunchKernelcuLaunchKernelhipFuncSetAttributecuFuncSetAttribute)librS/sda-disk/www/egybert/egybert_env/lib/python3.10/site-packages/torch/cuda/_utils.py_get_hip_runtime_library s  rcCstjdkr tdStdS)Nrz nvcuda.dllz libcuda.so.1)r r r r rrrr_get_cuda_librarys   rcCtjjrtStSN)rrrrrrrrr_get_gpu_runtime_library sr"resultcCsR|dkrdSt}t}||t||jdur |jnd}td|)NrUnknown CUDA error CUDA error: )r c_char_pr"rbyrefvaluedecode RuntimeError)r#err_strlibcuda error_messagerrr _check_cuda(sr.cCstjdkr ddtjjddtjjdg}td|d}ntd}|j|_ |j |_ |j |_ |j|_|j|_|j|_|j|_|j|_|j|_|j|_|S) Nr0rhiprtcr z libhiprtc.so)r r joinrrrr r hiprtcGetErrorStringnvrtcGetErrorStringhiprtcCreateProgramnvrtcCreateProgramhiprtcDestroyProgramnvrtcDestroyProgramhiprtcCompileProgramnvrtcCompileProgramhiprtcGetCodeSizenvrtcGetPTXSize hiprtcGetCode nvrtcGetPTXhiprtcGetProgramLogSizenvrtcGetProgramLogSizehiprtcGetProgramLognvrtcGetProgramLoghiprtcAddNameExpressionnvrtcAddNameExpressionhiprtcGetLoweredNamenvrtcGetLoweredName) version_strrrrr_get_hiprtc_library4s " rIc Csrttjjdd}tjdkrd|dg}nd|dg}|D]}zt|WSt y4Yq!wt d) N.rrnvrtc64_z0_0.dllz libnvrtc.so.z libnvrtc.soz Could not find any NVRTC library) intrrcudasplitr r r r OSError) major_version nvrtc_libslib_namerrr_get_nvrtc_libraryIs   rScCr r!)rrrrIrSrrrr_get_gpu_rtc_library\srTcs>ddlm}m}dhfdd|D}tjjr|||S)z Get HIPCC/NVCC flags that are compatible with NVRTC compilation. Returns: List of HIPCC/NVCC flags that can be safely used with NVRTC. r)COMMON_HIPCC_FLAGSCOMMON_NVCC_FLAGSz--expt-relaxed-constexprcsg|]}|vr|qSrr.0flagnvrtc_unsupported_flagsrr ssz1_get_gpu_rtc_compatible_flags..)torch.utils.cpp_extensionrUrVrrrextend)rUrVcompatible_flagsrrZr_get_gpu_rtc_compatible_flagses  r`F kernel_source kernel_namecompute_capabilitycuda_include_dirs nvcc_optionsauto_pchc sddl}tddtddffdd }|d}|dur8|j|j} |jjr0| j }n| j | j }g} |jjrI| d|n | d |dd l m} | d } | D] } | d | q_|r}|D] }| d |qp|rt|jjd ksJd|durg}| d|r|D] }| |dqt}| dd|Dt| }tj|| }t}|t|||dddd|d}||||||}|krt}|t|t|j}||td|j t}|!|t|t|j}|"||t}|#||t||jdurJ|j }nd}$t||jjr\|j%n|j}||fS)a Compiles a CUDA kernel using NVRTC and returns the PTX code. Args: kernel_source (str): The CUDA kernel source code as a string kernel_name (str): The name of the kernel function to compile compute_capability (str, None): The compute capability to target (e.g., "86"). If None, will detect from current device. cuda_include_dirs (list, None): List of directories containing CUDA headers nvcc_options (list, None): Additional options to pass to NVRTC auto_pch (bool): Enable automatic precompiled headers (CUDA 12.8+) Returns: Tuple[bytes, str]: The compiled PTX code and mangled kernel name rNr#rcsL|kr$t}|t||jdur|jnd}td|dS)Nr$r%)r r&r5r'r(r)r*)r#r+r- NVRTC_SUCCESSlibnvrtcrr check_nvrtcs  z#_nvrtc_compile..check_nvrtcutf-8z--offload-arch=z--gpu-architecture=sm_) include_pathsrMz-Iz12.8zPCH requires CUDA 12.8+z--pchcSsg|]}|dqS)rk)encoderWrrrr\sz"_nvrtc_compile..z.cuzKernel compilation failed: r/)& torch.cudarTrLrmrMget_device_propertiescurrent_devicerr gcnArchNamemajorminorappendr]rlstrr`r^lenr r&c_void_pr7r'rEr;c_size_trAcreate_string_bufferr(rCr*r)r=r?rGr9raw)rarbrcrdrerfrrj source_bytespropsoptionsrlcuda_include_paths cuda_path directoryoptionnvrtc_compatible_flags num_options options_arrayprog c_kernel_namereslog_sizelogptx_sizeptxc_mangled_name mangled_name ptx_bytesrrgr_nvrtc_compile}s            rc@s2eZdZdejddfddZdeddfdd ZdS) _CudaModulemodulerNcCs||_i|_dSr!)_module_kernels)selfrrrr__init__s z_CudaModule.__init__name _CudaKernelc Cs||jvr |j|Sddlm}|}t}zt|t||j| dt ||j}||j|<|WSt yJ}z t d|d|d}~ww)Nr)r"rkzNo kernel named 'z' in this module) rtorch.cuda._utilsr"r rwr.rr'rrmrr*AttributeError)rrr"r,funckernelerrrrr __getattr__s$     z_CudaModule.__getattr__)__name__ __module__ __qualname__r rwrrurrrrrrsrc @seZdZdZdejdejddfddZ    dd eeeefd eeeefd e e d ede e ddf ddZ deddfddZ dS)rzT Represents a compiled CUDA kernel that can be called with PyTorch tensors. rrrNcCs||_||_d|_dS)Nr)rr_max_shared_mem_bytes)rrrrrrr3s z_CudaKernel.__init__rrrgridblockargs shared_memstreamcCsddl}|jj}|sg}g}g} |D]Y} t| |jr?| js*| jr&| s*t dt | } | | | t | qt| trRt | } | t | qt| tret | } | t | qtdt| t j t| }t| D] \}} t | t j ||<qz|durddl}|j}|dkr|jdks||jkr|jdkrdnd|jd}td |d |d t||j|d|d |d |d|d |d ||j|d dS)a Call the compiled CUDA kernel Args: grid (tuple): Grid dimensions (grid_x, grid_y, grid_z) block (tuple): Block dimensions (block_x, block_y, block_z) args (list): List of arguments to pass to the kernel. PyTorch tensor arguments will be automatically converted to pointers. shared_mem (int): Shared memory size in bytes stream (torch.cuda.Stream): CUDA stream to use. If None, uses current stream. rNz?All tensor arguments must be CUDA tensors or pinned CPU tensorszUnsupported argument type: znot configuredzonly z bytes configuredzKernel requires z' bytes of shared memory (>= 48KB), but ze. Call kernel.set_shared_memory_config(shared_mem) after compilation and before launching the kernel.rr1) rrM_utilsr" isinstanceTensoris_cudais_cpu is_pinned ValueErrorr rwdata_ptrrtr'rLc_intfloatc_double TypeErrortyperv enumeratecastrncurrent_streamrr*r.rr_as_parameter_)rrrrrrrr,processed_argsc_argsargptrrr c_args_arrayiconfigured_msgrrr__call__8sl          z_CudaKernel.__call__shared_mem_bytescCs|dkr ||_dSt}tj}tjjr|jdkrdnd}nt|dd}||kr4t d|d|dd }t | |j ||||_dS) Nrgfx950iishared_memory_per_block_optinzRequested shared memory (z bytes) exceeds device limit (z= bytes). Consider reducing block size or shared memory usage.) rr"rrMrorrrqgetattrr*r.rr)rrr, device_propsmax_shared_mem+cudaFuncAttributeMaxDynamicSharedMemorySizerrrset_shared_memory_configs4  z$_CudaKernel.set_shared_memory_config)rrNrN)rrr__doc__r rwrtuplerLrlistrrrrrrrr.s,   arr kernel_namesc Csddl}t}t|tr|d}t}|j}|t | t ||Wdn1s2wY|s=t |Si}|D]}t}t | t |||dt||||<qA|S)a, Loads a CUDA module from PTX code and returns a module object that can access kernels. Args: ptx (bytes or str): The PTX code to load kernel_names (list, optional): List of kernel names to extract from the module. If None, will return a module object with __getattr__. Returns: object: If kernel_names is None, returns a module object with __getattr__ to access kernels. If kernel_names is provided, returns a dict mapping kernel names to _CudaKernel objects. rNrk)rnr"rrurmr rwrMrr.rr'rrr) rrrr,rrkernelsrrrrr_cuda_load_modules*   rdeviceoptional allow_cpucCst|tr|St|trt|}t|tjr2|r&|jdvr%td|n |jdkr2td|tjsAt|tj jrA|j St |||S)aGet the device index from :attr:`device`, which can be a torch.device object, a Python integer, or ``None``. If :attr:`device` is a torch.device object, returns the device index if it is a CUDA device. Note that for a CUDA device without a specified index, i.e., ``torch.device('cuda')``, this will return the current default CUDA device if :attr:`optional` is ``True``. If :attr:`allow_cpu` is ``True``, CPU devices will be accepted and ``-1`` will be returned in this case. If :attr:`device` is a Python integer, it is returned as is. If :attr:`device` is ``None``, this will return the current default CUDA device if :attr:`optional` is ``True``. )rMcpuz(Expected a cuda or cpu device, but got: rMz!Expected a cuda device, but got: ) rrLrurrrrjit is_scriptingrMidx_torch_get_device_index)rrrrrrrs        r)NNNFr!)FF)r r typingrrrr torch._utilsrrr rrr"rLr.rIrSrTrrur`boolrbytesrrrdictrrrrrsl         1