o
    2h΁                     @   s  d dl Z d dlm  mZ d dlmZ d dlmZ d dl	m
Z
 d dl	mZ d dl	mZ d dlmZ d dlmZ d d	lmZ d d
lmZ dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdddZdd Zd d! Zdd#d$Zd%d& Zdd(d)Z d*d+ Z!d,d- Z"dd/d0Z#d1d2 Z$dd4d5Z%dd6d7Z&dd9d:Z'd;d< Z(dd=d>Z)d?d@ Z*ddAdBZ+ddCdDZ,ddEdFZ-	GddHdIZ.	GddJdKZ/dLdM Z0dNdO Z1dPdQ Z2		R	ddSdTZ3		R	ddUdVZ4	G	R		GddWdXZ5	G	R		GddYdZZ6	G	R		Gdd[d\Z7	G	R			Gdd]d^Z8dd`daZ9ddbdcZ:ddddeZ;ddfdgZ<ddhdiZ=ddjdkZ>	lddmdnZ?ddodpZ@	3	ddqdrZA	s	t	G	3	 ddudvZBdwdx ZCdydz ZD	_dd{d|ZE				_		dd}d~ZFdS )    N)backend)-compute_conv_transpose_padding_args_for_torch)cast)convert_to_tensor)
get_device)expand_dims)maximum)where)standardize_tuplec                 C      t | } t| S N)r   tnnrelux r   U/var/www/html/chatgem/venv/lib/python3.10/site-packages/keras/src/backend/torch/nn.pyr         
r   c                 C   r   r   )r   r   relu6r   r   r   r   r      r   r   c                 C   r   r   )r   r   sigmoidr   r   r   r   r      r   r   c                 C   sP   t | } t| dktjd| j| jdt| dktjd| j| jdd| d  S )N        devicedtype         ?      ?)r   torchr	   tensorr   r   r   r   r   r   sparse_sigmoid    s   
r    c                 C   r   r   )r   r   tanhr   r   r   r   r!   -   r   r!   c                 C   r   r   )r   r   
tanhshrinkr   r   r   r   tanh_shrink2   r   r#   c                 C   r   r   )r   r   softplusr   r   r   r   r$   7   r   r$   c                 C   r   r   )r   r   softsignr   r   r   r   r%   <   r   r%   r   c                 C      t | } tj| |dS N)lambd)r   r   
softshrinkr   	thresholdr   r   r   soft_shrinkA      r,   c              
   C   s:   t | } t| dkt| t| dk d| d d  | S )Nr   r   g      ?   )r   r   r	   
zeros_liker   r   r   r   sparse_plusF   s   r0   c                 C   r   r   )r   r   silur   r   r   r   r1   O   r   r1      c                 C   s.   t | } t |}| t| d |  }|d S )Nr.   )r   r   sqrt)r   byr   r   r   
squareplusT   s   r6   c                 C   r   r   )r   r   
logsigmoidr   r   r   r   log_sigmoid[   r   r8   皙?c                 C   r&   )N)negative_slope)r   r   
leaky_relu)r   r:   r   r   r   r;   `   r-   r;   c                 C   r   r   )r   r   hardsigmoidr   r   r   r   hard_sigmoide   r   r=   c                 C   r   r   )r   r   	hardswishr   r   r   r   	hard_siluj   r   r?   r   c                 C   s   t | } t| |S r   )r   r   elur   alphar   r   r   r@   o   s   r@   c                 C   r   r   )r   r   selur   r   r   r   rC   t   r   rC   Tc                 C   s$   t | } |rtj| ddS t| S )Nr!   )approximate)r   r   gelu)r   rD   r   r   r   rE   y   s   
rE   c                 C   r&   )N)rB   )r   r   celurA   r   r   r   rF      r-   rF   r   c                 C   r&   )Ndim)r   r   glu)r   axisr   r   r   rI      r-   rI   c                 C   s   t | } tj| dddS )Ng      r   )min_valmax_val)r   r   hardtanhr   r   r   r   	hard_tanh      rN   c                 C   r&   r'   )r   r   
hardshrinkr*   r   r   r   hard_shrink   r-   rQ   c                 C   s   t | } tj| ||dS )N)r+   value)r   r   r+   )r   r+   default_valuer   r   r   r+      rO   r+   c                 C      t | } t| j}t dkrt| jdkrt| d} |d u r6t| dg}tj	|dd}t|| j
}ntj	| |d}t||S Ncpufloat16float32r   rG   )r   r   standardize_dtyper   r   r   r   reshaper   softmaxshaper   rJ   r   outputr   r   r   r[         


r[   c                 C   rT   rU   )r   r   rY   r   r   r   r   rZ   r   log_softmaxr\   r]   r   r   r   r`      r_   r`   c                 C   s   t | }tj||dd\}}tj||d}tjd||d |j|jd}dg|j }d||< |	|}||d |  dk}tj
||dd}	t||tjd	|jd
}
tj
|
|ddd |	 }tj|| d	d}|S )NT)rH   
descendingrG   r   r   r   r   rH   keepdimr   r   )min)r   r   sortcumsumarangesizer   r   ndimviewsumr	   r   clamp)r   rJ   logitslogits_sorted_logits_cumsumrr_shapesupportklogits_cumsum_safetaur^   r   r   r   	sparsemax   s"   
rx   r   c           	      C   sd   |d | d }|dkr|d }n| | d | }t d|d | | |  }|d }|| }||fS )zSCompute padding length along one dimension with support
    for asymmetric padding.r   r   r.   )max)	input_lengthkernel_lengthstridedilation_rateeffective_k_sizetotal_paddingoutput_sizeleft_paddingright_paddingr   r   r   _compute_padding_length   s   
r   c                 C   s   | j dd }t|}g }|dkrt||d}t|D ]}	|dkr#dn||	 }
t||	 ||	 ||	 |
}|| qtdd |D rL| dd	 |D fS g }t|D ]}|| qR|dkr`d
nd}t	j
| t||ddfS )aZ  Apply same padding to the input tensor.

    This function will evaluate if the padding value is compatible with torch
    functions. To avoid calling `pad()` as much as possible, which may cause
    performance or memory issues, when compatible, it does not apply the padding
    to the tensor, but returns the input tensor and the padding value to pass to
    the torch functions. If not compatible, it returns the padded tensor and 0
    as the padding value.

    Returns:
        tensor: A padded tensor or the inputs.
        padding: The padding value, ready to pass to the torch functions.
    r.   Npoolingr}   r   c                 s   s    | ]	\}}||kV  qd S r   r   ).0leftrightr   r   r   	<genexpr>  s    z&_apply_same_padding.<locals>.<genexpr>c                 S   s   g | ]\}}|qS r   r   )r   r   rp   r   r   r   
<listcomp>  s    z'_apply_same_padding.<locals>.<listcomp>	replicateconstant)padmoder   )r\   lenr
   ranger   appendallreversedextendr   r   tuple)inputskernel_sizestridesdata_formatoperation_typer}   spatial_shapenum_spatial_dimspaddingidilr   flattened_paddingr   r   r   r   _apply_same_padding   s(   r   c                 C   sX   | j d }|dkrt| dS |dkrt| dS |dkr#t| dS td| j d)	z=Transpose inputs from channels_last to channels_first format.r.   r   r   r.   r   )r      r   r.   r   )r   r2   r   r.   r   z^Inputs must have ndim=3, 4 or 5, corresponding to 1D, 2D and 3D inputs. Received input shape: .)rj   r   permute
ValueErrorr\   )r   rj   r   r   r   _transpose_spatial_inputs  s   
r   c                 C   V   t | jd }|dkrt| d} | S |dkrt| d} | S |dkr)t| d} | S )Nr.   r   r   )r   r.   r   r   r   )r   r.   r   r2   r   r   r\   r   r   )outputsr   r   r   r   _transpose_spatial_outputs-  s   r   c                 C   r   )Nr.   r   )r.   r   r   )r   r.   r   r   r   )r2   r   r   r   r.   r   )kernelr   r   r   r   _transpose_conv_kernel9  s   r   validc                 C   s  t | } | jd }t||d}|du r|}nt||d}t|}|dkr)t| } |dkr8t| |||d\} }nd}t }|d	krKtj	| j
| jd
d} |dkrYtj| |||d}n%|dkrgtj| |||d}n|dkrutj| |||d}n	td| j
 d||}|dkrt|}|S )z!Fixed max pooling implementation.r.   	pool_sizeNr   channels_lastsamer   r   metarV   )ri   r   r   r   )r   r|   r   r   lInputs to pooling op must have ndim=3, 4 or 5, corresponding to 1D, 2D and 3D inputs. Received input shape: r   )r   rj   r
   r   standardize_data_formatr   r   r   r   emptyr\   r   r   
max_pool1d
max_pool2d
max_pool3dr   tor   )r   r   r   r   r   r   r   r   r   r   r   max_poolF  sP   





r   c                 C   s   t | } | jd }t||d}|du r|nt||d}t|}|}|dkr*t| } |dkr9t| ||dd\} }nd	}|d
krJtj| |||dd}n'|dkrYtj	| |||dd}n|dkrhtj
| |||dd}n	td| j d|dkryt|}|S )z7Fixed average pooling with correct padding calculation.r.   r   Nr   r   r   channels_firstr   r   r   F)r   r|   r   count_include_padr   r   r   )r   rj   r
   r   r   r   r   r   
avg_pool1d
avg_pool2d
avg_pool3dr   r\   r   )r   r   r   r   r   r   orig_formatr   r   r   r   average_pool  sh   



r   c                 C   s:  t | } t |}| jd }t||d}t|}|dkr t| } t|}| jd }|jd }|| dkr?td| d| d|| }	|d	krXt	| |jdd
 ||d|\} }nd}|dkrjt
j| |||||	d}
n)|dkrzt
j| |||||	d}
n|dkrt
j| |||||	d}
n	td| j d|dkrt|
}
|
S )z&Convolution with fixed group handling.r.   r   r   r   r   zInput channels (z.) must be divisible by kernel input channels ()r   Nconv)r|   r   dilationgroupsr   zrInputs to conv operation should have ndim=3, 4, or 5,corresponding to 1D, 2D and 3D inputs. Received input shape: r   )r   rj   r
   r   r   r   r   r\   r   r   r   conv1dconv2dconv3dr   )r   r   r   r   r   r}   r   in_channelskernel_in_channelsr   r   r   r   r   r     s|   	




		r   c                 C   sH   t |}t||jd d d|jd |jd  f }t| |||||S )Nr   r   )r   r   rZ   r\   r   )r   r   r   r   r   r}   r   r   r   depthwise_conv!  s
   &r   c                 C   s&   t | |||||}t||dd||dS )Nr   r   )r   r   r   r}   )r   r   )r   depthwise_kernelpointwise_kernelr   r   r   r}   depthwise_conv_outputr   r   r   separable_conv0  s    	r   c                 C   s  t | } t |}| jd }t||d}t|}t| j|j||||d\}}	|dkr.t| } t|}|jdd  }
t	|t
rE|gt|
 }|dkrUtj| ||||	|d}n)|dkretj| ||||	|d}n|dkrutj| ||||	|d}n	td| j d	|dkrt|}|S )
Nr.   r   )input_shapekernel_shaper   r   output_paddingr}   r   r   )r|   r   r   r   r   z|Inputs to conv transpose operation should have ndim=3, 4, or 5,corresponding to 1D, 2D and 3D inputs. Received input shape: r   )r   rj   r
   r   r   r   r\   r   r   
isinstanceintr   r   conv_transpose1dconv_transpose2dconv_transpose3dr   r   )r   r   r   r   r   r   r}   r   torch_paddingtorch_output_paddingkernel_spatial_shaper   r   r   r   conv_transposeK  sr   	

	
	r   Fc           
      C   s   |rt dt| tjd} tdtjd}tt| d|}tt| dddk||}t||d}|	 }|dkr\||kr\t
t|}d||< t|d |D ]
}	||	  d8  < qL||}|S )N2Unsupported value `sparse=True` with torch backendr   r   r   rJ   r   )r   r   r   longr   one_hotr   r	   r   rH   listr   r   )
r   num_classesrJ   r   sparsezeror^   dimsnew_axes_orderaxr   r   r   r     s   
r   c                 C   sN   |rt dt| } t| jdkrdnd}tjtt| d|||d|d}|S )Nr   r   r   int32)rJ   r   rG   )r   r   r   r\   r   amaxr   r   )r   r   rJ   r   r   reduction_axisr   r   r   r   	multi_hot  s   r   c                 C   s   t | } t |}| j|jkrtd| j d|j t| jdk r-td| j d|j |r7tj||d}n|tj||dd }t|t	
 dt	
  }t|}tj| | |d S )	NQArguments `target` and `output` must have the same shape. Received: target.shape=, output.shape=r   zPArguments `target` and `output` must be at least rank 1. Received: target.shape=rG   Trb   r   )r   r\   r   r   r   r`   r   rl   clipr   epsilonlog)targetr^   from_logitsrJ   log_probr   r   r   categorical_crossentropy  s0   
r   c                 C   s  t | tjd} t |}t| jt|jkr#| jd dkr#tj| dd} t|jdk r2td|j t|j}||= t| j|krMtd| j d|j |rWtj	||d}n|tj
||dd	 }t|t d
t  }t|}t| |j| |d} tj
| | |d S )Nr   r   r   rG   zBArgument `output` must be at least rank 1. Received: output.shape=zcArguments `target` and `output` must have the same shape up until the last dimension: target.shape=r   Trb   r   r   )r   r   r   r   r\   squeezer   r   r   r`   rl   r   r   r   r   r   )r   r^   r   rJ   output_shape_without_class_dimr   r   r   r   sparse_categorical_crossentropy  s6   "

r   c                 C   sr   t | } t |}| j|jkrtd| j d|j |r$tj|| ddS t|t dt  }tj	|| ddS )Nr   r   none)	reductionr   )
r   r\   r   r    binary_cross_entropy_with_logitsr   r   r   r   binary_cross_entropy)r   r^   r   r   r   r   binary_crossentropy  s    r   c                 C   s   |rt dt| } d}t| j}|dkrd}t| d} tj| |dd}tjt| |ddt| }|sCt	||}t	||}|rqt
|ttjjttjj}t
|ttjjttjj}t||}t||}||fS )Nz9Argument synchronized=True is not supported with PyTorch.FrW   TrX   rb   )NotImplementedErrorr   r   rY   r   r   r   meansquarer   r   finforW   re   ry   )r   axeskeepdimssynchronized	need_cast	ori_dtyper   variancer   r   r   moments  sB   


r  MbP?c                 C   s   t | } t |}t |}dgt| j }|jd ||< t||}t||}|d ur6t |}t||}nt|}|d urJt |}t||}nt|}| ||	|
 ||S )Nr   r   )r   r   r\   r   rZ   r/   	ones_likesubtractmul_addrsqrt_muladd_)r   r   r  rJ   offsetscaler   r\   r   r   r   batch_normalization5  s&   

r  c                 C   sn   t | } t |}t |}t |}t|jd}t||}t|dd}tj|dd}tj	|| |||dd}|S )NrX   r   r   r   rG   r   )blankr   )
r   r   result_typer   r   r   	transposer   r`   ctc_loss)r   r^   target_lengthoutput_length
mask_indexr   rn   lossr   r   r   r  S  s"   
r  c                 C   s  t | } t |dd}| j\}}}|d u r|d }tj| dd}t|d}tj| ddd }tj||jdd d d f }	|	|d d d f k}	t|	||}t|	d|}|rv|d d dd f |d d d df k}
t	
|
d	}
t|
||}||k}t|d|}tjtj||jddd
}t||df}t|||}tj|dd
}tj||dd
}tj|ddd d d f  }tj|dd
}||fS )Nr   r   r   r   r   r   rd   r   )r   r   r   r   rG   )r   r\   r   argmaxr   ry   rh   r   r	   r   r   	unsqueezetileargsorttake_along_dimrl   )r   sequence_lengthsmerge_repeatedr  
batch_size
max_lengthr   indicesscoresseqlen_maskrepeatinvalid_maskorderr   r   r   _ctc_greedy_decodej  s8   
(r*  greedyd   c                 C   sX   t | } t| jd}t| |} |dkrt| |||dS |dkr$tdtd| d)NrX   r+  )r!  r  beam_searchzKTorch backend doesn't yet support the beam search strategy for CTCdecoding.zInvalid strategy z2. Supported values are 'greedy' and 'beam_search'.)r   r   r  r   r   r*  r   r   )r   r   strategy
beam_width	top_pathsr!  r  r   r   r   r   
ctc_decode  s"   	

r1  c                 C   sx   | j |j krtd| j  d|j  dt| t|} }t|| jd}t| | d }dt| dt|  }|S )NzInput shapes z and z" must match for PSNR calculation. r   r.      
   )r\   r   r   r   r   r   log10)x1x2rL   msepsnrr   r   r   r8    s   r8  c                 C   s,   t | } | dkrd}nd}t|d | dS )NrW   g    @g̓$Ggffffffr   )r   rY   r   )r   valr   r   r   _get_large_negative  s
   
r:  c           	   
   C   s   zddl m} ddl m} W n ty   |rtdY dS w z|| |||d|d}W n ty=   || |||d|}Y nw |rK||ddu rKtd||dS )	z+Verify the availability of flash attention.r   )
SDPAParams)can_use_flash_attentionzFlash attention is not supported in your current PyTorch version. Please update it by following the official guide: https://pytorch.org/get-started/locally/Fr   TzfFlash attention is not supported with the provided inputs. Please check the warnings for more details.)torch.backends.cudar;  r<  ImportError	TypeErrorRuntimeError)	querykeyrR   mask	is_causalraise_errorr;  r<  spda_paramsr   r   r   _can_use_flash_attention  sF   		

rG  c	              	   C   s  |d urt dt| } t|}t|}t| jdks)t|jdks)t|jdkr:t d| j d|j d|j d|d u r@|nt|dd}|d urVd	}t|d
t| j}d\}	}
t| |	|
} t||	|
}t||	|
}|d u r|t	| ||||}n|du rt	| ||||dd |rtj
jjtj
jjjgd tj
jj| |||||d}W d    n1 sw   Y  n|d ur| }tj
jj|  | | |||d}t||
|	S )Nz7torch's `dot_product_attention` doesn't support `bias`.r2   zG`dot_product_attention` only supports 4D inputs. Received: query.shape=z, key.shape=z, value.shape=r   boolr   Fr   )r   r.   T)rE  )backends)	attn_maskrD  r  )r   r   r   r\   r   r	   r:  r   r  rG  nn	attentionsdpa_kernel
SDPBackendFLASH_ATTENTION
functionalscaled_dot_product_attention
contiguous)rA  rB  rR   biasrC  r  rD  flash_attentionattn_logits_soft_capaxis0axis1attention_outputr   r   r   dot_product_attention  st   *
rY  )r   )r2   )r9   )r   )T)r   )r   )Nr   N)r   r   Nr   )r   r   NNr   )r   NF)Fr   )F)FF)NNr  )r   )TN)r+  r,  r   Tr   )NFF)NNNFNN)Gr   torch.nn.functionalrK  rP  r   	keras.srcr   &keras.src.backend.common.backend_utilsr   keras.src.backend.torch.corer   r   r   keras.src.backend.torch.numpyr   r   r	   #keras.src.utils.argument_validationr
   r   r   r   r    r!   r#   r$   r%   r,   r0   r1   r6   r8   r;   r=   r?   r@   rC   rE   rF   rI   rN   rQ   r+   r[   r`   rx   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r*  r1  r8  r:  rG  rY  r   r   r   r   <module>   s    
	











-
A
M
V



H



 
.


1
 

0