o
    2h                     @   s  d dl Z d dlZd dl mZ d dlmZ d dlmZ d dlm	Z	 d dlm
Z
 d dlmZ d dlmZ d	d
 Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd~ddZdd Zdd Zdd!d"Zd#d$ Zdd&d'Zd(d) Zd*d+ Zdd-d.Zd/d0 Zdd2d3Z dd4d5Z!dd7d8Z"d9d: Z#d~d;d<Z$d=d> Z%dd?d@Z&ddAdBZ'ddCdDZ(	E	1ddFdGZ)		HddIdJZ*		H	ddKdLZ+		H	ddMdNZ,	E	OddPdQZ-	R	H		RddSdTZ.	R	H		RddUdVZ/	R	H		RddWdXZ0	R	H			RddYdZZ1dd[d\Z2dd]d^Z3dd_d`Z4ddadbZ5ddcddZ6ddedfZ7	gddhdiZ8ddjdkZ9	1	ddldmZ:	n	R	ddodpZ;	q	n	R	1	 ddrdsZ<dtdu Z=dvdw Z>dxdy Z?dzd{ Z@				O		dd|d}ZAdS )    N)lax)backend)+compute_conv_transpose_padding_args_for_jax)cast)convert_to_tensor)	is_tensor)scipyc                 C      t | } t| td| jS N        r   npmaximumarraydtypex r   U/var/www/html/chatgem/venv/lib/python3.10/site-packages/keras/src/backend/numpy/nn.pyrelu      r   c              	   C   s0   t | } tt| td| jtd| jS )Nr         @)r   r   minimumr   r   r   r   r   r   r   relu6   s    r   c                 C   s0   t | } td| jtd| jt|    S N      ?)r   r   r   r   expr   r   r   r   sigmoid   s   (r   c                 C   sN   t | } t| dktd| jt| dktd| jtd| d  | jS )Nr      r         ?r   r   wherer   r   r   r   r   r   sparse_sigmoid"   s   &r#   c                 C   s
   t | S N)r   tanhr   r   r   r   r%   -   s   
r%   c                 C   s   t | } | t|  S r$   )r   r   r%   r   r   r   r   tanh_shrink1   s   r&   c                 C   r	   r
   )r   r   	logaddexpr   r   r   r   r   r   softplus6   r   r(   c                 C   s$   t | } | td| jt|   S r   )r   r   r   r   absr   r   r   r   softsign;   s   r*   r    c                 C   sN   t | |kt j| | | jdt | | k t j| | | jdt jd| jdS )Nr   r   )r   r"   r   r   r   	thresholdr   r   r   soft_shrink@   s   r.   c                 C   sD   t | dkt j| | jdt | dk t jd| d d  | jd| S )Nr   r+   r   g      ?   )r   r"   
zeros_liker   r   r   r   r   r   sparse_plusL   s
   (r1   c                 C   s   t | } | t|  S r$   )r   r   r   r   r   r   siluT      r2      c                 C   s4   t | } t || jd}| t| d |  }|d S )Nr+   r/   )r   r   r   sqrt)r   byr   r   r   
squareplusY   s   r8   c                 C   s   t | } t|   S r$   )r   r(   r   r   r   r   log_sigmoid`   r3   r9   皙?c                 C   s"   t | } t| t|| j|  S r$   r   )r   negative_sloper   r   r   
leaky_relue   s   r<   c                 C   sT   | t d| j t d| j } t | dkt d| jt | dkt d| j| S )Nr   r    r   r   )r   r   r   r"   r   r   r   r   hard_sigmoidj   s    r=   c                 C   s   | t |  S r$   )r=   r   r   r   r   	hard_siluu   s   r>   r   c                 C   s8   t | } t| td| jk| t|| jt|  S r
   )r   r   r"   r   r   expm1r   alphar   r   r   eluy   s   (rB   c                 C   s(   d}d}t | } t|| jt| | S )Ng,x?g2֫?)r   r   r   r   rB   )r   rA   scaler   r   r   selu   s   rD   Tc              
   C   s   t | } |r:tdtj | j}td| jtd| jt|| td| j| d | j     }| | S td| j}| tj	
| | d | j td| j S )Nr/   r    r   gHm?   r   )r   r   r5   piastyper   r   r%   r   specialerf)r   approximatesqrt_2_over_picdfsqrt_2r   r   r   gelu   s(    rN   c                 C   sT   t | } t|| j}t| tjd| jd|tt| tjd| jd|   S Nr   r+   )r   r   r   r   r   r?   r   r@   r   r   r   celu   s
   rP   r   c                 C   sZ   t | } | j| d dkrtd| j d| t| d|\}}|ddt|    S )Nr/   r   z4axis size must be divisible by 2. Received: x.shape=z with axis=r   )r   shape
ValueErrorr   splitr   )r   axisx1x2r   r   r   glu   s   rW   c                 C   s>   t | } td| j}td| j}tjt| ||| jdS )N      r   r+   )r   r   asarrayr   r   clip)r   min_valmax_valr   r   r   	hard_tanh   s   r]   c              	   C   sF   t | } t|| j}tjtt| |k| tjd| jd| jdS rO   )r   r   rY   r   r   r"   r)   r,   r   r   r   hard_shrink   s   "r^   c                 C   s&   t | } t| |k| tj|| jdS )Nr+   r!   )r   r-   default_valuer   r   r   r-      s   r-   c                 C   s.   t | t j| |dd }|t j||dd S NTrT   keepdims)r   r   maxsum)r   rT   exp_xr   r   r   softmax   s   rf   c                 C   s:   t j| |dd}t t | | j|dd}| | | S r`   )r   rc   logr   rd   )r   rT   max_x	logsumexpr   r   r   log_softmax   s   rj   c                 C   s   t | }dtjd| |d }tj||d}td|j| d }dg|j }d||< ||}||d |  dk}tj||dd}t	||d}	tj|	|ddd | }
t
||
 d}|S )	NrX   rT   r   r   r   Tra   r   )r   r   sortcumsumarangerQ   ndimreshaperd   r"   r   )r   rT   logitslogits_sortedlogits_cumsumrr_shapesupportklogits_cumsum_safetauoutputr   r   r   	sparsemax   s   
r{   channels_lastc                 C   sD   t | tr
| f| n| } |s| S |dkrd|  d } | S d|  } | S )Nr|   r   )r   r   )
isinstanceint)r   num_spatial_dimsdata_formatinclude_batch_and_channelsr   r   r   _convert_to_spatial_operand   s   r   validc              
   C   s:   |dvrt d| d| }tt| |||||S )aC  Helper function to define pooling functions.

    Args:
        inputs: input data of shape `N+2`.
        initial_value: the initial value for the reduction.
        reduce_fn: a reduce function of the form `(T, T) -> T`.
        pool_size: a sequence of `N` integers, representing the window size to
            reduce over.
        strides: a sequence of `N` integers, representing the inter-window
            strides (default: `(1, ..., 1)`).
        padding: either the string `same` or `valid`.

    Returns:
        The output of the reduction for each window slice.
    )samer   zInvalid padding 'z', must be 'same' or 'valid'.)rR   upperr   r   r   reduce_window)inputsinitial_value	reduce_fn	pool_sizestridespaddingr   r   r   _pool   s   
r   c                 C   sT   t |}| jd }t|||}|d u r|n|}t|||}t| tj tj|||S )Nr/   )	r   standardize_data_formatro   r   r   r   infr   rc   )r   r   r   r   r   r   r   r   r   max_pool  s   

r   c           	      C   s   t |}| jd }t|||}|d u r|n|}t|||}t| dtj|||}|dkr3|t| S dd t	| j
|D }tt|| jdtj|||}|| S )Nr/   r   r   c                 S   s    g | ]\}}|d kr|nd qS r}   r   ).0ar6   r   r   r   
<listcomp>H  s    z average_pool.<locals>.<listcomp>)r   r   ro   r   r   r   addr   prodziprQ   onesr   )	r   r   r   r   r   r   pooledrQ   window_countsr   r   r   average_pool.  s0   


r   Fc                 C   s   | d }|dkrt td|d }d|d f| }nt td|}d| }|r8|d |d ft t|d  }n|d |d ft t|d  }tj|||dS )z9Create a `lax.ConvDimensionNumbers` for the given inputs.r/   r|   r   r   r   r   )lhs_specrhs_specout_spec)tupleranger   ConvDimensionNumbers)r   r   	transposenum_dimsspatial_dims	inputs_dn	kernel_dnr   r   r   &_convert_to_lax_conv_dimension_numbersV  s   " r   r   c                 C   s   t |}| jd }t||dd}t|||dd}t|||dd}|dkr+| jd }n| jd }|jd }	||	 d	krFtd
| d|	 d||	 }
tt	j
j| t|rV|n| |||||
dS )Nr/   Fr   r   r|   r   r   r   zgThe number of input channels must be evenly divisible by kernel's in_channels. Received input channels z and kernel in_channels z. rhs_dilationdimension_numbersfeature_group_count)r   r   ro   r   r   rQ   rR   r   r   jaxr   conv_general_dilatedr   numpy)r   kernelr   r   r   dilation_rater   r   channelskernel_in_channelsr   r   r   r   convo  sV   



r   c           	      C   s   t |}| jd }t||dd}t|||dd}t|||dd}|dkr*| jd n| jd }tt|r7|n|	 |jd d d||jd  f }t
tjj| ||||||d	S )
Nr/   Fr   r   r|   r   r   r   r   )r   r   ro   r   r   rQ   r   rp   r   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r   r   r   r   depthwise_conv  sH   

r   c                 C   s0   t |}t| |||||}t||dd||dS )Nr   r   )r   r   r   r   )r   r   r   r   )r   depthwise_kernelpointwise_kernelr   r   r   r   depthwise_conv_outputr   r   r   separable_conv  s"   
	r   c           
      C   s   t |}| jd }t| j|j||||d}t||dd}	t|||dd}t|||dd}tt	j
j| t|r9|n| ||||	ddS )Nr/   )input_shapekernel_shaper   r   output_paddingr   Fr   r   T)r   r   r   transpose_kernel)r   r   ro   r   rQ   r   r   r   r   r   r   conv_transposer   r   )
r   r   r   r   r   r   r   r   padding_valuesr   r   r   r   r     sL   
	
r   c           
      C   s   |rt d|d u rd}t| } | j}| d} |s!t| d }| jd }tj||f|d}| dk}d|t|| | | f< ||f }	t||	}|dkrVt|d|}|S )N2Unsupported value `sparse=True` with numpy backendfloat32r   r   r   r+   )	rR   r   rQ   rp   r   rc   zerosrn   moveaxis)
r   num_classesrT   r   sparser   
batch_sizecategoricalvalid_indicesoutput_shaper   r   r   one_hot   s$   


r   c                 C   sN   |rt dt| } t| jdkrdnd}tjtt| d|||d|d}|S )Nr   r   r   int32)rT   r   rk   )rR   r   lenrQ   r   rc   r   r   )r   r   rT   r   r   reduction_axisoutputsr   r   r   	multi_hot<  s   r   c                 C   s   t | } t |}| j|jkrtd| j d|j t| jdk r/td| j d|j |r8t||d}n|t j||dd }t |t	 dt	  }t 
|}t j| | |d S )	NQArguments `target` and `output` must have the same shape. Received: target.shape=, output.shape=r   zPArguments `target` and `output` must be at least rank 1. Received: target.shape=rk   Trb   r   )r   r   rQ   rR   r   rj   rd   rZ   r   epsilonrg   targetrz   from_logitsrT   log_probr   r   r   categorical_crossentropyH  s0   


r   c                 C   s  t j| dd} t |}t| jt|jkr$| jd dkr$t j| dd} t|jdk r3td|j | j|jd d krItd| j d|j |rRt||d}n|t j||d	d
 }t |t	
 dt	
  }t |}t| |j| |d} t j| | |d S )Nr   r+   r   r   rk   zBArgument `output` must be at least rank 1. Received: output.shape=zcArguments `target` and `output` must have the same shape up until the last dimension: target.shape=r   Tr   r   )r   r   r   rQ   squeezerR   rj   rd   rZ   r   r   rg   r   r   r   r   r   sparse_categorical_crossentropyb  s2   
"
r   c                 C   s   t | } t |}| j|jkrtd| j d|j |r"t|}t |t dt  }| t | }|d|  t d|  7 }| S )Nr   r   r   )	r   r   rQ   rR   r   rZ   r   r   rg   )r   rz   r   bcer   r   r   binary_crossentropy~  s    

r   c                 C   s   |rt dt|trt|n|}d}t| j}|dkr$d}t| d} tj	| |dd}tj	t
| |ddt
| }|sJt||}t||}|rxt|ttjjttjj}t|ttjjttjj}t||}t||}||fS )Nz7Argument synchronized=True is not supported with NumPy.Ffloat16Tr   r   ra   )NotImplementedErrorr~   listr   r   standardize_dtyper   r   r   meansquarer   rZ   finfor   minrc   )r   axesrb   synchronized	need_cast	ori_dtyper   variancer   r   r   moments  s.   
 "

r   MbP?c           
      C   s   dgt | j }|jd ||< t||}t||}dt||  }|d ur2t||}|| }| | }	|d urEt||}|	| }	| | |	 S )Nr   r   r   )r   rQ   r   rp   r5   )
r   r   r   rT   offsetrC   r   rQ   invresr   r   r   batch_normalization  s   
r   c                    s  t | dd} t |}t |d}t |d}|j\ }}| j\ }dt|jd}||}dd }	|	||}
|	||}|
|j}
||j}t|dd}|tj|
d	dtj	 }| d d d df | d d d	d f ktj
td
|d d d d ||d	 f }t|d}t| |d}td||}t|d}tj |d	 f|jd }d|d d df< tj |f|jd }dd  fdd}dd }|||df}||||f|\}\}}|d |d }||d< t||d	 d}td|| }|S )Nr   r+   g     jr   c                 S   s<   t |d| j |f }t j| dd} || k }t |S )Nr}   r   rk   )r   rn   rp   ro   expand_dimslogical_not)lengths
max_lengthindices
elem_validr   r   r   _lengths_to_paddings  s   

z&ctc_loss.<locals>._lengths_to_paddingsr   rk   r   )r   r   r   r   r   r/   )r   zbtk,bnk->btnr   r   c                 S   s:   t j| d d d df t | d d dd f |gddS )Nr   r   rk   )r   concatenater'   )phiadded_scorer   r   r   update_phi_score  s   0z"ctc_loss.<locals>.update_phi_scorec           
         s   | \}}|}||  }|\}}}t |d d d df | || }|| }	|	|| d   }	| df}|| d| |  }|| d| |	  }	|	|f|	|ffS )Nr   r   r   )r   r'   rp   )
prevr   prev_phi	prev_emitprev_phi_origlogprob_emitlogprob_phipad	next_emitnext_phir   log_epsilonrepeatr   r   r   	loop_body  s   
zctc_loss.<locals>.loop_bodyc                    sn   |}g }t | D ]}| ||\}}|| qg }tt|d D ] |t fdd|D  q!||fS )Nr   c                    s   g | ]}|  qS r   r   )r   r7   ir   r   r   (      z-ctc_loss.<locals>.np_scan.<locals>.<listcomp>)r   appendr   r   r   stack)finitxscarryysr   r7   resultr   r  r   np_scan   s    zctc_loss.<locals>.np_scanr   r   zbn,bn->b)r   rQ   r   result_typer   rG   rj   r   rd   r   r   r  r   r   einsumr   )r   rz   target_lengthoutput_length
mask_indexmax_input_lengthr   max_label_lengthr   r   target_paddingsoutput_paddingslogprobslabel_lengthslogprobs_phi_one_hotlogprobs_emitlogalpha_phi_initlogalpha_emit_initr  r  r  _logalpha_philogalpha_emitlogalpha_phi_lastper_seq_lossr   r	  r   ctc_loss  s\   





0
r/  c                 C   sv  t | } t |dd}| j\}}}|d u r|d }tj| ddd}tj| dd}t|d d d f }	|	|d d d f k}	t|	||}t|	d|}|ro|d d dd f |d d d df k}
t|
d}
t|
||}||k}t|d|}tj	t|dd}t
||df}t|||}tj|dd}tj||dd}tj|ddd d d f  }tj	|dd}||fS )	Nr   r+   r   r   rk   r   )r   r  r   )r   rQ   r   argmaxrG   rc   rn   r"   r  r   tileargsorttake_along_axisrd   )r   sequence_lengthsmerge_repeatedr  r   r   r   r   scoresseqlen_maskrepeat_maskinvalid_maskorderr   r   r   _ctc_greedy_decode;  s2   (r;  d   c                    s8  t | } t |}| j\}}	t| dd} t|d d d f |d d d f k}d u r/	d tj| dd} 	 d dtj|d |ftjd}tt		g}	tj
| d d df ddd d |	 d f }
t|
k|
}||d d d |	df< tj|d ftj | jd}tj| d d df |
dd|d d d |	f< |d d d d df k}	fddd	d
 	fddfddfdd	
fdd  fddt|||| |D }tdd |D }tdd |D }t|k	| d }t|g d}||fS )Nr   rk   r   r/   r+   r   c           
         s   t j| dd} t |}t |}t j|  kdd}t d  }| ||d f }t |dk |}t } |< t |d }|}| k}| ||k@ }	t |	 |}|| ||f< t |d }|| }| ||fS )Nr   rk   r   r/   )r   r  r0  rn   r"   r1  )
pathsr6  maskedr   path_tail_indexpaths_arange
path_tailsclassesprev_maskedmasked_repeat)_pad
beam_widthr  r   r   r   _extend_paths  s$   

z._ctc_beam_search_decode.<locals>._extend_pathsc                 S   sZ   t |}t || }t |}t| D ]\}}||  || 7  < qt || }|S r$   )r   rc   r   r0   	enumeraterg   )unique_inverser6  
scores_max
scores_expr  ur   r   r   _merge_scores  s   

z._ctc_beam_search_decode.<locals>._merge_scoresc           	         s2  t j| ddd\} }d  t|  }|dkr&t j| d|gddggd} | d d   } t|jdkr>t j|dd}t |t j |}t ||t j } ||} ||}t ||}t j	|dd	 d  }| | } || }|| }t 
| d
} t ||g}t t tt tg}| ||fS )NTr   return_inverserT   r/   constant_valuesr   rk   stable)kind)r/   r   )r   uniquer   r  rQ   r   r"   r   r'   r2  r1  r   r   boolr   )	r=  r6  r>  rI  pad_sizeemit_scoresmask_scorestotal_scorestop_indices)rM  rE  rF  r   r   r   _prune_paths  s,   


z-_ctc_beam_search_decode.<locals>._prune_pathsc                    s0    | |||\} }}| ||\} }}| ||fS r$   r   )r=  r6  r>  r   )rG  r[  r   r   _decode_step  s   
z-_ctc_beam_search_decode.<locals>._decode_stepc                    s8   | \}}}|\}}|s ||||\}}}|||fd fS r$   r   )r   r   r=  r6  r>  r7  )r\  r   r   _step  s
   
z&_ctc_beam_search_decode.<locals>._stepc                    s   dd }|| ||f|dd  |dd  f\\}}}}	t j|ddd\}}
d  t| }|dkrBt j|d|gddggd}|d d   }t|
jdkrZt j|
dd	}
 |
|}t | d  d d d
 }|| }|| }||fS )Nc                 S   s(   |}t | D ]	}| ||\}}q|d fS r$   )r   )r  r  r  r  r   r7   r   r   r   np_scan_only_carry  s   zJ_ctc_beam_search_decode.<locals>._decode_batch.<locals>.np_scan_only_carryr   Tr   rN  r/   rP  rk   r   )r   rT  r   r  rQ   r   r2  )
init_pathsinit_scoresinit_maskedr   r7  r^  r=  r6  r>  r*  rI  rV  rZ  )rM  rE  r]  rF  r   	top_pathsr   r   _decode_batch  s$   
z._ctc_beam_search_decode.<locals>._decode_batchc                    s&   g | ]\}}}}} |||||qS r   r   )r   psmr  sm)rc  r   r   r     s    z+_ctc_beam_search_decode.<locals>.<listcomp>c                 S      g | ]}|d  qS r   r   r   rt   r   r   r   r     r  c                 S   rh  r}   r   rj  r   r   r   r     r  r   )r   rQ   rj   r   rn   flipfullr   r   r   r2  r"   r   r   r3  r   r  r   )r   r4  rF  rb  r  r   max_seq_lenr7  r_  num_init_pathsmax_classesinit_classesr`  ra  resultsr=  r6  r   )rc  r\  rG  rM  rE  r[  r]  rF  r  r   rb  r   _ctc_beam_search_decodef  sP   &,	

rr  greedyc                 C   sb   t | } t| jd}t| |} |dkrt| |||dS |dkr)t| ||||dS td| d)Nr   rs  )r5  r  beam_search)rF  rb  r  zInvalid strategy z2. Supported values are 'greedy' and 'beam_search'.)r   r   r  r   r   r;  rr  rR   )r   r4  strategyrF  rb  r5  r  r   r   r   r   
ctc_decode  s*   	

rv  c                 C   sh   | j |j krtd| j  d|j  dt||jd}tt| | }dt| dt|  }|S )NzInput shapes z and z" must match for PSNR calculation. r+      
   )rQ   rR   r   r   r   r   r   log10)rU   rV   r\   msepsnrr   r   r   r{  .  s   r{  c                 C   s,   t | } | dkrdnd}tj|d | dS )Nr   g    @g̓$Ggffffffr+   )r   r   r   rY   )r   valr   r   r   _get_large_negative;  s   
r}  c                 C   s   |d u r|s| S t j| t jd}|d urt ||}|rF| jd | jd }}t t j||ft jd}|d d d d d d f }t ||}t || t| j	}|S )Nr+   r/   rE   )
r   	ones_likebool_logical_andrQ   trilr   r"   r}  r   )rq   mask	is_causalcombined_maskTSpadded_logitsr   r   r   _apply_masksA  s   r  c                 C   s   |j }t| j tj}t|j dkr|d}|d}td| |}	|	|}	|	tj||	j d9 }	|d ur?|	| |	j }	t	|	||}
|
tj}
t
|
dd|}|j }t|j dkri|d}|d}td||}||}|S )Nbfloat16r   zBTNH,BSNH->BNTSr+   r   rk   zBNTS,BSNH->BTNH)r   r   promote_typesr   r   r   rG   r  r   r  rf   )querykeyvaluebiasr  r  rC   original_dtypelogits_dtyperq   r  probsencoded_dtypeencodedr   r   r   _dot_product_attention_xlaU  s(   





r  c	                 C   s   |d u rd}|rt dt| } t|}t|}t| jdkr0t d| j d|j d|j d|j\}	}	}	}
|d u rBdt|
 n|}t| ||||||S )	NFz2Flash attention is not supported in numpy backend.r4   zG`dot_product_attention` only supports 4D inputs. Received: query.shape=z, key.shape=z, value.shape=.r   )rR   r   r   rQ   r   r5   r  )r  r  r  r  r  rC   r  flash_attentionattn_logits_soft_capr*  Hr   r   r   dot_product_attentionr  s,   r  )r    )r4   )r:   )r   )T)r   )r|   T)Nr   )Nr   N)r|   F)r   r   Nr   )r   r   NNr   )r   NF)Fr   )F)FF)NNr   ri  )TN)r<  r   N)rs  r<  r   Tr   )NNNFNN)Br   r   r   r   	keras.srcr   &keras.src.backend.common.backend_utilsr   keras.src.backend.numpy.corer   r   r   keras.src.utils.module_utilsr   r   r   r   r#   r%   r&   r(   r*   r.   r1   r2   r8   r9   r<   r=   r>   rB   rD   rN   rP   rW   r]   r^   r-   rf   rj   r{   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r/  r;  rr  rv  r{  r}  r  r  r  r   r   r   r   <module>   s    	







	




+

*

7
3


2




$

t
.
 )
#!