o
    ·2úhb  ã                   @   sT   d dl mZ d dlmZ d dlmZ edƒG dd„ dejƒƒZej 	dej
¡e_dS )	é    )Úops)Úkeras_export)Ú	optimizerzkeras.optimizers.Lambc                       s^   e Zd ZdZ															d‡ fd
d„	Z‡ fdd„Zdd„ Z‡ fdd„Z‡  ZS )ÚLamba9  Optimizer that implements the Lamb algorithm.

    Lamb is a stochastic gradient descent method that
    uses layer-wise adaptive moments to adjusts the
    learning rate for each parameter based on the ratio of the
    norm of the weight to the norm of the gradient
    This helps to stabilize the training process and improves convergence
    especially for large batch sizes.

    Args:
        learning_rate: A float, a
            `keras.optimizers.schedules.LearningRateSchedule` instance, or
            a callable that takes no arguments and returns the actual value to
            use. The learning rate. Defaults to `0.001`.
        beta_1: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            exponential decay rate for the 1st moment estimates. Defaults to
            `0.9`.
        beta_2: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            exponential decay rate for the 2nd moment estimates. Defaults to
            `0.999`.
        epsilon: A small constant for numerical stability.
            Defaults to `1e-7`.
        {{base_optimizer_keyword_args}}

    References:
        - [Yang et al.](https://arxiv.org/pdf/1904.00962)
    çü©ñÒMbP?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?çH¯¼šò×z>NFç®Gáz®ï?Úlambc                    s@   t ƒ jd|||||||	|
|||dœ|¤Ž || _|| _|| _d S )N)Úlearning_rateÚnameÚweight_decayÚclipnormÚ	clipvalueÚglobal_clipnormÚuse_emaÚema_momentumÚema_overwrite_frequencyÚloss_scale_factorÚgradient_accumulation_steps© )ÚsuperÚ__init__Úbeta_1Úbeta_2Úepsilon)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   r   Úkwargs©Ú	__class__r   úT/var/www/html/chatgem/venv/lib/python3.10/site-packages/keras/src/optimizers/lamb.pyr   &   s$   õô
zLamb.__init__c                    s2   | j rdS tƒ  |¡ |  |ddg¡\| _| _dS )zÌInitialize optimizer variables.

        Lamb optimizer has 2 types of variables: momentums and velocities

        Args:
            var_list: list of model variables to build Lamb variables on.
        NÚmomentumÚvelocity)Úbuiltr   ÚbuildÚadd_optimizer_variablesÚ
_momentumsÚ_velocities)r   Úvar_listr   r   r!   r%   J   s   ÿz
Lamb.buildc              
   C   sz  t  ||j¡}t  ||j¡}t  | jd |j¡}t  t  | j|j¡|¡}t  t  | j|j¡|¡}| j|  |¡ }| j	|  |¡ }	|  
|t  t  ||¡d| j ¡¡ |  
|	t  t  t  |¡|	¡d| j ¡¡ t  |d| ¡}
t  t  t  |	d| ¡¡| j¡}t  |
|¡}t  t  t  |d¡¡¡}t  t  t  |d¡¡¡}t  t  |d¡t  t  |d¡|| d¡d¡}|  ||| | ¡ dS )z=Update step given gradient and the associated model variable.é   g      ð?é   r   N)r   ÚcastÚdtypeÚ
iterationsÚpowerr   r   r'   Ú_get_variable_indexr(   Ú
assign_addÚmultiplyÚsubtractÚsquareÚdivideÚaddÚsqrtr   ÚsumÚwhereÚgreaterÚ
assign_sub)r   ÚgradientÚvariabler   ÚlrÚ
local_stepÚbeta_1_powerÚbeta_2_powerÚmÚvÚm_t_hatÚv_sqrtÚupdateÚw_normÚg_normÚratior   r   r!   Úupdate_stepY   sB   ÿÿÿÿþÿ
ýzLamb.update_stepc                    s&   t ƒ  ¡ }| | j| j| jdœ¡ |S )N)r   r   r   )r   Ú
get_configrF   r   r   r   )r   Úconfigr   r   r!   rK   †   s   
ýÿzLamb.get_config)r   r   r   r	   NNNNFr
   NNNr   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r%   rJ   rK   Ú__classcell__r   r   r   r!   r      s(     ñ$-r   z{{base_optimizer_keyword_args}}N)Ú	keras.srcr   Úkeras.src.api_exportr   Úkeras.src.optimizersr   Ú	Optimizerr   rP   ÚreplaceÚbase_optimizer_keyword_argsr   r   r   r!   Ú<module>   s     
ÿ