o
    2h{[                     @   s.  d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ dZG dd deZG dd deejZG dd deejZG dd deZG dd deejZd%ddZd%ddZd%dd ZG d!d" d"ejZG d#d$ d$ej Z!dS )&zcVarious classes representing TPU distributed values.

Note that the tests are in values_test.py .

    )packed_distributed_variable)tpu_replicated_variable)tpu_util)values)values_util)context)tape)ops)gen_resource_variable_ops)math_ops)variable_scopez{op_name} is only supported for distributed variable (variable created within certain `tf.distribute.Strategy` scope) with NONE  aggregation, got: {aggregation}.c                       s   e Zd ZdZ fddZ fddZ fddZdd	 Zed
d Z	edd Z
dd Z fddZ fddZ fddZedd Zd fdd	Z  ZS )TPUVariableMixinzMixin for TPU variables.c                    sH   t t| j|i | t r| jd tt| j | _	d S | j| _	d S )N_)
superr   __init__r	   #executing_eagerly_outside_functions_common_namestrid_primary
_handle_idselfargskwargs	__class__ b/var/www/html/chatgem/venv/lib/python3.10/site-packages/tensorflow/python/distribute/tpu_values.pyr   +   s   zTPUVariableMixin.__init__c                    s,   t  d u rtt| |S td| d)Nz`TPUVariableMixin.z&` not accessible within a TPU context.)r   enclosing_tpu_contextr   r   __getattr__AttributeError)r   namer   r   r   r    5   s
   
zTPUVariableMixin.__getattr__c                    s"   t  d u rtt|  S td)Nz?`TPUVariableMixin.get()` is not supported within a TPU context.)r   r   r   r   getNotImplementedErrorr   r   r   r   r#   <   s
   zTPUVariableMixin.getc                 C   s   |   S N)
read_valuer%   r   r   r   _get_as_operandC   s   z TPUVariableMixin._get_as_operandc                 C   sp   t  }|du st r|  }t|tjr| S |j	S | j
du}| j}|r+| j
g}|| j| j||  |S )z2The handle by which this variable can be accessed.N)r   r   r   executing_eagerly_get_on_device_or_primary
isinstancepackedPackedVarAndDeviceon_device_handlehandle_packed_var_valuesget_replicated_var_handler   r   _is_mirrored)r   tpu_contextvar	is_packedvalr   r   r   r/   F   s   
zTPUVariableMixin.handlec                 C   s   | j jS r&   )r/   devicer%   r   r   r   r8   \   s   zTPUVariableMixin.devicec                 C   sr   | j rt|  | j}t|ddr2t|  j t	|| j
W  d   S 1 s+w   Y  dS t	|| j
S )z!Reads the value of this variable.r6   FN)	trainabler   variable_accessedr/   getattrr	   r8   r*   r
   read_variable_opdtype)r   r/   r   r   r   _read_variable_op`   s   
$z"TPUVariableMixin._read_variable_opc                    "   t  d u rtt|  S |  S r&   )r   r   r   r   r'   r>   r%   r   r   r   r'   m      zTPUVariableMixin.read_valuec                    r?   r&   )r   r   r   r   valuer>   r%   r   r   r   rA   s   r@   zTPUVariableMixin.valuec                    s   t  d u rtt|  S d S r&   )r   r   r   r   _as_graph_elementr%   r   r   r   rB   y   s   z"TPUVariableMixin._as_graph_elementc                 C   s8   t  r| jjS t| jjj| jjj| jjj| jjj	S r&   )
r   is_saving_non_distributedr   opr   DistributedVarOpr"   graph	tracebacktyper%   r   r   r   rD      s   zTPUVariableMixin.opNFc                    sV   t  du rtt| j|||dS |dur"|| jkr"t|  |S |r'| j	S |  S )z Converts a variable to a tensor.N)r=   r"   as_ref)
r   r   r   r   _dense_var_to_tensorr=   r   castr'   r/   )r   r=   r"   rI   r   r   r   rJ      s   
z%TPUVariableMixin._dense_var_to_tensor)NNF)__name__
__module____qualname____doc__r   r    r#   r(   propertyr/   r8   r>   r'   rA   rB   rD   rJ   __classcell__r   r   r   r   r   (   s"    



r   c                   @   st   e Zd ZdZdddZdddZdd	d
ZdddZdddZdddZ	dddZ
dddZdddZdddZdS )TPUDistributedVariablez-DistributedVariable subclass for TPUStrategy.FNTc                 C   0   t  r| j||||S | jj| ||||dS Nuse_lockingr"   r'   )r   rC   r   
assign_sub_policyr   rA   rV   r"   r'   r   r   r   rW      
   
z!TPUDistributedVariable.assign_subc                 C   rS   rT   )r   rC   r   
assign_addrX   rY   r   r   r   r[      rZ   z!TPUDistributedVariable.assign_addc                 C   rS   rT   )r   rC   r   assignrX   rY   r   r   r   r\      rZ   zTPUDistributedVariable.assignc                 C   ,   t  r| j|||S | jj| |||dS N)rV   r"   )r   rC   r   scatter_subrX   r   sparse_deltarV   r"   r   r   r   r_      
   z"TPUDistributedVariable.scatter_subc                 C   r]   r^   )r   rC   r   scatter_addrX   r`   r   r   r   rc      rb   z"TPUDistributedVariable.scatter_addc                 C   r]   r^   )r   rC   r   scatter_mulrX   r`   r   r   r   rd      rb   z"TPUDistributedVariable.scatter_mulc                 C   r]   r^   )r   rC   r   scatter_divrX   r`   r   r   r   re      rb   z"TPUDistributedVariable.scatter_divc                 C   r]   r^   )r   rC   r   scatter_minrX   r`   r   r   r   rf      rb   z"TPUDistributedVariable.scatter_minc                 C   r]   r^   )r   rC   r   scatter_maxrX   r`   r   r   r   rg      rb   z"TPUDistributedVariable.scatter_maxc                 C   r]   r^   )r   rC   r   scatter_updaterX   r`   r   r   r   rh      rb   z%TPUDistributedVariable.scatter_updateFNTFN)rL   rM   rN   rO   rW   r[   r\   r_   rc   rd   re   rf   rg   rh   r   r   r   r   rR      s    








rR   c                       s   e Zd ZdZdd Ze fddZdd	d
ZdddZdddZ	dd Z
dd Zdd Zdd Zdd Zdd Zdd Z  ZS )TPUMirroredVariablezHHolds a map from replica to TPU variables whose values are kept in sync.c                 C   s   t | jtjS )zReturns whether each of the underlying variables is replicated or sharded to logical cores.

    If True, the handles of the underlying variables are not available outside a
    TPU context.
    )r+   r   r   TPUReplicatedVariabler%   r   r   r   *_is_replicated_or_sharded_to_logical_cores   s   z>TPUMirroredVariable._is_replicated_or_sharded_to_logical_coresc                    s(   |   rt d u r| jjS tt| jS r&   )rm   r   r   r   r8   r   rk   r%   r   r   r   r8      s   zTPUMirroredVariable.deviceFNTc                 C   r   t  }|  r|d u rdd }| j|||||dS |r0| jtjjkr0t t	j
| ||||dS t| ||||dS )Nc                 _      | j |i |S r&   )rW   vakar   r   r   <lambda>       z0TPUMirroredVariable.assign_sub.<locals>.<lambda>	update_fnrA   rV   r"   r'   rA   rV   r"   r'   rU   )r   r   rm   _updateaggregationr   VariableAggregationNONEmake_raw_assign_fnr
   assign_sub_variable_oprW   )r   rA   rV   r"   r'   r4   assign_sub_fnr   r   r   rW      2   
zTPUMirroredVariable.assign_subc                 C   rn   )Nc                 _   ro   r&   )r[   rp   r   r   r   rt     ru   z0TPUMirroredVariable.assign_add.<locals>.<lambda>rv   rx   rU   )r   r   rm   ry   rz   r   r{   r|   r}   r
   assign_add_variable_opr[   )r   rA   rV   r"   r'   r4   assign_add_fnr   r   r   r[      r   zTPUMirroredVariable.assign_addc                 C   sv   t  }|  r|d u rdd }| j|||||dS t  r2| jtjjkr2t t	j
| ||||dS t| ||||dS )Nc                 _   ro   r&   )r\   rp   r   r   r   rt     ru   z,TPUMirroredVariable.assign.<locals>.<lambda>rv   rx   rU   )r   r   rm   ry   rz   r   r{   r|   r}   r
   assign_variable_opr\   )r   rA   rV   r"   r'   r4   	assign_fnr   r   r   r\     s2   
zTPUMirroredVariable.assignc                 O      t  r| jj|i |S tr&   )r   rC   r   r_   r$   r   r   r   r   r_   0     zTPUMirroredVariable.scatter_subc                 O   r   r&   )r   rC   r   rc   r$   r   r   r   r   rc   5  r   zTPUMirroredVariable.scatter_addc                 O   r   r&   )r   rC   r   rg   r$   r   r   r   r   rg   :  r   zTPUMirroredVariable.scatter_maxc                 O   r   r&   )r   rC   r   rf   r$   r   r   r   r   rf   ?  r   zTPUMirroredVariable.scatter_minc                 O   r   r&   )r   rC   r   rd   r$   r   r   r   r   rd   D  r   zTPUMirroredVariable.scatter_mulc                 O   r   r&   )r   rC   r   re   r$   r   r   r   r   re   I  r   zTPUMirroredVariable.scatter_divc                 O   r   r&   )r   rC   r   rh   r$   r   r   r   r   rh   N  r   z"TPUMirroredVariable.scatter_updateri   )rL   rM   rN   rO   rm   rP   r8   rW   r[   r\   r_   rc   rg   rf   rd   re   rh   rQ   r   r   r   r   rk      s    	


rk   c                       sR   e Zd ZdZdd Zd fdd	Zd fd	d
	Zd fdd	Z fddZ  Z	S )TPULazyDistributedVariablez:TPU Mirrored variable to be initialized lazily in a batch.c                 C   s$   t | ddrd S | j  d| _d S )N_is_lazily_initializedFT)r;   _lazy_scopeinitialize_allr   r%   r   r   r   _initialize_if_uninitializedW  s   

z7TPULazyDistributedVariable._initialize_if_uninitializedFNTc                       |    t ||||S r&   )r   r   rW   rY   r   r   r   rW   ^     z%TPULazyDistributedVariable.assign_subc                    r   r&   )r   r   r[   rY   r   r   r   r[   d  r   z%TPULazyDistributedVariable.assign_addc                    r   r&   )r   r   r\   rY   r   r   r   r\   j  s   z!TPULazyDistributedVariable.assignc                    s   |    t  S r&   )r   r   r'   r%   r   r   r   r'   q  s   
z%TPULazyDistributedVariable.read_valueri   )
rL   rM   rN   rO   r   rW   r[   r\   r'   rQ   r   r   r   r   r   T  s    r   c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	TPUSyncOnReadVariablezGHolds a map from replica to variables whose values are reduced on save.c                 O   D   t  d u rtjj| g|R i |S t tj| g|R i |S r&   )r   r   r   SyncOnReadVariablerW   r}   r
   r~   r   r   r   r   rW   y     z TPUSyncOnReadVariable.assign_subc                 O   r   r&   )r   r   r   r   r[   r}   r
   r   r   r   r   r   r[     r   z TPUSyncOnReadVariable.assign_addc                 O   r   r&   )r   r   r   r   r\   r}   r
   r   r   r   r   r   r\     s   zTPUSyncOnReadVariable.assignN)rL   rM   rN   rO   rW   r[   r\   r   r   r   r   r   v  s
    r   FNTc                 C       t tj}| j|||||dS Nrv   )r   r}   r
   r~   ry   )r5   rA   rV   r"   r'   r   r   r   r   rW        rW   c                 C   r   r   )r   r}   r
   r   ry   )r5   rA   rV   r"   r'   r   r   r   r   r[     r   r[   c                 C   r   r   )r   r}   r
   r   ry   )r5   rA   rV   r"   r'   r   r   r   r   r\     r   r\   c                   @   s   e Zd ZdZ			dddZ			dddZdd	d
Z		dddZdddZdddZ	dddZ
dddZdddZdddZdddZdS )TPUOnWritePolicyzPolicy defined for `tf.VariableSynchronization.ON_WRITE` synchronization.

  This policy is created when `synchronization` is set to
  `tf.VariableSynchronization.AUTO` or `tf.VariableSynchronization.ON_WRITE`.
  FNTc                 C   B   t  r|jtjjkrt tj|||||dS t	|||||dS Nrx   rU   )
r   r   rz   r   r{   r|   r}   r
   r~   rW   r   r5   rA   rV   r"   r'   r   r   r   rW        
zTPUOnWritePolicy.assign_subc                 C   r   r   )
r   r   rz   r   r{   r|   r}   r
   r   r[   r   r   r   r   r[     r   zTPUOnWritePolicy.assign_addc                 C   r   r   )
r   r   rz   r   r{   r|   r}   r
   r   r\   r   r   r   r   r\     s   
zTPUOnWritePolicy.assignc                 C   sV   t |}t  r"| jtjjkrttj	|| jd|||||dS |j
||||dS )N)op_namerz   )ra   rV   r"   )rw   rA   rV   r"   )r   make_raw_scatter_xxx_fnr   _aggregationr   r{   r|   r$   _scatter_error_msgformatry   )r   raw_scater_xxx_fnr   r5   ra   rV   r"   scater_xxx_fnr   r   r   _scatter_xxx  s"   
zTPUOnWritePolicy._scatter_xxxc                 C      |  tjd||||S )Nr_   )r   r
   resource_scatter_subr   r5   ra   rV   r"   r   r   r   r_        zTPUOnWritePolicy.scatter_subc                 C   r   )Nrc   )r   r
   resource_scatter_addr   r   r   r   rc     r   zTPUOnWritePolicy.scatter_addc                 C   r   )Nrg   )r   r
   resource_scatter_maxr   r   r   r   rg     r   zTPUOnWritePolicy.scatter_maxc                 C   r   )Nrf   )r   r
   resource_scatter_minr   r   r   r   rf     r   zTPUOnWritePolicy.scatter_minc                 C   r   )Nrd   )r   r
   resource_scatter_mulr   r   r   r   rd     r   zTPUOnWritePolicy.scatter_mulc                 C   r   )Nre   )r   r
   resource_scatter_divr   r   r   r   re     r   zTPUOnWritePolicy.scatter_divc                 C   r   )Nrh   )r   r
   resource_scatter_updater   r   r   r   rh     r   zTPUOnWritePolicy.scatter_updateri   rj   )rL   rM   rN   rO   rW   r[   r\   r   r_   rc   rg   rf   rd   re   rh   r   r   r   r   r     s*    	









r   c                       sp   e Zd ZdZ fddZ fddZ fddZdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Z  ZS )TPUOnReadPolicya{  Policy defined for `tf.VariableSynchronization.ON_READ` synchronization.

  This policy is created when `synchronization` is set to
  `tf.VariableSynchronization.ON_READ` and `aggregation` is set to any of the
  values allowed by the `tf.VariableAggregation` enum such as `NONE`, `SUM`,
  `MEAN` or `ONLY_FIRST_REPLICA`when creating a `tf.Variable` in `tf.distribute`
  scope.
  c                    H   t  d u rtt| j|g|R i |S t tj|g|R i |S r&   )r   r   r   r   rW   r}   r
   r~   r   r5   r   r   r   r   r   rW   /     zTPUOnReadPolicy.assign_subc                    r   r&   )r   r   r   r   r[   r}   r
   r   r   r   r   r   r[   7  r   zTPUOnReadPolicy.assign_addc                    r   r&   )r   r   r   r   r\   r}   r
   r   r   r   r   r   r\   ?  s   zTPUOnReadPolicy.assignc                 O      t r&   r$   r   r   r   r   r_   F     zTPUOnReadPolicy.scatter_subc                 O   r   r&   r   r   r   r   r   rc   I  r   zTPUOnReadPolicy.scatter_addc                 O   r   r&   r   r   r   r   r   rg   L  r   zTPUOnReadPolicy.scatter_maxc                 O   r   r&   r   r   r   r   r   rf   O  r   zTPUOnReadPolicy.scatter_minc                 O   r   r&   r   r   r   r   r   rd   R  r   zTPUOnReadPolicy.scatter_mulc                 O   r   r&   r   r   r   r   r   re   U  r   zTPUOnReadPolicy.scatter_divc                 O   r   r&   r   r   r   r   r   rh   X  r   zTPUOnReadPolicy.scatter_update)rL   rM   rN   rO   rW   r[   r\   r_   rc   rg   rf   rd   re   rh   rQ   r   r   r   r   r   %  s    	r   ri   )"rO   tensorflow.python.distributer   r,   r   r   r   r   tensorflow.python.eagerr   r   tensorflow.python.frameworkr	   tensorflow.python.opsr
   r   r   r   objectr   DistributedVariablerR   MirroredVariablerk   r   r   r   rW   r[   r\   OnWritePolicyr   OnReadPolicyr   r   r   r   r   <module>   s.   m@"


r