o
    2hk                  	   @   sb  d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlmZ! ddl"m#Z# ddl$m%Z& ddl'm(Z( ddl)m*Z* da+da,e
j-Z-dd Z.e.ej/ej0Z0e.ej1ej2Z2e.ej3ej4Z4e
j5Z5			dddZ6d d! Z7d"d# Z8dd%d&Z9ej:d'd(d)Z;de;fd*d+Z<	$dd,d-Z=e=d$d.e9dd/d0Z>e=d$d.e9dd/dd1Z?e=d$d2e9dd/d0Z@ejAd3ejBdd/ZCejAd4d5d6 dd/ZejAd7d8d6 d.d/ZDejAd9d:d6 dd/ZEejAd;d<d6 d.d/ZFejAd=e6d(d>d$d?ZejAd@e6d(d$dAd$d?ZGejAdBe6d(d$dCd$d?ZHejAdDe6d.d>d$d?ZIejAdEe6d(d$dFd$d?ZJejAdGe6d.d$dFd$d?ZKejAdHe6d(d>d$d$dIZLeAdJdKd6 ZMejAdLdMd6 d.d/ZNejAdNdOd6 d.d/ZOejAdPdQd6 dd/ZPejAdRdSd6 d(d/ZQejAdTdUd6 d(dVZReAdWdXd6 ZSdYeS_ ejAdZd[d6 d(d/ZTejAd\d]d6 d.d/ZUejAd^e9dd/d$d.e>d$d_ZVejAd`e9d.d/d$d.d.e>ddaZWejAdbe9d.d/d$d.d.e?ddaZXejAdce9d(d/d$d.d(e>d$ddZYejAdee9d(ddfd$d.d(e>d$dgZZejAdhe9dd/d$d2e@d$d_Z[de;fdidjZ\e\dkd2d(dlZ]e\dmd.d(dlZ^e\dnd2d(d.doZ_e\dpd.d(d.doZ`dqdrgZadsdt ZbeCeeDeOeQeUgZceeDeOeQgZdeeIeGeLgZeedee Zfecee ZgeOeQeVeWeeIeUgZheYe[gZieOeQeeIgZjeVeWeYgZkdudv Zldwdx Zmdydz Znd{d| Zod}d~ Zpe*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd dS )z1Strategy combinations for combinations.combine().    N)
config_pb2)tf2)central_storage_strategycluster_resolver)collective_all_reduce_strategy)combinations)distribute_lib)mirrored_strategy)multi_process_runner)multi_worker_test_base)one_device_strategy)parameter_server_strategy_v2)sharded_variable)	test_util)tpu_strategy)tpu_cluster_resolver)context)remote)device)errors)flags)device_assignment)
server_lib)	tf_exportFc                    s    fdd}|S )Nc                     s$   t  r| i |S  | i |S N)r   enabled)argskwargstf1_clstf2_cls m/var/www/html/chatgem/venv/lib/python3.10/site-packages/tensorflow/python/distribute/strategy_combinations.pycreator5   s   z!_version_chooser.<locals>.creatorr"   )r    r!   r$   r"   r   r#   _version_chooser4   s   r%   c                    s    fdd}|S )Nc               	      s  t j} zt }d}W n+ ty6   d}tjt| dr| jpdt| dr'| jp(d t| dr0| jp1d d}Y nw t	sMt
| ddsA|rHt| da	t|ad }rYtjttjd}t rjtj||fd	i}ntj||fi } r}r}td
 |_|S )NTFtpu zoneproject)r&   r(   r)   )core_assignment"experimental_spmd_xla_partitioningz0Packed Variable is not compatible with SPMD mode)r   FLAGSr   TPUClusterResolver
ValueErrorhasattrr&   r(   r)   _did_connect_to_clustergetattrr   connect_to_clusterinitialize_tpu_system	_topologydevice_assignment_libDeviceAssignmentSINGLE_CORE_ASSIGNMENTr   r   tpu_libTPUStrategyV2TPUStrategyV1%_enable_packed_variable_in_eager_mode)r,   resolverdid_automatically_resolver   strategyenable_packed_variableenable_spmd_xla_paritioningr   steps_per_runuse_single_corer"   r#   _create_tpu_strategyU   sT   


z7_get_tpu_strategy_creator.<locals>._create_tpu_strategyr"   )rB   rC   r@   rA   r   rD   r"   r?   r#   _get_tpu_strategy_creatorN   s   5rE   c                 C   sd   t dd | D }|tt dkrt  td| tj j	d7  _	tj
 j	d7  _	t
| S )Nc                 s   s&    | ]}t j|jd krdV  qdS )CPU   N)	tf_device
DeviceSpecfrom_stringdevice_type).0dr"   r"   r#   	<genexpr>   s    z>_mirrored_strategy_with_collective_key_base.<locals>.<genexpr>rF   順 )sumlenr   list_logical_devices_reset_contextr   set_logical_devices_to_at_leastmirrored_libMirroredStrategyV1_collective_key_baseMirroredStrategy)devicesrequired_cpus_numsr"   r"   r#   +_mirrored_strategy_with_collective_key_base   s   r[   c                 C   s8   t j jd7  _t j jd7  _t| }dd |j_|S )NrO   c                   S      dS NFr"   r"   r"   r"   r#   <lambda>       z7_mirrored_strategy_with_no_merge_call.<locals>.<lambda>)rU   rV   rW   rX   extended_use_merge_call)rY   outr"   r"   r#   %_mirrored_strategy_with_no_merge_call   s
   rc   Tc                    s   fdd  fdd}|S )Nc               	      s   t  } |  }| jr|td| j d  }t j|  | j| j|| j	d i| jp)dd}dt
_t jdd t  t|d}W d    n1 sKw   Y  sXd	d
 |j_z	t   W |S  tyk   Y |S w )Nz%s://GPUgrpc)cluster_spec	task_typetask_idmasterenvironmentnum_accelerators	rpc_layerFr'   )service_typer   c                   S   r\   r]   r"   r"   r"   r"   r#   r^      r_   z[_get_multi_worker_mirrored_creator.<locals>._create_multi_worker_mirrored.<locals>.<lambda>)r   TFConfigClusterResolverri   rl   rQ   SimpleClusterResolverrf   rg   rh   rj   CollectiveAllReduceExtended_enable_check_healthr   configure_coordination_service
eager_modeCollectiveAllReduceStrategyr`   ra   r   get_barrierwaitr.   )	tf_configri   r<   r>   required_gpususe_merge_callr"   r#   _create_multi_worker_mirrored   s8   
zI_get_multi_worker_mirrored_creator.<locals>._create_multi_worker_mirroredc               
      sV   z  W S  t jy* }  zd| jv r%ttjdkr%dtjd v r%td d } ~ ww )NCould not start gRPC serverrG   bazelr   Cannot start std servers.)r   UnknownErrormessagerQ   sysargvunittestSkipTest)e)r{   r"   r#    skip_if_cannot_start_grpc_server   s   

zL_get_multi_worker_mirrored_creator.<locals>.skip_if_cannot_start_grpc_serverr"   )ry   rz   r   r"   )r{   ry   rz   r#   "_get_multi_worker_mirrored_creator   s   -r         )min_shard_bytes
max_shardsc                    s    dd   fdd}|S )Nc                 S   s   t j| |dS )N)variable_partitioner)r   ParameterServerStrategyV2)r<   r   r"   r"   r#   _create_ps_strategy   s   z5_get_ps_strategy_creator.<locals>._create_ps_strategyc               
      s   t  rtjdd} tjt| didd} |S t }|	 
 } | s,d S tjt| di|j|j|j|jp?dd}|jdv ryt }d|_ztj| |j|jd|d}W n tjyt } zd	|jv rotd
 d }~ww |   |S )Nre   )num_workersnum_psrl   rd   )rk   rl   )rk   rg   rh   rj   rl   )workerps   )job_name
task_indexprotocolconfigr|   r~   )framework_test_utilis_xla_enabledr   create_in_process_clusterr   ro   r   ClusterSpecrn   rf   as_dictrg   rh   rj   rl   r   ConfigProtointer_op_parallelism_threadsServerr   r   r   r   r   join)cluster_defr<   rw   worker_configserverr   r   r   r   ry   r   r"   r#   _create_parameter_server   sR   





z:_get_ps_strategy_creator.<locals>._create_parameter_serverr"   )r   r   ry   r   r   r"   r   r#   _get_ps_strategy_creator   s   =r   c                    s   g   fdd}|S )a  Returns a callable that returns the pool runner.

  It creates the pool runner only upon first invocation. This avoids creating it
  when this file is imported.

  Args:
    has_chief: whether there should be a chief.
    num_workers: the number of workers excluding the chief.
    initializer: initializer of each process.
    share_gpu: whether to share GPU between the workers.

  Returns:
    A callable that returns the runner.
  c                     s8    st jddd} tj| d} |  d S )Nr   F)	has_chiefr   r   has_eval)initializer	share_gpu)r   create_cluster_specr   MultiProcessPoolRunnerappend)rf   runner	containerr   r   r   r   r"   r#   get_or_createL  s   
z,_deferred_pool_runner.<locals>.get_or_creater"   )r   r   r   r   r   r"   r   r#   _deferred_pool_runner8  s   r   rG   )ry   )r   r   r   )r   r   r   r      DefaultOneDeviceCPUc                   C      t dS N/cpu:0OneDeviceStrategyr"   r"   r"   r#   r^   w      r^   OneDeviceGPUc                   C   r   N/gpu:0r   r"   r"   r"   r#   r^   z  r   OneDeviceOnWorker1CPUc                   C   r   )Nz"/job:worker/replica:0/task:1/cpu:0r   r"   r"   r"   r#   r^   ~  r   OneDeviceOnWorker1GPUc                   C   r   )Nz"/job:worker/replica:0/task:1/gpu:0r   r"   r"   r"   r#   r^     r   TPU)rB   )required_tpuTPUPackedVar)rB   r@   
TPUUseSPMD)rB   rA   
TPUOneStep
TPUOneCore)rB   rC   TPUOneStepOneCoreCloudTPU)r   use_cloud_tpuMirrored1CPUc                   C   
   t dgS r   r[   r"   r"   r"   r#   r^        
 Mirrored1GPUc                   C   r   r   r   r"   r"   r"   r#   r^     r   MirroredCPUAndGPUc                   C      t ddgS Nr   r   r   r"   r"   r"   r#   r^         Mirrored2CPUsc                   C   r   )Nr   /cpu:1r   r"   r"   r"   r#   r^     r   Mirrored2GPUsc                   C   r   Nr   z/gpu:1r   r"   r"   r"   r#   r^     r   Mirrored2GPUsNoMergeCallc                   C   r   r   )rc   r"   r"   r"   r#   r^     r   )required_physical_gpusMirrored2CPUc                   C   r   )Nr   z/cpu:2r   r"   r"   r"   r#   r^     r   zYMirrored strategy with 2 virtual CPUs.

    Should set up logical devices before use
    CentralStorage2GPUsc                   C   r   r   CentralStorageStrategyr"   r"   r"   r#   r^     r   CentralStorageCPUAndGPUc                   C   r   r   r   r"   r"   r"   r#   r^     r   MultiWorkerMirrored2x1CPU)r   r   pool_runner_fnno_xlaMultiWorkerMirrored2x1GPU)r   r   ry   r   r    MultiWorkerMirrored2x1GPUNoShareMultiWorkerMirrored2x2GPU)r   r   ry   r   r   $MultiWorkerMirrored2x2GPUNoMergeCallrx   )r   r   r   r   r   MultiWorkerMirrored4x1CPUc                 C   s"   t j| t||||d||d|dS )N)r   r   ry   r   T)ry   r   r   r   )r   NamedDistributionr   )namer   r   ry   r   r"   r"   r#   parameter_server_strategy_fn  s   r   ParameterServer3Worker2PSCPU)r   r   ParameterServer1Worker2PSCPUParameterServer3Worker2PS1GPU)r   r   ry   ParameterServer1Worker2PS1GPUgrapheagerc                 C   s   t d|  d S )NrF   )r   rT   )num_virtual_cpusr"   r"   r#   set_virtual_cpus_to_at_least>     r   c                   C   s   t jtddgdS Nr   r   distributionmode)r   combinestrategies_minus_tpur"   r"   r"   r#   strategy_minus_tpu_combinations|  s   r   c                   C   s   t jtdgdS )Nr   r   )r   r   tpu_strategiesr"   r"   r"   r#   tpu_strategy_combinations  r   r   c                   C      t  t  S r   )r   r   r"   r"   r"   r#   all_strategy_combinations  s   r   c                   C   s   t jttttgddgdS r   )r   r   r   one_device_strategy_gpu"mirrored_strategy_with_gpu_and_cpumirrored_strategy_with_two_gpusr"   r"   r"   r#   /all_strategy_minus_default_and_tpu_combinations  s   r   c                   C   r   r   )r   r   r"   r"   r"   r#   'all_strategy_combinations_minus_default  s   r  zN__internal__.distribute.combinations.central_storage_strategy_with_gpu_and_cpu)v1)central_storage_strategy_with_gpu_and_cpuzK__internal__.distribute.combinations.central_storage_strategy_with_two_gpus&central_storage_strategy_with_two_gpusz7__internal__.distribute.combinations.cloud_tpu_strategycloud_tpu_strategyz5__internal__.distribute.combinations.default_strategydefault_strategyzG__internal__.distribute.combinations.mirrored_strategy_with_cpu_1_and_2"mirrored_strategy_with_cpu_1_and_2zD__internal__.distribute.combinations.mirrored_strategy_with_two_cpusmirrored_strategy_with_two_cpuszG__internal__.distribute.combinations.mirrored_strategy_with_gpu_and_cpur   zC__internal__.distribute.combinations.mirrored_strategy_with_one_cpumirrored_strategy_with_one_cpuzC__internal__.distribute.combinations.mirrored_strategy_with_one_gpumirrored_strategy_with_one_gpuzD__internal__.distribute.combinations.mirrored_strategy_with_two_gpusr   zR__internal__.distribute.combinations.mirrored_strategy_with_two_gpus_no_merge_call-mirrored_strategy_with_two_gpus_no_merge_callzB__internal__.distribute.combinations.multi_worker_mirrored_2x1_cpumulti_worker_mirrored_2x1_cpuzB__internal__.distribute.combinations.multi_worker_mirrored_2x1_gpumulti_worker_mirrored_2x1_gpuzJ__internal__.distribute.combinations.multi_worker_mirrored_2x1_gpu_noshare%multi_worker_mirrored_2x1_gpu_nosharezB__internal__.distribute.combinations.multi_worker_mirrored_2x2_gpumulti_worker_mirrored_2x2_gpuzP__internal__.distribute.combinations.multi_worker_mirrored_2x2_gpu_no_merge_call+multi_worker_mirrored_2x2_gpu_no_merge_callz8__internal__.distribute.combinations.one_device_strategyr   z<__internal__.distribute.combinations.one_device_strategy_gpur   z1__internal__.distribute.combinations.tpu_strategyr   zN__internal__.distribute.combinations.parameter_server_strategy_3worker_2ps_cpu)parameter_server_strategy_3worker_2ps_cpuzN__internal__.distribute.combinations.parameter_server_strategy_1worker_2ps_cpu)parameter_server_strategy_1worker_2ps_cpuzO__internal__.distribute.combinations.parameter_server_strategy_3worker_2ps_1gpu*parameter_server_strategy_3worker_2ps_1gpuzO__internal__.distribute.combinations.parameter_server_strategy_1worker_2ps_1gpu*parameter_server_strategy_1worker_2ps_1gpuz:__internal__.distribute.combinations.tpu_strategy_one_coretpu_strategy_one_corez<__internal__.distribute.combinations.tpu_strategy_packed_vartpu_strategy_packed_var)FFF)T)NT)s__doc__r   r   tensorflow.core.protobufr   tensorflow.pythonr   tensorflow.python.distributer   r   r   r   r	   r
   rU   r   r   r   one_device_libr   r   r   r   r8   -tensorflow.python.distribute.cluster_resolverr   tensorflow.python.eagerr   r   tensorflow.python.frameworkr   rH   r   r   tensorflow.python.platformr   tensorflow.python.tpur   r5   tensorflow.python.trainingr    tensorflow.python.util.tf_exportr   r0   r4   rp   r%   rV   rX   CentralStorageStrategyV1r   OneDeviceStrategyV1r   rt   rE   r[   rc   r   MinSizePartitionerDEFAULT_PARTITIONERr   r   _two_worker_pool_two_worker_pool_noshare_four_worker_poolr   _get_default_strategyr  r   one_device_strategy_on_worker_1#one_device_strategy_gpu_on_worker_1r  tpu_strategy_spmdtpu_strategy_one_stepr  tpu_strategy_one_step_one_corer  r	  r
  r   r  r   r  r  r  r  r  r  r  r  r  multi_worker_mirrored_4x1_cpur   r  r  r  r  graph_and_eager_modesr   r    strategies_minus_default_and_tpur   all_strategies_minus_defaultall_strategiestwo_replica_strategiesfour_replica_strategiesmultidevice_strategiesmultiworker_strategiesr   r   r   r   r  export_constant__name__r"   r"   r"   r#   <module>   s$  	
?
	@
L
$




	
	



























