o
    l¨Êhq.  ã                   @   sœ  d dl mZmZmZ d dlZd dlZd dlmZ d dlmZ ddl	m
Z
mZmZ ddlmZ ddlmZmZ e d	¡Zd d
lmZmZ d dlmZ d dlmZmZ dd„ ZdCdd„Zdd„ Zdd„ Zdd„ Zdd„ ZdDdd„Z G dd„ de!ƒZ"dd„ Z#d d!„ Z$d"d#„ Z%d$d%„ Z&d&d'„ Z'd(d)„ Z(d*d+„ Z)d,d-„ Z*d.d/„ Z+d0d1„ Z,d2d3„ Z-d4d5„ Z.d6d7„ Z/ej0 1d8d9d:g¡ej0j2ej3d;ko¿ej3d<kd=d>�d?d@„ ƒƒZ4dAdB„ Z5dS )Eé    )Úprint_functionÚdivisionÚabsolute_importN©Úrandom)Úsleepé   )ÚParallelÚdelayedÚparallel_backend)ÚThreadingBackend)ÚDaskDistributedBackendÚTimeoutErrorÚdistributed)ÚClientÚLocalCluster)Útime)ÚclusterÚincc                  O   s   d S ©N© )ÚargsÚkwargsr   r   úG/var/www/html/env/lib/python3.10/site-packages/joblib/test/test_dask.pyÚnoop   s   r   çš™™™™™©?c                 C   s   t |ƒ | r
tdƒ‚d S )Nzcondition evaluated to True)r   Ú
ValueError)Ú	conditionÚdurationr   r   r   Úslow_raise_value_error   s   ÿr   c                 C   sF  t ƒ �–\}\}}t|d | d��q}tdƒ�]\}}tƒ dd„ tdƒD ƒƒ}|dd„ tdƒD ƒks1J ‚t t¡� tƒ d	d„ tdƒD ƒƒ W d   ƒ n1 sMw   Y  tƒ d
d„ tdƒD ƒƒ}|dd„ tdƒD ƒkskJ ‚W d   ƒ n1 suw   Y  W d   ƒ n1 s„w   Y  W d   ƒ d S W d   ƒ d S 1 sœw   Y  d S )NÚaddress©ÚloopÚdaskc                 s   ó   � | ]	}t tƒ|ƒV  qd S r   ©r
   r   ©Ú.0Úir   r   r   Ú	<genexpr>    ó   € ztest_simple.<locals>.<genexpr>é
   c                 S   ó   g | ]}t |ƒ‘qS r   ©r   r&   r   r   r   Ú
<listcomp>!   ó    ztest_simple.<locals>.<listcomp>c                 s   s    � | ]}t tƒ|d kƒV  qdS )é   N)r
   r   r&   r   r   r   r)   $   s   € ÿc                 s   r$   r   r%   r&   r   r   r   r)   '   r*   c                 S   r,   r   r-   r&   r   r   r   r.   (   r/   )r   r   r   r	   ÚrangeÚpytestÚraisesr   ©r"   ÚsÚaÚbÚclientÚbaÚ_Úseqr   r   r   Útest_simple   s$   

ÿÿ÷€ÿÿ"ÿr<   c                   C   s   t ƒ S r   r   r   r   r   r   Úrandom2+   s   r=   c           	   
   C   sÈ   t ƒ �W\}\}}t|d | d��2}tdƒ�\}}tƒ dd„ tdƒD ƒƒ\}}||ks,J ‚W d   ƒ n1 s6w   Y  W d   ƒ n1 sEw   Y  W d   ƒ d S W d   ƒ d S 1 s]w   Y  d S )Nr    r!   r#   c                 s   ó   � | ]}t tƒƒ V  qd S r   )r
   r=   r&   r   r   r   r)   3   s   € z3test_dont_assume_function_purity.<locals>.<genexpr>r   ©r   r   r   r	   r1   )	r"   r5   r6   r7   r8   r9   r:   ÚxÚyr   r   r   Ú test_dont_assume_function_purity/   s   þ€ÿÿ"ÿrB   c              
   C   sä   t ƒ �e\}\}}t|d | d��@}tdƒ�\}}tƒ dd„ tdƒD ƒƒ\}}W d   ƒ n1 s0w   Y  dd„ }	| |	¡}
td	d„ |
D ƒƒsIJ ‚W d   ƒ n1 sSw   Y  W d   ƒ d S W d   ƒ d S 1 skw   Y  d S )
Nr    r!   r#   c                 s   r$   r   r%   r&   r   r   r   r)   ;   r*   z%test_dask_funcname.<locals>.<genexpr>r   c                 S   s
   t | jƒS r   )ÚlistÚtransition_log)Údask_schedulerr   r   r   Úf=   ó   
ztest_dask_funcname.<locals>.fc                 s   s   � | ]
}|d    d¡V  qdS )r   z	inc-batchN)Ú
startswith)r'   Útupr   r   r   r)   @   s   € )r   r   r   r	   r1   Úrun_on_schedulerÚall)r"   r5   r6   r7   r8   r9   r:   r@   rA   rF   Úlogr   r   r   Útest_dask_funcname7   s   ÿ
ùÿ"ÿrM   c                 C   s   | | | | | S r   r   )r6   r7   ÚcÚdÚer   r   r   Úadd5C   s   rQ   c                   @   s(   e Zd Zdd„ Zdd„ ZeZdd„ ZdS )ÚCountSerializedc                 C   s   || _ d| _d S )Nr   )r@   Úcount)Úselfr@   r   r   r   Ú__init__H   s   
zCountSerialized.__init__c                 C   s   | j t|d|ƒ S )Nr@   )r@   Úgetattr)rT   Úotherr   r   r   Ú__add__L   s   zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Né   )rS   rR   r@   )rT   r   r   r   Ú
__reduce__Q   s   zCountSerialized.__reduce__N)Ú__name__Ú
__module__Ú__qualname__rU   rX   Ú__radd__rZ   r   r   r   r   rR   G   s
    rR   c                 C   s   t dƒ}t dƒ}t dƒ}tƒ �œ\}\}}t|d | d��€}td||gd��<\}}	ttƒ}
|
|||dd	d
�|
|||d	dd
�|
||||d	d
�|
|||||d
�g}dd„ |D ƒ}tƒ |ƒ}W d   ƒ n1 sdw   Y  t t	¡� td| dd�� W d   ƒ n1 s�w   Y  W d   ƒ n1 s�w   Y  W d   ƒ n1 sŸw   Y  W d   ƒ n1 s®w   Y  ||ks¹J ‚|j
dksÀJ ‚|j
dksÇJ ‚|j
dksÎJ ‚d S )NrY   r   r0   r    r!   r#   )Úscatteré   é   )rO   rP   c                 S   s    g | ]\}}}||i |¤Ž‘qS r   r   )r'   Úfuncr   r   r   r   r   r.   c   s    ÿz'test_manual_scatter.<locals>.<listcomp>)r"   r_   )rR   r   r   r   r
   rQ   r	   r2   r3   Ú	TypeErrorrS   )r"   r@   rA   Úzr5   r6   r7   r8   r9   r:   rF   ÚtasksÚexpectedÚresultsr   r   r   Útest_manual_scatterV   s>   ýÿøÿ€ÿ€ô€ÿrh   c              
      s  t  d¡}|jtdƒ|jd�‰ |jtdƒ|jd�}ˆ gd |gd  }dd„ }tƒ �[\}\}}t|d | d��?}td	ƒ�\}	}
tƒ d
d„ t	|ƒD ƒƒ W d   ƒ n1 sUw   Y  |d|ƒ}||d  ||d   dksoJ ‚W d   ƒ n1 syw   Y  W d   ƒ n1 sˆw   Y  tƒ �j\}\}}t|d | d��E}td	ƒ�\}	}
tƒ ‡ fdd„t
dƒD ƒƒ W d   ƒ n1 s½w   Y  |d|ƒ}||d  dksÑJ ‚||d  dksÛJ ‚W d   ƒ n1 såw   Y  W d   ƒ d S W d   ƒ d S 1 sýw   Y  d S )NÚnumpyg     ˆÃ@)Údtyper`   c                    sF   |  dd„ ¡}i }| ¡ D ]\}}t‡ fdd„t|ƒD ƒƒ||< q|S )Nc                 S   s   | j S r   )rL   )Údask_workerr   r   r   Ú<lambda>{   s    z9test_auto_scatter.<locals>.count_events.<locals>.<lambda>c                    s   g | ]
}|d  ˆ kr|‘qS )rY   r   )r'   Úevent©Ú
event_namer   r   r.   ~   s    ÿz;test_auto_scatter.<locals>.count_events.<locals>.<listcomp>)ÚrunÚitemsÚlenrC   )ro   r8   Úworker_eventsÚevent_countsÚwÚeventsr   rn   r   Úcount_eventsz   s
    z'test_auto_scatter.<locals>.count_eventsr    r!   r#   c                 s   s(   � | ]\}}t tƒ||||d �V  qdS ))ÚoptN©r
   r   )r'   r(   Údatar   r   r   r)   ‡   s   € ÿz$test_auto_scatter.<locals>.<genexpr>zreceive-from-scatterr   c                 3   s&   � | ]}t tƒˆ d d… |ƒV  qd S )Nr0   ry   r&   ©Údata1r   r   r)   ’   s   €$ ra   r   )r2   ÚimportorskipÚonesÚintÚuint8r   r   r   r	   Ú	enumerater1   )r"   ÚnpÚdata2Údata_to_processrw   r5   r6   r7   r8   r9   r:   Úcountsr   r{   r   Útest_auto_scattert   s:   


ÿý
"ö€ÿÿ
ùÿ"ÿr†   c           	   
      s€  dd„ ‰ t ƒ �¯\}\}}t|d | d��=}tdƒ�)\}}tdd�‡ fdd	„td
ƒD ƒƒ}|D ]}tt|ƒƒdks:J ‚q.W d   ƒ n1 sEw   Y  W d   ƒ n1 sTw   Y  t|d | d��=}tdƒ�)\}}tdd�‡ fdd	„td
ƒD ƒƒ}|D ]}tt|ƒƒdks‡J ‚q{W d   ƒ n1 s’w   Y  W d   ƒ n1 s¡w   Y  W d   ƒ d S W d   ƒ d S 1 s¹w   Y  d S )Nc                  S   sH   t tdd�dd„ tdƒD ƒƒƒ} | t tdd�dd„ tdƒD ƒƒƒO } | S )Nr   ©Ún_jobsc                 s   ó   � | ]	}t tjƒƒ V  qd S r   ©r
   ÚosÚgetpid©r'   r:   r   r   r   r)   œ   r*   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   r‰   r   rŠ   r�   r   r   r   r)   �   r*   )Úsetr	   r1   )Úpidsr   r   r   Úget_nested_pids›   s    $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr    r!   r#   r   r‡   c                 3   ó   � | ]}t ˆ ƒƒ V  qd S r   ©r
   r�   ©r�   r   r   r)   £   ó
   € ÿ
ÿz6test_nested_backend_context_manager.<locals>.<genexpr>r+   c                 3   r‘   r   r’   r�   r“   r   r   r)   ­   r”   )r   r   r   r	   r1   rr   rŽ   )	r"   r5   r6   r7   r8   r9   r:   Ú
pid_groupsÚ	pid_groupr   r“   r   Ú#test_nested_backend_context_managerš   s6   þÿû€ÿ
þÿû€ÿõ"õr—   c                    s8  dd„ ‰ ‡ fdd„‰t ƒ �…\}\}}t|d | d��`}tdƒ�L\}}tƒ �$}ˆ |ƒdks.J ‚|jd	ks5J ‚|‡fd
d„tdƒD ƒƒ}W d   ƒ n1 sLw   Y  |D ]\}	}
|	dks]J ‚|
d	kscJ ‚qSW d   ƒ n1 snw   Y  W d   ƒ n1 s}w   Y  W d   ƒ d S W d   ƒ d S 1 s•w   Y  d S )Nc                 S   s
   | j jjS r   )Ú_backendÚ	__class__r[   ©Úpr   r   r   Ú_backend_type¹   rG   zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t ƒ �} ˆ | ƒ| jfW  d   ƒ S 1 sw   Y  d S r   )r	   rˆ   rš   )rœ   r   r   Úget_nested_implicit_n_jobs¼   s   $ÿzWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr    r!   r#   r   éÿÿÿÿc                 3   r‘   r   r’   r�   )r�   r   r   r)   Æ   r”   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r   )r   r   r   r	   rˆ   r1   )r"   r5   r6   r7   r8   r9   r:   r›   Úall_nested_n_jobsÚbackend_typeÚnested_n_jobsr   )rœ   r�   r   Ú3test_nested_backend_context_manager_implicit_n_jobsµ   s,   
þýþø€ÿÿ"ÿr¢   c              	   C   sl   t  t¡�}tdƒ� W d   ƒ n1 sw   Y  W d   ƒ n1 s$w   Y  dt|jƒ ¡ v s4J ‚d S )Nr#   zcreate a dask client)r2   r3   r   r   ÚstrÚvalueÚlower)r"   Úinfor   r   r   Útest_errorsÏ   s   
ÿ€ÿr§   c              
   C   s8  t ƒ ��\}\}}t|d | d��j}tdƒ�%\}}tdd�dd„ tdƒD ƒƒ}t|d	 d	 d	 tƒs3J ‚W d   ƒ n1 s=w   Y  tdƒ�%\}}tdd�d
d„ tdƒD ƒƒ}t|d	 d	 d	 tƒsdJ ‚W d   ƒ n1 snw   Y  W d   ƒ n1 s}w   Y  W d   ƒ d S W d   ƒ d S 1 s•w   Y  d S )Nr    r!   r#   r   r‡   c                 s   s   � | ]
}t tƒd d�V  qd S )N©Únested_require©r
   Úouterr�   r   r   r   r)   Ü   s   € 
ÿz.test_correct_nested_backend.<locals>.<genexpr>rY   r   c                 s   s   � | ]
}t tƒd d�V  qdS )Ú	sharedmemr¨   Nrª   r�   r   r   r   r)   â   s
   € ÿ
ÿ)r   r   r   r	   r1   Ú
isinstancer   r   )r"   r5   r6   r7   r8   r9   r:   Úresultr   r   r   Útest_correct_nested_backend×   s&   ÿýþü€øÿ"ÿr¯   c                    s"   t ddd�‡ fdd„tdƒD ƒƒS )Nr   Úthreads)rˆ   Úpreferc                 3   s   � | ]	}t tƒˆ ƒV  qd S r   )r
   Úmiddler�   r¨   r   r   r)   é   ó   € 
ÿzouter.<locals>.<genexpr>rY   ©r	   r1   r¨   r   r¨   r   r«   è   s   ÿr«   c                 C   s   t d| d�dd„ tdƒD ƒƒS )Nr   )rˆ   Úrequirec                 s   r>   r   )r
   Úinnerr�   r   r   r   r)   ï   s   € 

ÿzmiddle.<locals>.<genexpr>rY   r´   )rµ   r   r   r   r²   î   s   ÿr²   c                   C   s   t ƒ jS r   )r	   r˜   r   r   r   r   r¶   ô   s   r¶   c              	   C   sˆ   t | ddd��3 tdƒ� tdd�dd„ td	ƒD ƒƒ W d   ƒ n1 s%w   Y  W d   ƒ d S W d   ƒ d S 1 s=w   Y  d S )
NFT)r"   Ú	processesÚset_as_defaultr#   r`   r‡   c                 s   r$   r   )r
   Úidr&   r   r   r   r)   ü   r*   z0test_secede_with_no_processes.<locals>.<genexpr>r   )r   r   r	   r1   r!   r   r   r   Útest_secede_with_no_processesø   s   
ÿÿ"ÿrº   c                 C   s   ddl m} |ƒ jS )Nr   )Ú
get_worker)r   r»   r    )r:   r»   r   r   r   Ú_worker_addressÿ   s   r¼   c              
   C   s8  t ƒ ��\}\}}t|d | d��j}td|d d��!\}}tƒ dd„ tdƒD ƒƒ}||d gd ks3J ‚W d   ƒ n1 s=w   Y  td|d d��!\}}tƒ dd„ tdƒD ƒƒ}||d gd ksdJ ‚W d   ƒ n1 snw   Y  W d   ƒ n1 s}w   Y  W d   ƒ d S W d   ƒ d S 1 s•w   Y  d S )	Nr    r!   r#   )Úworkersc                 s   r$   r   ©r
   r¼   r&   r   r   r   r)     r³   z-test_dask_backend_keywords.<locals>.<genexpr>r+   c                 s   r$   r   r¾   r&   r   r   r   r)     r³   r?   r4   r   r   r   Útest_dask_backend_keywords  s&   
ÿý
ÿý€úÿ"ÿr¿   c              	   C   s°   t d| d��H}tdƒ� tƒ dd„ tdƒD ƒƒ W d   ƒ n1 s"w   Y  tƒ }|jjjrAtdƒ tƒ |d k s<J ‚|jjjs/|j	rFJ ‚W d   ƒ d S 1 sQw   Y  d S )	NF)r·   r"   r#   c                 s   r$   r   r%   r&   r   r   r   r)     r*   ztest_cleanup.<locals>.<genexpr>r+   g{®Gáz„?ra   )
r   r   r	   r1   r   r   Ú	schedulerre   r   Úfutures)r"   r8   Ústartr   r   r   Útest_cleanup  s   
ÿ

þ"÷rÃ   Úcluster_strategyÚadaptiveÚlate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)Úreasonc                 C   sÀ   t dddd�}t|ƒ}| dkr|jddd� n	| dkr | d¡ z6tdƒ� tƒ d	d
„ tdƒD ƒƒ W d   ƒ n1 s<w   Y  W | ¡  | ¡  d S W | ¡  | ¡  d S | ¡  | ¡  w )Nr   Fr   ©Ú	n_workersr·   Úthreads_per_workerrÅ   )ÚminimumÚmaximumrÆ   r#   c                 s   r$   r   r%   r&   r   r   r   r)   2  r*   z(test_wait_for_workers.<locals>.<genexpr>r+   )r   r   ÚadaptÚscaler   r	   r1   Úclose)rÄ   r   r8   r   r   r   Útest_wait_for_workers  s"   

ýúÿ
rÐ   c               	   C   sB  t dddd�} t| ƒ}zŒtddd��- d}tjt|d	�� tƒ d
d„ tdƒD ƒƒ W d   ƒ n1 s3w   Y  W d   ƒ n1 sBw   Y  tddd��- d}tjt|d	�� tƒ dd„ tdƒD ƒƒ W d   ƒ n1 snw   Y  W d   ƒ n1 s}w   Y  W | 	¡  |  	¡  d S W | 	¡  |  	¡  d S | 	¡  |  	¡  w )Nr   Fr   rÈ   r#   gš™™™™™¹?)Úwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)Úmatchc                 s   r$   r   r%   r&   r   r   r   r)   A  r*   z0test_wait_for_workers_timeout.<locals>.<genexpr>r+   z+DaskDistributedBackend has no active workerc                 s   r$   r   r%   r&   r   r   r   r)   G  r*   )
r   r   r   r2   r3   r   r	   r1   ÚRuntimeErrorrÏ   )r   r8   Úmsgr   r   r   Útest_wait_for_workers_timeout8  s0   ÿ€ýÿ€ýùÿ
rÕ   )r   )r   r   )6Ú
__future__r   r   r   r‹   r2   r   r   r   Ú r	   r
   r   Úparallelr   Ú_daskr   r   r}   r   r   r   Údistributed.metricsÚdistributed.utils_testr   r   r   r   r<   r=   rB   rM   rQ   ÚobjectrR   rh   r†   r—   r¢   r§   r¯   r«   r²   r¶   rº   r¼   r¿   rÃ   ÚmarkÚparametrizeÚskipifÚ__version__rÐ   rÕ   r   r   r   r   Ú<module>   sP    


&þ