o
    l¨Êh‹[  ã                   @   s:  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z  d dlm!Z! d dlm"Z" d dlm#Z# d dlm$Z% dd„ Z&dd„ Z'dd„ Z(dd„ Z)e	edd„ ƒƒZ*e	edd„ ƒƒZ+e	d d!„ ƒZ,e	eed"eegd#d$gd%�d&d'„ ƒƒƒZ-e	eed"eegd#d$gd%�d(d)„ ƒƒƒZ.e	eed"eegd#d$gd%�d*d+„ ƒƒƒZ/e	eed"eegd#d$gd%�d,d-„ ƒƒƒZ0e	eeed"eegd#d$gd%�d.d/„ ƒƒƒƒZ1e	eeed"eegd#d$gd%�d0d1„ ƒƒƒƒZ2e	eed"eegd#d$gd%�d2d3„ ƒƒƒZ3d4d5„ Z4e	eed"eegd#d$gd%�d6d7„ ƒƒƒZ5d8d9„ Z6e	eed"eegd#d$gd%�d:d;„ ƒƒƒZ7d<d=„ Z8e	eejd>kd?d@�edAdBdCgƒdDdE„ ƒƒƒZ9e	dFdG„ ƒZ:dHdI„ Z;e	edJdK„ ƒƒZ<dS )Lé    N)Úsleep)Ú
with_numpyÚnp)Úsetup_autokill)Úteardown_autokill)Úwith_multiprocessing)Úwith_dev_shm)ÚraisesÚparametrizeÚskipif)Úmake_memmap)ÚParallelÚdelayed)ÚMemmappingPool)Ú_TestingMemmappingExecutor©Úhas_shareable_memory)ÚArrayMemmapReducer©Úreduce_memmap)Ú_strided_from_memmap)Ú_get_backing_memmap)Ú_get_temp_dir)Ú_WeakArrayKeyMapc                   C   s   t tdd� d S )Ni,  )Útimeout)r   Ú__name__© r   r   úM/var/www/html/env/lib/python3.10/site-packages/joblib/test/test_memmapping.pyÚsetup_module   s   r   c                   C   s   t tƒ d S ©N)r   r   r   r   r   r   Úteardown_module"   s   r    c                 C   s    | \}}}t j || |¡ dS )z�Dummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    N©r   ÚtestingÚassert_array_equal©ÚargsÚdataÚpositionÚexpectedr   r   r   Úcheck_array&   s   
r)   c                 C   sD   | \}}}|| |ksJ ‚||  d9  < t j || d| ¡ dS )zÜDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

    é   Nr!   r$   r   r   r   Úinplace_double1   s   
	r+   c                    s  t jj}|  d¡j}t j|t jddd�}dt j|jd |j	d� |dd…< | 
¡  t j|t jd	d
ddd�}t  d¡ |j¡|dd…< |dd…dd…dd…f }t  |¡}|j}td| jdƒ‰ ‡ fdd„}dd„ }	|	|ƒ}
t|
ƒsrJ ‚t|
t jƒszJ ‚||
|ƒ |	|ƒ}t|ƒs‰J ‚|||ƒ ||ƒ}t|t jƒršJ ‚t|ƒs J ‚|||ƒ ||ƒ}t|t jƒr±J ‚t|ƒs·J ‚|||ƒ |d }t|ƒrÆJ ‚|	|ƒ}t|ƒrÐJ ‚t|t jƒrØJ ‚|||d ƒ t  |¡}t|ƒrêJ ‚||ƒ}t|t jƒsöJ ‚t|ƒrüJ ‚|||ƒ dS )z9Check that it is possible to reduce a memmap backed arrayú	test.mmapiô  úw+©ÚdtypeÚshapeÚmodeg      ð¿r   ©r/   N)é   é   é   úr+ÚFr5   )r/   r0   r1   ÚorderÚoffseté<   é   éÿÿÿÿr*   Úcc                    s   ˆ | ƒ\}}||Ž S r   r   ©ÚxÚconsr%   ©Úreducerr   r   Úreconstruct_arraya   ó   z;test_memmap_based_array_reducing.<locals>.reconstruct_arrayc                 S   s   t | ƒ\}}||Ž S r   r   r>   r   r   r   Úreconstruct_memmape   rD   z<test_memmap_based_array_reducing.<locals>.reconstruct_memmapr3   )r   r"   r#   ÚjoinÚstrpathÚmemmapÚfloat64Úaranger0   r/   ÚflushÚreshapeÚasarrayÚTr   r   Ú
isinstanceÚndarray)Útmpdirr#   ÚfilenameÚbufferÚaÚbr=   ÚdrC   rE   Úa_reconstructedÚb_reconstructedÚc_reconstructedÚd_reconstructedÚa3Úa3_reconstructedÚb3Úb3_reconstructedr   rA   r   Ú test_memmap_based_array_reducing@   sT   "ÿ





r_   c                 C   s>  t jj}|  d¡j}t j|t jddd�}t  d¡ |j	¡|d d …< |dd… }|d d …dd…f }|d d …d d …d d …df }|d	d
d… }dd„ }||ƒ}	t
|	ƒsTJ ‚t|	t jƒs\J ‚||	|ƒ ||ƒ}
t
|
ƒskJ ‚||
|ƒ ||ƒ}t
|ƒszJ ‚|||ƒ ||ƒ}t
|ƒs‰J ‚|||ƒ ||ƒ}t
|ƒs˜J ‚|||ƒ d S )Nr,   )éd   é   ra   r3   r-   r.   i¬ r   é
   r4   r;   r3   r5   c                 S   s   t | ƒ\}}||Ž }|S r   r   )r?   r@   r%   Úresr   r   r   rE   ¤   s   zEtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_memmap)r   r"   r#   rF   rG   rH   rI   rJ   rL   r0   r   rO   )rQ   r#   rR   rT   rU   r=   rV   ÚerE   rW   rX   rY   rZ   Úe_reconstructedr   r   r   Ú)test_high_dimension_memmap_array_reducing’   s6   ÿ



rf   c              
   C   sž   |   d¡j}dtj }tjd }tj|d|| d�}t|dd|d|d d d	�}t|tjƒs.J ‚|j|ks5J ‚t|dd|d|d
 fd|d	�}t	|ƒj|ksMJ ‚d S )Nr,   r4   r;   r-   )r1   r0   Úuint8ÚrÚC)r/   r1   r9   r8   r0   ÚstridesÚtotal_buffer_lenr*   )r*   )
rF   rG   ÚmmapÚALLOCATIONGRANULARITYr   rH   r   rO   r9   r   )rQ   ÚfnameÚsizer9   Ú
memmap_objÚmemmap_backed_objr   r   r   Útest__strided_from_memmap¿   s    

þ
ýrr   ÚfactoryÚmultiprocessingÚloky)Úidsc                    sÜ  t jj}| d¡j}| dd|d�}zÖ| d¡j}t j|t jddd�‰ ˆ  d	¡ | 	t
‡ fd
d„tˆ jd ƒD ƒ¡ |ˆ dt  ˆ j¡ ƒ t j|t jddd�‰| 	t
‡fdd„tˆjd ƒD ƒ¡ t |¡g ksiJ ‚|ˆ dt  ˆ j¡ ƒ |ˆdt  ˆj¡ ƒ t j|t jdddd�‰ttƒ� | 	t‡fdd„tˆjd ƒD ƒ¡ W d  ƒ n1 s«w   Y  tttfƒ� | 	t
‡fdd„tˆjd ƒD ƒ¡ W d  ƒ n1 sÓw   Y  W | ¡  ~dS W | ¡  ~dS | ¡  ~w )z@Check that subprocess can access and update shared memory memmapÚpoolrb   r*   ©Ú
max_nbytesÚtemp_folderr,   ©r3   r4   r-   r.   ç      ð?c                    ó.   g | ]}t ˆ jd  ƒD ]	}ˆ ||fdf‘qqS ©r;   r|   ©Úranger0   ©Ú.0ÚiÚj)rT   r   r   Ú
<listcomp>ã   ó    þþz)test_pool_with_memmap.<locals>.<listcomp>r   )r4   r3   r=   c                    r}   )r;   ç       @r   r�   )rU   r   r   r…   ì   r†   ©rb   rh   é   )r/   r0   r1   r9   c                    ó   g | ]}ˆ |d f‘qS )g      @r   ©r‚   rƒ   ©r=   r   r   r…   ý   ó    Nc                    rŠ   )r‡   r   r‹   rŒ   r   r   r…     r�   )r   r"   r#   ÚmkdirrG   rF   rH   Úfloat32ÚfillÚmapr+   r€   r0   ÚonesÚosÚlistdirr	   ÚAssertionErrorr)   ÚRuntimeErrorÚ
ValueErrorÚ	terminate©rs   rQ   r#   Úpool_temp_folderÚprR   r   )rT   rU   r=   r   Útest_pool_with_memmapÓ   sD   
ÿÿÿ
&ÿ&ÿûÿrœ   c                    sô   t jj}| d¡j}| dd|d�}zb| d¡j}t j|t jddd�‰ ˆ  d	¡ t  	ˆ ¡‰t
ˆt jƒr4J ‚tˆƒs:J ‚| t‡ ‡fd
d„tˆ jd ƒD ƒ¡ |ˆ dt  ˆ j¡ ƒ |ˆdt  ˆ j¡ ƒ t |¡g kslJ ‚W | ¡  ~dS | ¡  ~w )z?Check that subprocess can access and update shared memory arrayrw   rb   r*   rx   r,   r{   r-   r.   r|   c                    s.   g | ]}t ˆ jd  ƒD ]	}ˆ||fdf‘qqS r~   r   r�   ©rT   Úa_viewr   r   r…     r†   z4test_pool_with_memmap_array_view.<locals>.<listcomp>r   N)r   r"   r#   rŽ   rG   rF   rH   r�   r�   rM   rO   r   r‘   r+   r€   r0   r’   r“   r”   r˜   r™   r   r�   r   Ú test_pool_with_memmap_array_view	  s(   

ÿÿrŸ   c              	      s‚  t  |j¡g ks
J ‚| dd|jdd�}zžt  |j¡g ksJ ‚t j |j¡r'J ‚tjdtjd�‰ˆj	dks6J ‚| 
t‡fdd	„tˆjd
 ƒD ƒ¡ t  |j¡g ksRJ ‚tjdtjd�‰ ˆ j	dksaJ ‚| 
t‡ fdd	„tˆ jd
 ƒD ƒ¡ t j |j¡s|J ‚t  |j¡}t|ƒdksŠJ ‚tjdgd dd�}| 
t|g¡}|d
 r¡J ‚W | ¡  t j |j¡r¯J ‚~dS | ¡  t j |j¡r¿J ‚~w )z0Check that large arrays are not copied in memoryr3   é(   r*   )ry   rz   Úverboser4   r2   r‰   c                    rŠ   ©r|   r   r‹   )Úsmallr   r   r…   D  r�   z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>r   r`   é   c                    rŠ   r¢   r   r‹   ©Úlarger   r   r…   L  r�   r;   ÚabcÚobjectN)r“   r”   rG   ÚpathÚexistsÚ_temp_folderr   r’   r�   Únbytesr‘   r)   r€   r0   rI   ÚisdirÚlenÚarrayr   r˜   )rs   rQ   r›   Údumped_filenamesÚobjectsÚresultsr   )r¦   r£   r   Ú%test_memmapping_pool_for_large_arrays0  s0   $$þr³   c                    s˜   | dd|j d�}z=t |j ¡g ksJ ‚tjdtjd�‰ ˆ jdks"J ‚| t‡ fdd„t	ˆ j
d	 ƒD ƒ¡ t |j ¡g ks>J ‚W | ¡  ~dS | ¡  ~w )
z2Check that large arrays memmapping can be disabledr3   Nrx   r`   r2   r¤   c                    rŠ   r¢   r   r‹   r¥   r   r   r…   q  r�   zBtest_memmapping_pool_for_large_arrays_disabled.<locals>.<listcomp>r   )rG   r“   r”   r   r’   rI   r¬   r‘   r)   r€   r0   r˜   )rs   rQ   r›   r   r¥   r   Ú.test_memmapping_pool_for_large_arrays_disableda  s   $ÿr´   c                 C   s  t j}zƒtdƒt _| ddd�}zc|j}d}| |¡sJ ‚tj |¡s$J ‚tj	dtj
d�}|jdks3J ‚| t|gd ¡ tt |¡ƒd	ksGJ ‚tj	dtj
d�d
 }|jdksXJ ‚| t|gd ¡ tt |¡ƒd
kslJ ‚W | ¡  ~n| ¡  ~w tj |¡r�J ‚W |t _dS |t _w )z1Check that memmapping uses /dev/shm when possibleg    €„~Ar3   rb   ©ry   z"/dev/shm/joblib_memmapping_folder_r`   r2   r¤   r;   r*   N)ÚjmrÚSYSTEM_SHARED_MEM_FS_MIN_SIZEÚintr«   Ú
startswithr“   r©   rª   r   r’   rI   r¬   r‘   Úidr®   r”   r˜   )rs   Ú	orig_sizer›   rš   Úfolder_prefixrT   rU   r   r   r   Ú'test_memmapping_on_large_enough_dev_shm|  s.   
ÿr½   c                 C   sr   t j}z1tdƒt _| ddd�}z|j}| d¡rJ ‚W | ¡  ~n| ¡  ~w tj |¡r/J ‚W |t _d S |t _w )Ng@³ËËï6Dr3   rb   rµ   z/dev/shm)	r¶   r·   r¸   r«   r¹   r˜   r“   r©   rª   )rs   r»   r›   rš   r   r   r   Ú$test_memmapping_on_too_small_dev_shm¬  s   
ÿr¾   c                 C   sj   t jj}| dd|jd�}z"|jt jdd�}| ¡ }t|ƒrJ ‚||t  d¡ƒ W | ¡  ~dS | ¡  ~w )z:Check that large arrays are not copied in memory in returnr3   rb   rx   )éè  ©r%   r¿   N)	r   r"   r#   rG   Úapply_asyncr’   Úgetr   r˜   )rs   rQ   r#   r›   rc   r¦   r   r   r   Ú/test_memmapping_pool_for_large_arrays_in_returnÉ  s   	ÿrÃ   c                 C   s   t | ƒsJ ‚| | S )z4Multiplication function to be executed by subprocessr   )rT   Ún_timesr   r   r   Ú_worker_multiplyã  s   rÅ   c                 C   s–   t jj}| dd|jd�}z8t jt  d¡ d¡dd�dd…dd	…dd…f }|jt|dfd
� 	¡ }t
|ƒr6J ‚||d| ƒ W | ¡  ~dS | ¡  ~w )zÏCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r3   rb   rx   ip  )r¿   r*   r3   r7   )r8   Nr;   rÀ   )r   r"   r#   rG   rM   rJ   rL   rÁ   rÅ   rÂ   r   r˜   )rs   rQ   r#   r›   rT   rU   r   r   r   Ú6test_workaround_against_bad_memmap_with_copied_buffersé  s   
ÿÿÿrÆ   c                 C   s   | S r   r   )Úargr   r   r   Úidentity  s   rÈ   c                 C   s„   |  d¡j}dtj }tjd }t|d|d|d�}| d|jd�}|jt|fd	� ¡ }t|t	j
ƒs2J ‚|j|ks9J ‚t	j ||¡ d S )
Nr,   r4   r;   r-   rg   )r1   r0   r/   r9   r*   ©rz   rÀ   )rF   rG   rl   rm   r   rÁ   rÈ   rÂ   rO   r   rH   r9   r"   r#   )rs   rQ   rn   ro   r9   Úobjr›   Úresultr   r   r   Ú test_pool_memmap_with_big_offset
  s   	


ÿrÌ   c                 C   sn   d}t || jƒ\}}|du sJ ‚||  d¡jksJ ‚t |d d�\}}tj d¡r.|du s.J ‚| |¡s5J ‚d S )Nztest.tmpdirFrÉ   Úwin)r   rG   rF   ÚsysÚplatformr¹   Úendswith)rQ   Úpool_folder_nameÚpool_folderÚ
shared_memr   r   r   Útest_pool_get_temp_dir   s   rÔ   Úwin32z1This test fails with a PermissionError on Windows)ÚreasonÚ	mmap_moder6   r-   c                    sb   dd„ ‰ dd„ t dƒD ƒ}t| ddd�‡ fd	d
„t|ƒD ƒƒ}t|ƒD ]\}}tj ||¡ q#d S )Nc                 S   s   || d d …< | S r   r   )ÚarrÚvaluer   r   r   Úfunc1  s   z4test_numpy_arrays_use_different_memory.<locals>.funcc                 S   s   g | ]	}t jd dd�‘qS ))rb   rb   rI   r2   )r   Úzerosr‹   r   r   r   r…   5  s    z:test_numpy_arrays_use_different_memory.<locals>.<listcomp>rb   r   r*   )r×   ry   Ún_jobsc                 3   s"   � | ]\}}t ˆ ƒ||ƒV  qd S r   ©r   )r‚   rƒ   rØ   ©rÚ   r   r   Ú	<genexpr>7  s   € 
ÿz9test_numpy_arrays_use_different_memory.<locals>.<genexpr>)r€   r   Ú	enumerater   r"   r#   )r×   Úarraysr²   rƒ   rØ   r   rÞ   r   Ú&test_numpy_arrays_use_different_memory,  s   ÿÿrâ   c                     sl  ddd„} t  d¡}tƒ ‰ˆ |d¡ ˆ |¡dksJ ‚|}ˆ |¡dks'J ‚ˆ |d¡ ˆ |¡dks6J ‚~t ¡  tˆjƒdksDJ ‚ˆ |¡dksMJ ‚~| ˆjƒ t  d¡}ˆ |d¡ tˆjƒdksgJ ‚ˆ |¡dkspJ ‚t	t
ƒ� ˆ t  d¡¡ W d   ƒ n1 s‡w   Y  ~| ˆjƒ d	d
„ ‰ t‡ ‡fdd„tdƒD ƒƒ}t ¡ dkr²t|ƒdk s´J ‚d S d S )Nr`   c                 S   sD   t |ƒD ]}t| ƒdkr d S t ¡  tdƒ qt| ƒdks J ‚d S )Nr   gš™™™™™¹?)r€   r®   ÚgcÚcollectr   )Ú	containerÚretriesrƒ   r   r   r   Úassert_empty_after_gc_collectA  s   
z>test_weak_array_key_map.<locals>.assert_empty_after_gc_collecté*   rT   rU   r;   r=   c                 S   sb   t  d¡}ttƒ� |  |¡ W d   ƒ n1 sw   Y  |  ||¡ |  |¡|ks-J ‚t|ƒS )Nrè   )r   r’   r	   ÚKeyErrorrÂ   Úsetrº   )Úmrƒ   rT   r   r   r   Úget_set_get_collecth  s   

ÿz4test_weak_array_key_map.<locals>.get_set_get_collectc                    s   g | ]}ˆ ˆ|ƒ‘qS r   r   r‹   ©rì   rë   r   r   r…   p  r�   z+test_weak_array_key_map.<locals>.<listcomp>r¿   ÚCPython)r`   )r   r’   r   rê   rÂ   rã   rä   r®   Ú_datar	   ré   r€   rÏ   Úpython_implementation)rç   rT   rU   r=   Ú
unique_idsr   rí   r   Útest_weak_array_key_map>  s<   




ÿ
ûrò   c                  C   s@   t ƒ } ttjƒ� t | ¡ W d   ƒ d S 1 sw   Y  d S r   )r   r	   ÚpickleÚPicklingErrorÚdumps)rë   r   r   r   Ú#test_weak_array_key_map_no_picklingy  s   "ÿrö   c                    sÌ   t |  d¡ƒ‰tjddd�}| ˆ¡ ‡fdd„‰ dd„ ‰ˆ ƒ }td	d
�‡fdd„|fD ƒƒ}td	d
�‡fdd„|fD ƒƒ}tj ||¡ ‡ fdd„‰td	d
�‡fdd„tdƒD ƒƒ}tj |d |¡ d S )Nzarr.datrb   rg   r2   c                     sT   t ˆ ƒ�} tj|  ¡ dtjdd�}W d   ƒ n1 sw   Y  tjdtj|dd�S )Nr   )Úaccessr9   rˆ   )r/   rS   r9   )Úopenrl   ÚfilenoÚACCESS_READr   rP   rg   )ÚfdÚmm)Útestfiler   r   Ú_read_array†  s   
ÿz%test_direct_mmap.<locals>._read_arrayc                 S   s   | d S )Nr*   r   )r?   r   r   r   rÚ   ‹  s   ztest_direct_mmap.<locals>.funcr*   )rÜ   c                 3   ó   � | ]	}t ˆ ƒ|ƒV  qd S r   rÝ   ©r‚   r?   rÞ   r   r   rß   ‘  ó   € z#test_direct_mmap.<locals>.<genexpr>c                 3   rÿ   r   rÝ   r   rÞ   r   r   rß   ”  r  c                      s   ˆ ƒ S r   r   r   )rþ   r   r   Úworker˜  s   z test_direct_mmap.<locals>.workerc                 3   s   � | ]}t ˆ ƒƒ V  qd S r   rÝ   )r‚   Ú_)r  r   r   rß   ›  s   € r;   r   )	ÚstrrF   r   rJ   Útofiler   r"   r#   r€   )rQ   rT   rØ   Úrefr²   r   )rþ   rÚ   rý   r  r   Útest_direct_mmap  s   
 r  )=r“   rl   rÎ   rÏ   rã   ró   Útimer   Újoblib.test.commonr   r   r   r   r   r   Újoblib.testingr	   r
   r   Újoblib.backportsr   Újoblib.parallelr   r   Újoblib.poolr   Újoblib.executorr   Újoblib._memmapping_reducerr   r   r   r   r   r   r   Ú_memmapping_reducerr¶   r   r    r)   r+   r_   rf   rr   rœ   rŸ   r³   r´   r½   r¾   rÃ   rÅ   rÆ   rÈ   rÌ   rÔ   râ   rò   rö   r  r   r   r   r   Ú<module>   sÔ    P+

ÿ2
ÿ#
ÿ-
ÿ
ÿ+
ÿ
ÿ
ÿ
ÿ
: