o
    �¨Êh0  ã                   @   s  d Z ddlmZ ddlZddlZddlZddlZddlm	  m
Z ddlmZ ddlmZ ddlmZ ddlmZ ej d¡Zejjejjej dd	d
g¡ej dddg¡dd„ ƒƒƒƒZejjejjdd„ ƒƒZejdd„ ƒZejjej d¡e  ¡ G dd„ dƒƒƒƒZ!dS )zl
Tests parsers ability to read and parse non-local files
and hence require a network connection to be read.
é    )ÚBytesION)Ú	DataFrame)Úread_feather©Úread_csvz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningÚmodeÚexplicitÚinferÚengineÚpythonÚcc                 C   sœ   |dkr	t  d¡ || }t|dddd| ƒdƒ�}| j| ¡ d� W d   ƒ n1 s,w   Y  | jd	 | }	|d
kr>|}t|	d||d�}
t |
|¡ d S )NÚtarz5TODO: Add tar salaraies.csv to pandas/io/parsers/dataÚioÚparserÚdatazsalaries.csvÚrb©Úcontentz/salaries.csvr   ú	)ÚsepÚcompressionr
   )	ÚpytestÚskipÚopenÚserve_contentÚreadÚurlr   ÚtmÚassert_frame_equal)Ú
httpserverÚdatapathÚsalaries_tabler   r
   Úcompression_onlyÚcompression_to_extensionÚ	extensionÚfr   Ú	url_table© r'   úU/var/www/html/env/lib/python3.10/site-packages/pandas/tests/io/parser/test_network.pyÚtest_compressed_urls   s   
ÿr)   c                 C   sj   t |ddddƒdƒ�}| j| ¡ d� t| jddd	�}W d  ƒ n1 s%w   Y  |jd
 dks3J ‚dS )zN
    read_csv should honor the requested encoding for URLs.

    GH 10424
    r   r   r   zunicode_series.csvr   r   zlatin-1N)ÚencodingÚheader)é   é   u$   Ã� kÃ¶ldum klaka (Cold Fever) (1994))r   r   r   r   r   Úloc)r   r    r%   Údfr'   r'   r(   Útest_url_encoding_csv8   s
   þr0   c                 C   s   t | ddddƒƒS )z DataFrame with the tips dataset.r   r   Úcsvútips.csvr   )r    r'   r'   r(   Útips_dfF   s   r3   Ús3_resourcec                   @   sÌ   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zejjddd�dd„ ƒZejjddd�dd„ ƒZejjd d!„ ƒZejjd"d#„ ƒZd$d%„ Zd&d'„ Zd(S ))ÚTestS3c                 C   s\   t  d¡ dD ]$\}}td|j› d�| ||d�}t|tƒs J ‚|jr%J ‚t ||¡ qd S )NÚs3fs©)Ú N)ú.gzÚgzip)ú.bz2Úbz2ús3://ú	/tips.csv)r   Ústorage_options©	r   Úimportorskipr   ÚnameÚ
isinstancer   Úemptyr   r   ©ÚselfÚs3_public_bucket_with_datar3   Ús3soÚextÚcompr/   r'   r'   r(   Útest_parse_public_s3_bucketP   s   
ý
øz"TestS3.test_parse_public_s3_bucketc                 C   sH   t  d¡ td|j› d�|d�}t|tƒsJ ‚|jrJ ‚t ||¡ d S )Nr6   r=   r>   ©r?   r@   )rF   Ús3_private_bucket_with_datar3   rH   r/   r'   r'   r(   Útest_parse_private_s3_bucket^   s   
ÿ
z#TestS3.test_parse_private_s3_bucketc                 C   óJ   t d|j› d�d|d�}t|tƒsJ ‚|jrJ ‚t |jd d… |¡ d S )Nzs3n://r>   é
   ©Únrowsr?   ©r   rB   rC   r   rD   r   r   Úiloc©rF   rG   r3   rH   r/   r'   r'   r(   Útest_parse_public_s3n_bucketh   ó   ý
z#TestS3.test_parse_public_s3n_bucketc                 C   rO   )Nzs3a://r>   rP   rQ   rS   rU   r'   r'   r(   Útest_parse_public_s3a_buckets   rW   z#TestS3.test_parse_public_s3a_bucketc                 C   s^   dD ]*\}}t d|j› d�| d||d�}t|tƒsJ ‚|jr!J ‚t |jd d… |¡ qd S )Nr7   r=   r>   rP   )rR   r   r?   rS   rE   r'   r'   r(   Ú!test_parse_public_s3_bucket_nrows~   s   ü
÷z(TestS3.test_parse_public_s3_bucket_nrowsc              	   C   s²   d}dD ]R\}}t d|j› d�| |||d��7}|j|ksJ ‚dD ]%}| ¡ }	t|	tƒs.J ‚|	jr3J ‚|j|| ||d  … }
t 	|
|	¡ q!W d   ƒ n1 sQw   Y  qd S )Né   r7   r=   r>   )Ú	chunksizer   r?   ©r   r-   é   r-   ©
r   rB   r[   Ú	get_chunkrC   r   rD   rT   r   r   ©rF   rG   r3   rH   r[   rI   rJ   Ú	df_readerÚi_chunkr/   Útrue_dfr'   r'   r(   Ú#test_parse_public_s3_bucket_chunkedŒ   s,   ü
ÿ÷ù€ÿz*TestS3.test_parse_public_s3_bucket_chunkedc              	   C   s´   d}dD ]S\}}t d|j› d�| ||d|d��7}|j|ks J ‚dD ]%}| ¡ }	t|	tƒs/J ‚|	jr4J ‚|j|| ||d  … }
t 	|
|	¡ q"W d   ƒ n1 sRw   Y  qd S )	NrZ   r7   r=   r>   r   )r[   r   r
   r?   r\   r-   r^   r`   r'   r'   r(   Ú*test_parse_public_s3_bucket_chunked_python¤   s.   û
ÿøø€ÿz1TestS3.test_parse_public_s3_bucket_chunked_pythonc                 C   sT   dD ]%\}}t d|j› d�| d||d�}t|tƒsJ ‚|jr!J ‚t ||¡ qd S )Nr7   r=   r>   r   ©r
   r   r?   ©r   rB   rC   r   rD   r   r   rE   r'   r'   r(   Ú"test_parse_public_s3_bucket_python¼   s   ü
÷z)TestS3.test_parse_public_s3_bucket_pythonc                 C   sP   dD ]#}t d|j› d�| dd|d�}t|tƒsJ ‚|jrJ ‚t ||¡ qd S )N)r8   r9   r;   r=   r>   r   r	   rf   rg   )rF   rG   r3   rH   rI   r/   r'   r'   r(   Útest_infer_s3_compressionÊ   s   ü
÷z TestS3.test_infer_s3_compressionc                 C   s`   dD ]+\}}t d|j› d�| dd||d�}t|tƒsJ ‚|jr"J ‚t |jd d… |¡ qd S )Nr7   r=   r>   r   rP   )r
   rR   r   r?   rS   rE   r'   r'   r(   Ú(test_parse_public_s3_bucket_nrows_pythonÖ   s   û
öz/TestS3.test_parse_public_s3_bucket_nrows_pythonc                 C   sD   d}t jt|d�� td|d� W d   ƒ d S 1 sw   Y  d S )Nú#The specified bucket does not exist©Úmatchzs3://nyqpug/asdf.csvrL   )r   ÚraisesÚOSErrorr   )rF   rH   Úmsgr'   r'   r(   Útest_read_s3_failså   s   "ÿzTestS3.test_read_s3_failsc                 C   sJ   d}t jt|d�� td|j› d�ƒ W d   ƒ d S 1 sw   Y  d S )Nrk   rl   r=   z	/file.csv)r   rn   ro   r   rB   )rF   Ús3_private_bucketrH   rp   r'   r'   r(   Útest_read_s3_fails_privateê   s   "ÿz!TestS3.test_read_s3_fails_privatezGH#39155 s3fs upgradeF)ÚreasonÚstrictc                 C   sV   dd l }t|jjf}tj|dd�� |jd|d� W d   ƒ d S 1 s$w   Y  d S )Nr   rk   rl   z/s3://an_s3_bucket_data_doesnt_exit/not_real.csvrL   )ÚbotocoreÚFileNotFoundErrorÚ
exceptionsÚClientErrorr   rn   Úto_csv©rF   r3   rH   rv   Úerrorr'   r'   r(   Útest_write_s3_csv_failsñ   s   ÿ"ÿzTestS3.test_write_s3_csv_failsc                 C   s`   t  d¡ dd l}t|jjf}t j|dd�� |jd|d� W d   ƒ d S 1 s)w   Y  d S )NÚpyarrowr   rk   rl   z3s3://an_s3_bucket_data_doesnt_exit/not_real.parquetrL   )r   rA   rv   rw   rx   ry   rn   Ú
to_parquetr{   r'   r'   r(   Útest_write_s3_parquet_fails  s   
þ"ÿz"TestS3.test_write_s3_parquet_failsc                 C   sz   |  d¡}t| ¡ d  ¡ ƒ�}t|dd�}W d   ƒ n1 s w   Y  t|tƒs,J ‚|jr1J ‚t|ƒ}t 	||¡ d S )Nr2   ÚBodyÚutf8)r*   )
ÚObjectr   Úgetr   r   rC   r   rD   r   r   )rF   rG   Ú	tips_fileÚ	s3_objectÚbufferÚresultÚexpectedr'   r'   r(   Ú$test_read_csv_handles_boto_s3_object  s   
ÿ
z+TestS3.test_read_csv_handles_boto_s3_objectc           
   	   C   s   t t d¡tdƒd�}t| ¡  d¡ƒ�b}|jd|d� |j› d�}t	 
d|› d	�¡}|jtjd
d��+ td|› �d|d� |jD ]}t	 ||¡ }	rVt|	 d¡ƒdk sVJ ‚qAW d   ƒ n1 saw   Y  W d   ƒ d S W d   ƒ d S 1 syw   Y  d S )N)i † é   Úabcd)Úcolumnszutf-8zlarge-file.csv)ÚKeyr�   z/large-file.csvz^Fetch: z, 0-(?P<stop>\d+)$r6   )Úloggerr=   rZ   rQ   Ústopi z )r   ÚnpÚzerosÚlistr   rz   ÚencodeÚ
put_objectrB   ÚreÚcompileÚat_levelÚloggingÚDEBUGr   Úmessagesrm   ÚintÚgroup)
rF   Ús3_public_bucketÚcaplogrH   r/   ÚbufÚuriÚmatch_reÚlogrm   r'   r'   r(   Útest_read_csv_chunked_download#  s(   ý
€ýúü"üz%TestS3.test_read_csv_chunked_downloadc                 C   s&   t d|j› d�|d�}t ||¡ d S )Nr=   z/tips#1.csvrL   )r   rB   r   r   )rF   rG   r3   rH   rˆ   r'   r'   r(   Útest_read_s3_with_hash_in_key6  s   ÿz$TestS3.test_read_s3_with_hash_in_keyc                 C   s8   t  d¡ t|ƒ}td|j› d�|d�}t ||¡ d S )Nr~   r=   z/simple_dataset.featherrL   )r   rA   r   rB   r   r   )rF   rG   Úfeather_filerH   r‰   Úresr'   r'   r(   Útest_read_feather_s3_file_path=  s   
þz%TestS3.test_read_feather_s3_file_pathN)Ú__name__Ú
__module__Ú__qualname__rK   rN   rV   rX   rY   rd   re   rh   ri   rj   rq   rs   r   ÚmarkÚxfailr}   r€   Ú
single_cpurŠ   r¤   r¥   r¨   r'   r'   r'   r(   r5   L   s.    




r5   )"Ú__doc__r   r   r™   r–   Únumpyr‘   r   Úpandas.util._test_decoratorsÚutilÚ_test_decoratorsÚtdÚpandasr   Úpandas._testingÚ_testingr   Úpandas.io.feather_formatr   Úpandas.io.parsersr   r¬   ÚfilterwarningsÚ
pytestmarkÚnetworkr®   Úparametrizer)   r0   Úfixturer3   ÚusefixturesÚskip_if_not_us_localer5   r'   r'   r'   r(   Ú<module>   s8    ÿ

