o h,@sddlmZddlmZddlZddlmZddlmZddl m Z ddl m Z m Z ddlmZdd lmZdd lmZddlZdd lmZdd lmZmZdd lmZer\ddlmZGdddeZdS)) annotations) TYPE_CHECKINGN)using_pyarrow_string_dtype)lib)import_optional_dependency) ParserError ParserWarning)find_stack_level) pandas_dtype) is_integer) DataFrame)_arrow_dtype_mappingarrow_string_types_mapper) ParserBase) ReadBuffercsTeZdZdZdfdd Zddd Zdd d ZdddZdddZdddZ Z S)ArrowParserWrapperz7 Wrapper for the pyarrow engine for read_csv() srcReadBuffer[bytes]returnNonec s$t|||_||_|dSN)super__init__kwdsr _parse_kwds)selfrr __class__X/var/www/html/env/lib/python3.10/site-packages/pandas/io/parsers/arrow_parser_wrapper.pyr%s  zArrowParserWrapper.__init__cCsN|jd}|dur dn||_|jd}t|trtdt|jd|_dS)z? Validates keywords before passing to pyarrow. encodingNzutf-8 na_valuesz?The pyarrow engine doesn't support passing a dict for na_values)rgetr isinstancedict ValueErrorlistr!)rr r!rrrr,s   zArrowParserWrapper._parse_kwdscCsddddddd}|D]\}}||jvr'|j|dur'|j||j|<q |j}t|tr4|g}nd}||jd <d d |jD|_|jd }|durt|rZ||jd <n*|t j j krfd|jd <n|t j j krwddd}||jd <n |t j j krdd|jd <dd |jD|_d|jdv|jd<|jdurd|jvrdd|jdD|jd<|jdu|jdur|jn|jd|jd|_dS)z: Rename some arguments to pass to pyarrow include_columns null_values escape_charignore_empty_lines decimal_point quote_char)usecolsr! escapecharskip_blank_linesdecimal quotecharNtimestamp_parserscS&i|]\}}|dur|dvr||qS)N) delimiterr,r)r*r.0 option_name option_valuerrr Yz;ArrowParserWrapper._get_pyarrow_options.. on_bad_linesinvalid_row_handlerrstrcSs.tjd|jd|jd|jttddS)Nz Expected z columns, but found z: ) stacklevelskip)warningswarnexpected_columnsactual_columnstextrr ) invalid_rowrrrhandle_warningks z?ArrowParserWrapper._get_pyarrow_options..handle_warningcSsdS)Nr?r)_rrrvsz9ArrowParserWrapper._get_pyarrow_options..cSr3)N)r'r( true_values false_valuesr+r2rr5rrrr9xr:strings_can_be_nullcSsg|]}d|qS)fr)r6nrrr s z;ArrowParserWrapper._get_pyarrow_options..skiprows)autogenerate_column_names skip_rowsr )rr=)itemsrr"pop date_formatr#r= parse_optionscallablerBadLineHandleMethodERRORWARNSKIPconvert_optionsheaderr read_options)rmapping pandas_name pyarrow_namerUr;rFrrr_get_pyarrow_options:s^           z'ArrowParserWrapper._get_pyarrow_optionsframer c stj}d}|jdur7|jdur|jdurt||_t|j|kr3tt|t|j|j|_d}|j_|j\}|jdur|j}t |jD]S\}}t |r_j|||<n |jvrlt d|d|j dur|j |dur||j |fn j||j j|f\}} | dur|| |<|j |=qOj|ddd|jdur|sdgtjjj_|j durt|j trևfdd|j D|_ nt|j |_ z |j WSty} zt | d} ~ wwS) z Processes data read in based on kwargs. Parameters ---------- frame: DataFrame The DataFrame to process. Returns ------- DataFrame The processed DataFrame. TNFzIndex z invalid)dropinplacecs$i|]\}}|jvr|t|qSr)columnsr )r6kvrcrrr9s  z>ArrowParserWrapper._finalize_pandas_output..)lenrfr]namesranger&_do_date_conversions index_colcopy enumerater r%dtyper"astype set_indexindexr#r$rSr TypeError) rrcnum_colsmulti_index_namedrG index_to_setiitemkey new_dtypeerrir_finalize_pandas_outputsX             z*ArrowParserWrapper._finalize_pandas_outputcCs8t|rtdd|Dstdt|rtddS)Ncs|]}t|tVqdSrr#r=r6xrrr sz7ArrowParserWrapper._validate_usecols..zwThe pyarrow engine does not allow 'usecols' to be integer column positions. Pass a list of string column names instead.z=The pyarrow engine does not allow 'usecols' to be a callable.)r is_list_likeallr%rW)rr-rrr_validate_usecolssz$ArrowParserWrapper._validate_usecolsc Cstd}td}|z |jd i|j}Wn2tyI|jdd}|dur.|||jdt}t |rDt dd|DsHtdwz|j |j |j d i|j|jd i|j|d }Wn|jyv}zt||d}~ww|jd }|tjur|j} |} t|jjD]\} } |j| r| | | | | } q|| }|dkr|jtjd } n&|d krt }t!||"<|j|jd } nt#r|jt$d } n|} |%| S)z Reads the contents of a CSV file into a DataFrame and processes it according to the kwargs passed in the constructor. Returns ------- DataFrame The DataFrame created from the CSV file. pyarrowz pyarrow.csvr'Nr(csrrrrrrrrs  z*ArrowParserWrapper.read..z9The 'pyarrow' engine requires all na_values to be strings)r^rVr\ dtype_backend) types_mappernumpy_nullabler)&rrbConvertOptionsr\rur"rsetrrrread_csvr ReadOptionsr^ ParseOptionsrV ArrowInvalidrr no_defaultschemafloat64rptypesis_nullfield with_typecast to_pandaspd ArrowDtyper Int64Dtypenullrrr~)rpa pyarrow_csvr\includenullstabler}r new_schemanew_typery arrow_typerc dtype_mappingrrrreadsd          zArrowParserWrapper.read)rrrr)rr)rcr rr )rr ) __name__ __module__ __qualname____doc__rrrbr~rr __classcell__rrrrr s   [ I r) __future__rtypingrr@pandas._configr pandas._libsrpandas.compat._optionalr pandas.errorsrrpandas.util._exceptionsr pandas.core.dtypes.commonr pandas.core.dtypes.inferencer pandasrr pandas.io._utilr rpandas.io.parsers.base_parserrpandas._typingrrrrrrs"