o
    d8                     @  s  d dl mZ d dlmZ d dlmZmZmZmZ d dl	Z	d dl
Zd dlmZmZ d dlmZmZmZmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZmZ d dl m!Z! d dl"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z( erxd dl)m*Z*m+Z+ G dd de&Z,dddZ-dddZ.dS )    )annotations)defaultdict)TYPE_CHECKINGHashableMappingSequenceN)libparsers)	ArrayLikeDtypeArgDtypeObjReadCsvBuffer)import_optional_dependency)DtypeWarning)find_stack_level)is_categorical_dtypepandas_dtype)concat_compatunion_categoricals)ensure_index_from_sequences)dedup_namesis_potential_multi_index)
ParserBaseParserErroris_index_col)Index
MultiIndexc                      sp   e Zd ZU ded< ded< d! fd	d
Zd"ddZd"ddZ	d#d$ddZd%ddZdd Z	d&d'dd Z
  ZS )(CParserWrapperbool
low_memoryzparsers.TextReader_readersrcReadCsvBuffer[str]returnNonec                   s  t  | || _| }|dd| _| jdu|d< | j|d< | jj	|d< dD ]}||d  q(t
|dd |d< d|vsF|d tju rJd	|d< |d d
krTtd
 tj|fi || _| jj| _| jd u }| jjd u rrd | _n| | jj| j|\| _| _| _}| jd u rtt| jj| _| jd d  | _| jr| | j| j | jd usJ | jdkrt | js|   | j t!| jt! krՇ fddt"| jD | _t!| jt! k r|   | j | #| j | $  | j| _| j%s8| jj&dkrt'| jrd| _(| )| j| j\}| _| _| jd u r|| _| jjd u r8|s8| jd us/J d gt!| j | _| jj&dk| _*d S )Nr   FZallow_leading_colsusecolson_bad_lines)Zstorage_optionsencodingZ
memory_mapcompressiondtypeZdtype_backendnumpyZpyarrowstringc                   $   g | ]\}}| v s| v r|qS  r-   ).0inr%   r-   V/app/.heroku/python/lib/python3.10/site-packages/pandas/io/parsers/c_parser_wrapper.py
<listcomp>   s    z+CParserWrapper.__init__.<locals>.<listcomp>r   T)+super__init__kwdscopypopr   	index_colr%   r&   valueensure_dtype_objsgetr   Z
no_defaultr   r	   Z
TextReaderr    Zunnamed_colsnamesheaderZ_extract_multi_indexer_columnsindex_names	col_nameslistrangetable_width
orig_names_evaluate_usecolsZusecols_dtypesetissubsetZ_validate_usecols_nameslen	enumerateZ_validate_parse_dates_presence_set_noconvert_columns_has_complex_date_colleading_colsr   Z_name_processed_clean_index_namesZ_implicit_index)selfr!   r6   keyZpassed_namesr?   	__class__r1   r2   r5   <   s   


	

zCParserWrapper.__init__c                 C  s&   z| j   W d S  ty   Y d S w N)r    close
ValueError)rN   r-   r-   r2   rS      s
   zCParserWrapper.closec                   s^   | j dusJ dd t| j D   fdd| jD }| || j}|D ]}| j| q$dS )z
        Set the columns that should not undergo dtype conversions.

        Currently, any column that is involved with date parsing will not
        undergo such conversions.
        Nc                 S  s   i | ]\}}||qS r-   r-   )r.   r/   xr-   r-   r2   
<dictcomp>       z9CParserWrapper._set_noconvert_columns.<locals>.<dictcomp>c                   s   g | ]} | qS r-   r-   r.   rU   Z
names_dictr-   r2   r3          z9CParserWrapper._set_noconvert_columns.<locals>.<listcomp>)rD   rI   r=   Z_set_noconvert_dtype_columnsr    Zset_noconvert)rN   Zcol_indicesZnoconvert_columnscolr-   rY   r2   rJ      s   z%CParserWrapper._set_noconvert_columnsNnrows
int | None_tuple[Index | MultiIndex | None, Sequence[Hashable] | MultiIndex, Mapping[Hashable, ArrayLike]]c              
     s  z| j r| j|}t|}n| j|}W nR tyh   | jrcd| _t| jt	| j| j
}| j|| j
| j| jdd\} }|  | j | jd urQ|    fdd| D }| |f Y S |    w d| _| j}| jjr| jrztdg }| j
r| jjt| j
krtdt| j
 d| jj d	t| jjD ]"}| j
d u r||}	n|| j
| }	| j|	|d
d}	||	 qt|}| jd ur| |}t|t	|| j
}t | }
dd t!||
D }| "||\}}| || j}nUt | }
| jd usJ t#| j}t|t	|| j
}| jd ur| |}dd |
D }| jd u r2| $|| dd t!||
D }| "||\}}| %|||\}}|||fS )NFr)   r)   c                   s   i | ]\}}| v r||qS r-   r-   )r.   kvcolumnsr-   r2   rV     s    z'CParserWrapper.read.<locals>.<dictcomp>z file structure not yet supportedz,Could not construct index. Requested to use z number of columns, but z left to parse.T)try_parse_datesc                 S     i | ]	\}\}}||qS r-   r-   r.   r`   r/   ra   r-   r-   r2   rV   0      c                 S  s   g | ]}|d  qS )   r-   rX   r-   r-   r2   r3   H  rZ   z'CParserWrapper.read.<locals>.<listcomp>c                 S  re   r-   r-   rf   r-   r-   r2   rV   L  rg   )&r   r    Zread_low_memory_concatenate_chunksreadStopIterationZ_first_chunkr   rD   r   r9   Z_get_empty_metar?   r6   r<   Z_maybe_make_multi_index_columnsr@   r%   _filter_usecolsitemsrS   r=   rL   rK   NotImplementedErrorrH   r   rB   r8   _maybe_parse_datesappendr   sortedzipZ_do_date_conversionsrA   Z_check_data_lengthZ_make_index)rN   r\   chunksdatar=   indexZcol_dictZarraysr/   valuesZ	data_tupsZcolumn_namesZ	date_dataZalldatar-   rb   r2   rj      s   










zCParserWrapper.readr=   Sequence[Hashable]c                   s@   |  | j|  d urt|t kr fddt|D }|S )Nc                   r,   r-   r-   )r.   r/   namer1   r-   r2   r3   W  s    z2CParserWrapper._filter_usecols.<locals>.<listcomp>)rE   r%   rH   rI   )rN   r=   r-   r1   r2   rl   S  s   
zCParserWrapper._filter_usecolsc                 C  sH   t | jjd }d }| jjdkr | jd ur | || j\}}| _||fS )Nr   )rA   r    r>   rL   r9   rM   )rN   r=   Z	idx_namesr-   r-   r2   _get_index_names\  s   zCParserWrapper._get_index_namesTru   intrd   c                 C  s4   |r|  |r| j|| jd ur| j| nd d}|S )N)r[   )Z_should_parse_datesZ
_date_convr?   )rN   rv   ru   rd   r-   r-   r2   ro   g  s   z!CParserWrapper._maybe_parse_dates)r!   r"   r#   r$   )r#   r$   rR   )r\   r]   r#   r^   )r=   rw   r#   rw   )T)ru   rz   rd   r   )__name__
__module____qualname____annotations__r5   rS   rJ   rj   rl   ry   ro   __classcell__r-   r-   rP   r2   r   8   s   
  


u	r   rs   list[dict[int, ArrayLike]]r#   dictc           
        s   t | d  }g }i }|D ]G  fdd| D }dd |D }dd |D }| }t|r8t|dd| < qt|| < t|d	krU|  jtt	krU|
t  q|rpd
|}dd| dg}	tj|	tt d |S )z
    Concatenate chunks of data read with low_memory=True.

    The tricky part is handling Categoricals, where different chunks
    may have different inferred categories.
    r   c                   s   g | ]}|  qS r-   )r8   )r.   chunkrx   r-   r2   r3   |  rW   z'_concatenate_chunks.<locals>.<listcomp>c                 S  s   h | ]}|j qS r-   r_   )r.   ar-   r-   r2   	<setcomp>~  s    z&_concatenate_chunks.<locals>.<setcomp>c                 S  s   h | ]}t |s|qS r-   )r   rX   r-   r-   r2   r     s    F)Zsort_categoriesrh   , z	Columns (zK) have mixed types. Specify dtype option on import or set low_memory=False.)
stacklevel)rA   keysr8   r   r   r   rH   r)   npobjectrp   strjoinwarningswarnr   r   )
rs   r=   Zwarning_columnsresultZarrsZdtypesZnon_cat_dtypesr)   Zwarning_namesZwarning_messager-   r   r2   ri   p  s,    

ri   r)   *DtypeArg | dict[Hashable, DtypeArg] | None*DtypeObj | dict[Hashable, DtypeObj] | Nonec                   sx   t tr$t  t fdd} D ]
}t| ||< q|S t tr2fddD S dur:tS S )zc
    Ensure we have either None, a dtype object, or a dictionary mapping to
    dtype objects.
    c                     s    S rR   r-   r-   )default_dtyper-   r2   <lambda>  s    z#ensure_dtype_objs.<locals>.<lambda>c                   s   i | ]	}|t  | qS r-   )r   )r.   r`   r_   r-   r2   rV     rg   z%ensure_dtype_objs.<locals>.<dictcomp>N)
isinstancer   r   default_factoryr   r   )r)   Zdtype_convertedrO   r-   )r   r)   r2   r;     s   

r;   )rs   r   r#   r   )r)   r   r#   r   )/
__future__r   collectionsr   typingr   r   r   r   r   r*   r   Zpandas._libsr   r	   Zpandas._typingr
   r   r   r   Zpandas.compat._optionalr   Zpandas.errorsr   Zpandas.util._exceptionsr   Zpandas.core.dtypes.commonr   r   Zpandas.core.dtypes.concatr   r   Zpandas.core.indexes.apir   Zpandas.io.commonr   r   Zpandas.io.parsers.base_parserr   r   r   Zpandasr   r   r   ri   r;   r-   r-   r-   r2   <module>   s,      
:%