
    Owg                     N   d Z ddlmZ ddlmZ ddlmZ ddlZddl	Z	ddl
Zddl
mZ ddlmZ ddlmZ e	j$                  j'                  d      Ze	j$                  j*                  e	j$                  j,                  gZe	j$                  j1                  d	      d
        Zd Zed        Zy)zd
Tests multithreading behaviour for reading and
parsing files for each parser defined in parsers.py
    )	ExitStack)BytesIO)
ThreadPoolN)	DataFrame)Versionpyarrow_xfailz0ignore:Passing a BlockManager:DeprecationWarningc           	      j   | }|j                   dk(  ret        j                  d      }t        |j                        t        d      k  r/|j                  t        j                  j                  d             dd}fdt        |      D        }t               5 }|D cg c]  }|j                  t        |             }}|j                  t        d            }	|	j                  |j                  |      }
|
d	   }|
D ]  }t        j                   ||        	 d d d        y c c}w # 1 sw Y   y xY w)
Npyarrowz16.0z+# ValueError: Found non-unique column index)reasond   
   c              3      K   | ]E  }d j                  t              D cg c]  }|dd|dd|d c}      j                          G yc c}w w)
d,N)joinrangeencode).0_imax_row_ranges      _/var/www/horilla/myenv/lib/python3.12/site-packages/pandas/tests/io/parser/test_multi_thread.py	<genexpr>z7test_multi_thread_string_io_read_csv.<locals>.<genexpr>(   sU       			}1EFAaU!Aa5!A'FGNNPFs   AA
A   r   )enginepytestimportorskipr   __version__applymarkermarkxfailr   r   enter_contextr   r   mapread_csvtmassert_frame_equal)all_parsersrequestparserpa	num_filesbytes_to_dfstackbfilespoolresultsfirst_resultresultr   s                @r   $test_multi_thread_string_io_read_csvr5      s    F}}	!  +2>>"WV_4!!)V!W MIy!K 
 	8:EFQ$$WQZ0FF"":a=1((6??E2qz 	8F!!,7	8	8 	8F	8 	8s   D)!D$ AD)$D))D2c                 6     fd}t        |      D cg c]  }||z  |z  ||z  f }}t        |      5 }|j                  ||      }ddd       d   j                  }	|dd D ]	  }
|	|
_         t	        j
                  |      }|S c c}w # 1 sw Y   ExY w)a  
    Generate a DataFrame via multi-thread.

    Parameters
    ----------
    parser : BaseParser
        The parser object to use for reading the data.
    path : str
        The location of the CSV file to read.
    num_rows : int
        The number of rows to read per task.
    num_tasks : int
        The number of tasks to use for reading this DataFrame.

    Returns
    -------
    df : DataFrame
    c                     | \  }}|sj                  dd|dg      S j                  ddt        |      dz   |dg      S )aj  
        Create a reader for part of the CSV.

        Parameters
        ----------
        arg : tuple
            A tuple of the following:

            * start : int
                The starting row to start for parsing CSV
            * nrows : int
                The number of rows to read.

        Returns
        -------
        df : DataFrame
        r   date)	index_colheadernrowsparse_datesN   	   )r9   r:   skiprowsr;   r<   )r%   int)argstartr;   r*   paths      r   readerz0_generate_multi_thread_dataframe.<locals>.readerN   sh    $ u??!5vh #   Z!^  
 	
    )	processesNr   r=   )r   r   r$   columnspdconcat)r*   rC   num_rows	num_tasksrD   r   tasksr1   r2   r:   rfinal_dataframes   ``          r    _generate_multi_thread_dataframerO   :   s    ( 
F EJ)DT?@A	"H	$9:E  
i	( *D((65)* QZFQR[ 	 ii(O* *s   B
BBc                 $   d}d}| }d}t        t        j                  j                  d      j                  |      t        j                  j                  d      j                  |      t        j                  j                  d      j                  |      t        j                  j                  d      j                  |      t        j                  j                  d      j                  |      dg|z  dg|z  dg|z  t	        j
                  d|d	
      t        j                  |d      d
      }t        j                  |      5 }|j                  |       t        ||||      }t        j                  ||       d d d        y # 1 sw Y   y xY w)N   0   z__thread_pool_reader__.csv   foobarbazz20000101 09:00:00s)periodsfreqint64)dtype)
ar/   cr   erT   rU   rV   r8   r@   )r   nprandomdefault_rngrH   
date_rangearanger&   ensure_cleanto_csvrO   r'   )r(   rK   rJ   r*   	file_namedfrC   rN   s           r   )test_multi_thread_path_multipart_read_csvrh      sJ    IHF,I	&&q)00:&&q)00:&&q)00:&&q)00:&&q)00:7X%7X%7X%MM"5xcR99XW5	

B 
	# 3t
		$:D(I
 	b/23 3 3s   6FF)__doc__
contextlibr   ior   multiprocessing.poolr   numpyr_   r   pandasrH   r   pandas._testing_testingr&   pandas.util.versionr   r!   usefixturesxfail_pyarrow
single_cpuslow
pytestmarkfilterwarningsr5   rO   rh    rE   r   <module>ry      s    !  +      '''8
 KK
KK
 NO8 P8<CL 3 3rE   