
    OwgzV                     	   d Z ddlmZ ddlZddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ ej                  j!                  d      Zej                  j%                  d      Zej                  j%                  d      Zd	 Zd
 Zej                  j/                  dddgddgddgdgdgdgdgg      ej                  j/                  dddg      d               Zd Zej                  j/                  dddgg      d        Zd Zd Zej                  j/                  ddg e	ej:                  gdgdgd e
dgd            fdd g e	ej:                  gdgd! ej<                  d"gdd#g$            fdd#g e	ej:                  gdgd! ej<                  d"gdd#g$            fg      d%        Zej                  j/                  d&i  e	dd'ej:                  d(d)ej:                  d*gg d+d,d-d.ej:                  d/ej:                  d0gd1      fg g d2d3d4 e	g d5g d+g d6d1      fdgd3d4 e	ej:                  d'd7d(d)d8d*gg d+g d6d1      fdg g d2i e	dd'ej:                  d(d)ej:                  d*gg d+d,d-d.ej:                  d/ej:                  d0gd1      fg      d9        Z d: Z!d; Z"d< Z#ej                  j/                  d=d>d?g      d@        Z$eej                  j/                  dAdBddCgej:                  ej:                  gdDdEggfd3dFdCgd8dGgdHdEggfg      dI               Z%edJ        Z&ej                  j/                  dKdej:                  dLgdLej:                  ggfd ddMdNdLgej:                  ej:                  ggfg      dO        Z'dP Z(dQ Z)ej                  j/                  dR e*dS      dTz    e*dU      z   ddSgi e	 e*dS       e*dU      g      f e*dS      dVz   dWz   i  e	 e*dS      dgd7d gg      f e*dS      dXz   ddSgi e	ej:                  dg      fg      dY        Z+dZ Z,ej                  j/                  d[d3d7d\gfdBej:                  d]gfg      d^        Z-d_ Z.eej                  j/                  d`dBd3g      da               Z/eej                  j/                  dbdcdddedfdgdhgfdgddhifg      di               Z0edj        Z1dk Z2edl        Z3dm Z4dn Z5y)ozg
Tests that NA values are properly handled during
parsing for all of the parsers defined in parsers.py
    )StringION)STR_NA_VALUES)	DataFrameIndex
MultiIndexz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningpyarrow_xfailpyarrow_skipc                 2   | }d}|j                  t        |            }t        g ddt        j                  dgt        j                  ddggg d      }|j
                  d	k(  rd |j                  d
<   d |j                  d<   t        j                  ||       y )NzA,B,C
a,b,c
d,,f
,g,h
abcdfghABCcolumnspyarrow)   r   )   r   )	read_csvr   r   npnanengineloctmassert_frame_equalall_parsersparserdataresultexpecteds        \/var/www/horilla/myenv/lib/python3.12/site-packages/pandas/tests/io/parser/test_na_values.pytest_string_nasr*      s    FD
 __Xd^,F	3,rvvsC.@AH }}	!#V#V&(+    c                 R   | }d}t        ddgt        j                  dgt        j                  t        j                  ggddg      }|j                  dk(  r"d |j                  d	d
gdf<   d |j                  d<   |j                  t        |            }t        j                  ||       y )NzA,B
foo,bar
NA,baz
NaN,nan
foobarbazr   r   r   r   r   r   r   r   )	r   r   r   r   r    r   r   r!   r"   )r$   r%   r&   r(   r'   s        r)   test_detect_string_nar1   -   s    FD
 
"&&%266266*:;c3ZH }}	!$(aVS[!#V__Xd^,F&(+r+   	na_valuesz-999.0z-999ig     8r&   zA,B
-999,1.2
2,-999
3,4.5
z"A,B
-999,1.200
2,-999.000
3,4.500
c                 <   | }t        t        j                  dgdt        j                  gddggddg      }|j                  dk(  rUt	        d	 |D              sCd
}t        j                  t        |      5  |j                  t        |      |       d d d        y |j                  dk(  r5d|v r1t
        j                  j                  d      }|j                  |       |j                  t        |      |      }t        j                  ||       y # 1 sw Y   y xY w)Ng333333?       @g      @g      @r   r   r   r   c              3   <   K   | ]  }t        |t                y w)N)
isinstancestr).0xs     r)   	<genexpr>z,test_non_string_na_values.<locals>.<genexpr>_   s     -TQjC.@-Ts   9The 'pyarrow' engine requires all na_values to be stringsmatchr2   z-999.000z4pyarrow engined does not recognize equivalent floatsreason)r   r   r   r   allpytestraises	TypeErrorr   r   markxfailapplymarkerr!   r"   )	r$   r&   r2   requestr%   r(   msgrE   r'   s	            r)   test_non_string_na_valuesrJ   >   s    < F2663-#rvvc
CcSVZXH}}	!#-T)-T*TI]]9C0 	AOOHTNiO@	A	)	#
d(: {{  I ! 
 	D!__Xd^y_AF&(+	As   5DDc                 |  	 h d}|t         k(  sJ | }t        |      		fd}t        dj                  t	        |      D cg c]  \  }} |||       c}}            }t        t        j                  t        	      t        	            }|j                  |d       }t        j                  ||       y c c}}w )N>   #NAN/An/a#N/A-NaN-nan<NA>1.#IND-1.#IND1.#QNAN#N/A N/A-1.#QNAN NANaNr   NULLNonenullc                     | dk(  rd}n| dkD  rdj                  dg| z        } | }| dz
  k  r!dj                  dg| z
  dz
  z        }| | }|S )Nr   rX   ,r   )join)ivbufjoinednvs       r)   r   z!test_default_na_values.<locals>.f   sr    6CU''3%!)$CaSkrAv:WWcUb1fqj12FE&"C
r+   
)r   index)header)r   lenr   r`   	enumerater   r   r   ranger   r!   r"   )
r$   
_NA_VALUESr%   r   ra   rb   r&   r(   r'   re   s
            @r)   test_default_na_valuesrm   q   s    J* &&&F	ZB DII9Z3HI41aqAwIJKDr%)DH__T$_/F&(+	 Js   B8r/   c                    | }d}t        dt        j                  dgt        j                  dt        j                  gddt        j                  ggg d      }|j                  d	k(  rEd
}t	        j
                  t        |      5  |j                  t        |      |dg       d d d        y |j                  t        |      |dg      }t        j                  ||       y # 1 sw Y   y xY w)Nz3A,B,C
ignore,this,row
1,NA,3
-1.#IND,5,baz
7,8,NaN
      ?            r   r   r   z@skiprows argument must be an integer when using engine='pyarrow'r<   r   )r2   skiprowsr   r   r   r   rB   rC   
ValueErrorr   r   r!   r"   )r$   r2   r%   r&   r(   rI   r'   s          r)   test_custom_na_valuesrw      s    FD 
rvvq	BFFArvv.Arvv?H }}	!P]]:S1 	OOOHTNi1#ON	O__Xd^yA3_OF&(+	Os   C!!C*c                    d}| }|j                  t        |            }t        t        j                  dt        j
                  dgt              t        j                  ddt        j
                  gt              g dd      }|j                  dk(  rd |j                  d<   d |j                  d	<   t        j                  ||       y )
Nz1A,B,C
True,False,True
NA,True,False
False,NA,TrueTFdtype)TFTr   r   )r   r   r0   )r   r   r   r   arrayr   objectr   r    r!   r"   r$   r&   r%   r'   r(   s        r)   test_bool_na_valuesr~      s    D F__Xd^,F4/v>5$/v>$	
H }}	!#V#V&(+r+   c                    d}| }|j                   dk(  rHd}t        j                  t        |      5  |j	                  t        |      dgdgd       d d d        y |j	                  t        |      dgdgd      }t        t        j                  dt        j                  dgt        j                  dt        j                  dgt        j                  dt        j                  dgd	      }t        j                  ||       y # 1 sw Y   y xY w)
Nz3A,B,C
foo,bar,NA
bar,foo,foo
foo,bar,NA
bar,foo,foor   z;pyarrow engine doesn't support passing a dict for na_valuesr<   r-   r.   )r   r   r>   r   r   rB   rC   rv   r   r   r   r   r   r!   r"   r$   r&   r%   rI   dfr(   s         r)   test_na_value_dictr      s    D
 F}}	!K]]:S1 	TOOHTNUG5'6ROS	T	$%w3O	PB&&%/&&%/&&%/	
H "h'	Ts   "DDzindex_col,expectedr   rq   )r   r   r   r   namerg   r   )r   r   )r   r   r   )namesc                     d}| }|j                  t        |      t               |      }t        j                  ||       y )Nza,b,c,d
0,NA,1,5
)r2   	index_col)r   r   setr!   r"   )r$   r   r(   r&   r%   r'   s         r)   test_na_value_dict_multi_indexr      s:    0D F__Xd^su	_RF&(+r+   zkwargs,expectedr   r   er   r   r   rp      rq      rr   onetwothreefivesevenr   )r   r   Fr2   keep_default_nar   r   rX   r   r   r   r   )r   r   r   r   r   rX   r   rX   r   c                    d}| }|j                   dk(  rd|v rVt        |d   t              rCd}t        j                  t
        |      5   |j                  t        |      fi | d d d        y t        j                  j                         }|j                  |        |j                  t        |      fi |}t        j                  ||       y # 1 sw Y   y xY w)NzAA,B,C
a,1,one
b,2,two
,3,three
d,4,nan
e,5,five
nan,6,
g,7,seven
r   r2   ?The pyarrow engine doesn't support passing a dict for na_valuesr<   )r   r6   dictrB   rC   rv   r   r   rE   rF   rG   r!   r"   )	r$   kwargsr(   rH   r&   r%   rI   rE   r'   s	            r)   test_na_values_keep_defaultr     s    \	D F}}	!& Z{0CT%JSCz5 :9&9:{{  "D!V__Xd^6v6F&(+:s   CCc                     d}| }|j                  t        |      d      }t        g dg dg dd      }t        j                  ||       y )NzAA,B,C
a,1,None
b,2,two
,3,None
d,4,nan
e,5,five
nan,6,
g,7,seven
F)r   r   r   )r\   r   r\   r   r   rX   r   r   r   r   r   r!   r"   r}   s        r)   !test_no_na_values_no_keep_defaultr   K  sP    	D F__Xd^U_CF5&D	
H &(+r+   c                 x   d}| }|j                   dk(  rGd}t        j                  t        |      5  |j	                  t        |      ddgid       d d d        y |j	                  t        |      ddgid      }t        d	gt        j                  gd
      }t        j                  ||       y # 1 sw Y   y xY w)Nza,b
,2r   r   r<   r   2Fr   rX   r   r   r   )r$   r&   r%   rI   r'   r(   s         r)   &test_no_keep_default_na_dict_na_valuesr   e  s    DF}}	!O]]:S1 	OO3,  	 	__3,  F t266(34H&(+	 	s   !B00B9c                 t   d}| }|j                   dk(  rFd}t        j                  t        |      5  |j	                  t        |      ddid       d d d        y |j	                  t        |      ddid      }t        d	gt        j                  gd
      }t        j                  ||       y # 1 sw Y   y xY w)Nza,b
1,2r   r   r<   r   r   Fr   r   r   r   r   s         r)   -test_no_keep_default_na_dict_na_scalar_valuesr   y  s     DF}}	!O]]:S1 	WOOHTNsAhPUOV	W	$C8U	SBs"&&23H"h'	Ws    B..B7col_zero_na_valuesi 113125c                    d}| }t        t        j                  dgt        j                  dgdt        j                  gddgddgd	d
gt        j                  dgd      }|j                  dk(  rJd}t	        j
                  t        |      5  |j                  t        |      d dd
dd|d       d d d        y |j                  t        |      d dd
dd|d      }t        j                  ||       y # 1 sw Y   y xY w)Nz_113125,"blah","/blaha",kjsdkj,412.166,225.874,214.008
729639,"qwer","",asdfkj,466.681,,252.373
g    ND&Aqwerz/blahakjsdkjasdfkjg-y@g7A`*}@z225.874rX   g-o@)r   r   r   rp   r   rq   r   r   r   r<   Fz214.008blah)r   r   r   r   )rh   r   r2   ru   )r$   r   r&   r%   r(   rI   r'   s          r)   1test_no_keep_default_na_dict_na_values_diff_reprsr     s   D F!"&&!(#!2 	

H }}	!O]]:S1 	OO % Y6>PQ	  	 	__Y66HI	  F &(+	 	s   
$C11C:zna_filter,row_dataTr   rp   r   1r   3c                     d}| }|j                  t        |      dg|      }t        |ddg      }t        j                  ||       y )NzA,B
1,A
nan,B
3,C
r   )r2   	na_filterr   r   r   )r$   r   row_datar&   r%   r'   r(   s          r)   !test_na_values_na_filter_overrider     sJ    D F__Xd^u	_RFC:6H&(+r+   c                 V   | }d}|j                  t        |            }t        dddddt        j                  t        j                  t        j                  gdddd	d
t        j                  t        j                  t        j                  ggg d      }t        j                  ||       y )NzlDate,Currency,Symbol,Type,Units,UnitPrice,Cost,Tax
2012-03-14,USD,AAPL,BUY,1000
2012-05-12,USD,SBUX,SELL,500z
2012-03-14USDAAPLBUYi  z
2012-05-12SBUXSELLi  )DateCurrencySymbolTypeUnits	UnitPriceCostTaxr   )r   r   r   r   r   r!   r"   r#   s        r)   test_na_trailing_columnsr     s    F D
 __Xd^,F5&%rvvrvvrvvN5&&#rvvrvvrvvN	
	
H  &(+r+   zna_values,row_datar4   r   ro   c                 f   | }ddg}d}|j                   dk(  rot        |t              r_t        |t              r	t        }d}nt        }d}t        j                  ||      5  |j                  t        |      ||       d d d        y |j                   dk(  rDd}t        j                  t        |      5  |j                  t        |      ||       d d d        y |j                  t        |      ||      }t        ||	      }	t        j                  ||	       y # 1 sw Y   y xY w# 1 sw Y   y xY w)
Nr   r   1,2
2,1r   r   r;   r<   r   r2   r   )r   r6   r   rv   rD   rB   rC   r   r   r   r!   r"   )
r$   r2   r   r%   r   r&   errrI   r'   r(   s
             r)   test_na_values_scalarr     s	    F#JED}}	!jD&Ai&CSCCMC]]3c* 	NOOHTN%9OM	N	)	#I]]9C0 	NOOHTN%9OM	N__Xd^5I_NF51H&(+	N	Ns    D3D'D$'D0c                    | }ddd}|j                         }ddg}d}t        ddgt        j                  t        j                  gg|	      }|j                  d
k(  rDd}t        j                  t        |      5  |j                  t        |      ||       d d d        y |j                  t        |      ||      }t        j                  ||       t        j                  ||       y # 1 sw Y   y xY w)Nr   r   r   r   r   r   ro   r4   r   r   r   r<   r   )copyr   r   r   r   rB   rC   rv   r   r   r!   r"   assert_dict_equal)	r$   r%   r2   na_values_copyr   r&   r(   rI   r'   s	            r)   test_na_values_dict_aliasingr     s    Fa I^^%N#JED3*rvvrvv&67GH}}	!O]]:S1 	NOOHTN%9OM	N__Xd^5I_NF&(+N3	Ns   9C**C3c                 n   d}| }ddi}|j                   dk(  rCd}t        j                  t        |      5  |j	                  t        |      |       d d d        y |j	                  t        |      |      }t        dt        j                  d	gi      }t        j                  ||       y # 1 sw Y   y xY w)
Nza
foo
1r   r-   r   r   r<   r>   r   r   r   )r$   r&   r%   r2   rI   r'   r(   s          r)   test_na_values_dict_col_indexr     s    DFE
I}}	!O]]:S1 	AOOHTNiO@	A__Xd^y_AF#{+,H&(+	As   B++B4zdata,kwargs,expectedl            rf   l           z,1z
,2z
1c                    | }|j                   dk(  rId|v rEd}t        j                  t        |      5   |j                  t        |      fdd i| d d d        y |j                   dk(  r1t        j                  j                  d      }|j                  |        |j                  t        |      fdd i|}t        j                  ||       y # 1 sw Y   y xY w)Nr   r2   r;   r<   rh   z!Returns float64 instead of objectr?   )r   rB   rC   rD   r   r   rE   rF   rG   r!   r"   )	r$   r&   r   r(   rH   r%   rI   rE   r'   s	            r)   test_na_values_uint64r   .  s     F}}	!kV&;I]]9C0 	CFOOHTNB4B6B	C	)	#{{  (K LD!V__Xd^CDCFCF&(+	Cs   CCc                     d}| }t        ddgit        dgd            }|j                  t        |      dd	
      }t	        j
                  ||       y )Nza,1
b,2r   r   r   r   r   r   r   F)r   r   )r   r   r   r   r!   r"   )r$   r&   r%   r(   r'   s        r)   *test_empty_na_values_no_default_with_indexr   K  sO    DF#s5#S+ABH__Xd^q%_PF&(+r+   zna_filter,index_data5g      @c                 :   | }d}|j                   dk(  r5|du r1t        j                  j                  d      }|j	                  |       t        ddgdd	gd
t        |d            }|j                  t        |      dg|      }t        j                  ||       y )Na,b,c
1,,3
4,5,6r   Fzmismatched index resultr?   r   r   rp   r   )r   r   r   r   r   )r   r   )r   rB   rE   rF   rG   r   r   r   r   r!   r"   )	r$   r   
index_datarH   r%   r&   rE   r(   r'   s	            r)   test_no_na_filter_on_indexr   U  s     FD}}	!i5&8{{  (A BD!1vQF35RU;VWH__Xd^si_PF&(+r+   c                     | }d}|j                  t        |      dgddg      }t        dt        j                  gdt        j                  gdt        d	d
gd            }t        j                  ||       y )Nzidx,col1,col2
1,3,4
2,inf,-infr   infz-inf)r   r2   rp   r   )col1col2r   r   idxr   r   )r   r   r   r   r   r   r!   r"   )r$   r%   r&   outr(   s        r)   !test_inf_na_values_with_int_indexr   h  sm    F-D //(4.QCE6?/
SCRVVq"&&k2%AU:SH #x(r+   r   c                     | }d}|rt         j                  nd}t        ddg|dgddgd      }|j                  t	        |      |t
        	      }t        j                  ||       y )
Nr   rX   r   4r   r   6r   )r   rz   )r   r   r   r   r   r7   r!   r"   )r$   r   r%   r&   emptyr(   r'   s          r)   +test_na_values_with_dtype_str_and_na_filterr   u  sb     FD  BFFRESzC:NOH__Xd^y_LF&(+r+   zdata, na_values)zfalse,1
,1
trueN)zfalse,1
null,1
trueN)zfalse,1
nan,1
trueN)false,1
foo,1
truer-   r   r-   c                     | }dj                  ddg      }t        j                  t        |      5  |j	                  t        |      d ddgddi|       d d d        y # 1 sw Y   y xY w)	N|z(Bool column has NA values in column [0a]zRcannot safely convert passed user dtype of bool for object dtyped data in column 0r<   r   r   bool)rh   r   rz   r2   )r`   rB   rC   rv   r   r   )r$   r&   r2   r%   rI   s        r)   !test_cast_NA_to_bool_raises_errorr     su     F
((66	
C 
z	- 
TN*- 	 	

 
 
s   $AA'c           	          | }d}|j                  t        |      d g dt        t        t        d      j                         }t	        ddgddgdd	gdd
dg      }t        j                  ||       y )NzDFile: small.csv,,
10010010233,0123,654
foo,,bar
01001000155,4530,898)r   r   col3)rh   r   rz   100100102330100100015501234530654898r   rp   r   )r   r   r7   dropnar   r!   r"   r#   s        r)   test_str_nan_droppedr     s     FD
 __&C5	  
 fh  "M2V$EN	

 !fH &(+r+   c           	         | }d}|j                   dk(  rXd}t        j                  t        |      5  |j	                  t        |      t        t        d            ddi       d d d        y |j	                  t        |      t        t        d            ddi      }t        d	gdgt        j                  gd
      }t        j                  ||       y # 1 sw Y   y xY w)NzA,B,B
X,Y,Z
1,2,infr   r   r<   r   r   Zr   )rh   r2   r   ))r   X)r   Yr   )r   rB   rC   rv   r   r   listrk   r   r   r   r!   r"   )r$   r%   r&   rI   r'   r(   s         r)   test_nan_multi_indexr     s    F"D}}	!O]]:S1 	OOtE!H~*eAT  	 	__tE!H~*e9L  F 	
H &(+%	 	s   2CCc                     | }d}t        j                  t        d      5  |j                  t	        |      d       d d d        y # 1 sw Y   y xY w)N0
NaN
True
False
z	NA valuesr<   r   ry   rB   rC   rv   r   r   r$   r%   r&   s      r)   test_bool_and_nan_to_boolr     sI     FD
 
z	5 6f56 6 6   AAc                     | }d}t        j                  t        d      5  |j                  t	        |      d       d d d        y # 1 sw Y   y xY w)Nr   zconvert|NoneTyper<   intry   r   r   s      r)   test_bool_and_nan_to_intr    sH    FD
 
z);	< 5e45 5 5r   c                     | }d}|j                  t        |      d      }t        j                  dt        j
                  ddgi      }t        j                  ||       y )Nr   floatry   0ro   g        )r   r   r   	from_dictr   r   r!   r"   r#   s        r)   test_bool_and_nan_to_floatr    sV    FD
 __Xd^7_;F""C"&&#s);#<=H&(+r+   )6__doc__ior   numpyr   rB   pandas._libs.parsersr   pandasr   r   r   pandas._testing_testingr!   rE   filterwarnings
pytestmarkusefixturesxfail_pyarrowskip_pyarrowr*   r1   parametrizerJ   rm   rw   r~   r   r   from_tuplesr   r   r   r   r   r   r   r   r   r   r   r7   r   r   r   r   r   r   r   r   r   r  r   r+   r)   <module>r     s<      . 
 [[''C
 ''8{{&&~6,$," 	6	v		
			 
	
	,6,0-,` uug&67, 8,*,((2  CRVVHA3aS9sQTAUV	

 FvvhaS),j,,fXc3ZH	
 #JvvhaS),j,,fXc3ZH	
.,/.,  sBFFCbffcB.Q		
 !#,G=.L		
 %E:&&#r3UC@.L		
 ,-sBFFCbffcB.Q		
?),Z,[,Z,2,4,(($ -/AB$, C$,N 	C266266*QH56	#seS\C:67, , , ,6 	
bffc]S"&&M*+q	S#J(89:,,64,,"  JI.5'"s5z3y>23	

 
Ud	V	#RSZOb!W3M)NO	Ue	kE73Y{5KL,,", eb#Y/$1FG,, 
) tUm4, 5 , #'&'	%)	#u.


 
, , ,8,8 	6 	6	5
,r+   