
    =wg                        d Z ddlZddlmZ ddlmZ ddlmZmZ ddlm	Z	m
Z
mZmZ ddlmZmZmZmZ ddlmZ dd	lmZmZ dd
lmZmZmZ ddlmZmZ ddlmZ ddlmZm Z m!Z!m"Z" ddlm#Z#m$Z$ ddlm%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. 	 ddl/Z/ e	d      Z1 ejd                  dd      Z3 ejd                  d      Z4 ejd                  d      Z5 ejl                   ejn                               Z8 G d dejr                        Z:d Z;d Z< G d dejz                        Z> G d dej~                        Z@ G d d ej                        ZB G d! d"ej                        ZD G d# d$ej                        ZF G d% d&ej                        ZH G d' d(e      ZI G d) d*e      ZJ G d+ d,ej                        ZLy# e0$ r dZ/Y 0w xY w)-zH
This module implements a "codec" for writing/reading Whoosh X indexes.
    N)array)defaultdict)columnsformats)b
bytes_typestring_typeinteger_types)dumpsloads	iteritemsxrange)base)compound
filetables)ListMatcher
ReadTooFarLeafMatcher)TermInfoTermNotFound)
emptybytes)_SHORT_SIZE	_INT_SIZE
_LONG_SIZE_FLOAT_SIZE)pack_ushortunpack_ushort)pack_int
unpack_int	pack_longunpack_long)delta_encodedelta_decode)length_to_bytebyte_to_lengthW3BlB)defaultIic                   h    e Zd ZdZdZdZdZddZd Zd Z	ddZ
dd
Zd Zd Zd Zed        Zd Zy	)W3Codecz.trmz.pstz.vpsz.colc                 .    || _         || _        || _        y N)_blocklimit_compression_inlinelimit)self
blocklimitcompressioninlinelimits       K/var/www/horilla/myenv/lib/python3.12/site-packages/whoosh/codec/whoosh3.py__init__zW3Codec.__init__M   s    %''    c                     t        | ||      S r.   )W3PerDocWriterr2   storagesegments      r6   per_document_writerzW3Codec.per_document_writerU   s    dGW55r8   c                     t        | ||      S r.   )W3FieldWriterr;   s      r6   field_writerzW3Codec.field_writerY   s    T7G44r8   c                 ^    t        || j                  || j                  | j                        S )N)r3   byteidsr4   r5   )W3PostingsWriterr/   r0   r1   )r2   dbfilerC   s      r6   postings_writerzW3Codec.postings_writer^   s-    43C3C(/T=N=N,0,=,=? 	?r8   Nc           	          |j                         r(|j                         \  }}}t        |||||||      }	|	S |j                         \  }
}t	        ||
||||      }	|	S )N)scorertermterminforI   rH   )
is_inlinedinlined_postingsr   extentW3LeafMatcher)r2   rE   rJ   format_rI   rH   idsweightsvaluesmoffsetlengths               r6   postings_readerzW3Codec.postings_readerc   ss      $,#<#<#> C&C&'&!%:A  &__.NFFfffgD%+-Ar8   c                     t        ||      S r.   )W3PerDocReaderr;   s      r6   per_document_readerzW3Codec.per_document_readerr   s    gw//r8   c                     |j                  | j                        }|j                  |      }|j                  |      }|j                  || j                        }t        | |||      S r.   )make_filename	TERMS_EXTfile_length	open_file	POSTS_EXTW3TermsReader)r2   r<   r=   tinametilentifilepostfiles          r6   terms_readerzW3Codec.terms_readeru   s]    &&t~~6##F+""6*$$Wdnn=T65(;;r8   c                      yNT r2   s    r6   supports_columnszW3Codec.supports_columns   s    r8   c                 `    dj                  d|| j                  f      }|j                  |      S )N .)join
COLUMN_EXTr\   )clsr=   	fieldnameexts       r6   column_filenamezW3Codec.column_filename   s,    ggsIs~~67$$S))r8   c                     t        | |      S r.   )	W3Segment)r2   r<   	indexnames      r6   new_segmentzW3Codec.new_segment   s    y))r8   )         F)NN)__name__
__module____qualname__r]   r`   
VPOSTS_EXTrp   r7   r>   rA   rF   rW   rZ   rf   rk   classmethodrt   rx   ri   r8   r6   r,   r,   F   sW    IIJJ(65
?
0< * **r8   r,   c                     d| z  S )Nz_%s_vecri   rr   s    r6   	_vecfieldr          y  r8   c                     d| z  S )Nz_%s_lenri   r   s    r6   	_lenfieldr      r   r8   c                   T    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zy)r:   c                 T   || _         || _        || _        |j                  d|j                  z        }t        j                  |      | _        i | _        | j                  dt               t        t              | _        d| _        d | _        d | _        d| _        d| _        d | _        y )Nz%s.tmp_storedr   F)_codec_storage_segmenttemp_storagerw   r   CompoundWriter_cols_colwriters_create_columnSTORED_COLUMNr   int_fieldlengths	_doccount_docnum_storedfields_indoc	is_closed
_vpostfile)r2   codecr<   r=   tempsts        r6   r7   zW3PerDocWriter.__init__   s    %%h1B1B&BC,,V4
I}5(-! r8   c                 N    | j                   j                  | j                  |      S r.   r   create_filer   r2   rs   s     r6   _create_filezW3PerDocWriter._create_file       }}((<<r8   c                     || j                   v S r.   r   r2   rr   s     r6   _has_columnzW3PerDocWriter._has_column   s    D,,,,r8   c                     | j                   }||v rt        d|z        | j                  j                  |      }|j	                  |      ||<   y )NzAlready added column %r)r   	Exceptionr   r   writer)r2   rr   columnwritersfs        r6   r   zW3PerDocWriter._create_column   sM    ""5	ABBJJ""9-#]]1-	r8   c                      | j                   |   S r.   r   r   s     r6   _get_columnzW3PerDocWriter._get_column   s    	**r8   c                     | j                  t        j                        | _        | j                  j	                  t        d             y )NVPST)r   r,   r   r   writer   rj   s    r6   _prep_vectorszW3PerDocWriter._prep_vectors   s2    ++G,>,>? 	ai(r8   c                     | j                   rt        d      || j                  k7  rt        d|d| j                        || _        | xj                  dz  c_        i | _        d| _         y )Nz&Called start_doc when already in a doczCalled start_doc(z) was expecting r{   T)r   r   r   r   r   r2   docnums     r6   	start_doczW3PerDocWriter.start_doc   sb    ;;DEET^^#%t~~7 8 8 !r8   c                     ||| j                   |<   |rEt        |      }t        |      }| j                  |t        |       | j
                  |xx   |z  cc<   y y r.   )r   r   r$   add_column_valueLENGTHS_COLUMNr   )r2   rr   fieldobjvaluerV   lenfieldlbs          r6   	add_fieldzW3PerDocWriter.add_field   s\    ,1Dy) +H'B!!(NB?y)V3) r8   c                    |sy | j                   | j                          | j                  j                  | j                   d      }|j	                  |j
                  t                      |D ]  \  }}}|j                  |||        |j                         }t        |      }	|j                         \  }
}|
dk7  sJ | j                  |	t        |
       | j                  |	dz   t        |       y )NTrC   r   L)r   r   r   rF   start_postingsvector
W3TermInfoadd_postingfinish_postingsr   rN   r   VECTOR_COLUMNVECTOR_LEN_COLUMN)r2   rr   r   itemsvpostwritertextweightvbytesvinfovecfieldrU   rV   s               r6   add_vector_itemszW3PerDocWriter.add_vector_items   s    ??"  kk11$//41P""8??JLA$) 	: D&&##D&&9	: ++- Y'{{hv>hn.?Hr8   c                 |    | j                   }|r'| j                  dt        |       |j                          d| _        y )Nr   F)r   r   r   clearr   )r2   sfs     r6   
finish_doczW3PerDocWriter.finish_doc   s2    !!)]B?HHJr8   c                 B    t         j                  | j                  |      S r.   )r,   rt   r   r   s     r6   _column_filenamezW3PerDocWriter._column_filename   s    &&t}}i@@r8   c                    | j                   | j                          | j                  | j                  _        | j                  j                         D ]  }|j                  | j                          | j                  j                  | j                  | j                         | j                  r| j                  j                          d| _        y rh   )r   r   r   r   r   rS   finishr   r   save_as_filesr   r   r   closer   )r2   r   s     r6   r   zW3PerDocWriter.close  s    ;;"OO&*&8&8# &&--/ 	*FMM$..)	*

  0E0EF ??OO!!#r8   N)r}   r~   r   r7   r   r   r   r   r   r   r   r   r   r   r   ri   r8   r6   r:   r:      s@    *=-.+)
	4I.Ar8   r:   c                   <    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
y	)
r@   c                    || _         || _        || _        d | _        d | _        d | _        d | _        d | _        | j                  t        j                        }t        j                  |      | _        i x| _        | j                  j                  d<   | j                  t        j                         | _        d | _        d| _        d| _        y )NfieldmapF)r   r   r   
_fieldname_fieldid_btext	_fieldobj_formatr   r,   r]   r   OrderedHashWriter_tindex	_fieldmapextrasr`   	_postfile_postwriter_infieldr   )r2   r   r<   r=   _tifiles        r6   r7   zW3FieldWriter.__init__  s    ##G$5$56!33G<;==,,Z8**7+<+<=r8   c                 N    | j                   j                  | j                  |      S r.   r   r   s     r6   r   zW3FieldWriter._create_file*  r   r8   c                    | j                   }||v r||   | _        nt        |      | _        | j                  ||<   || _        || _        |j
                  | _        d| _        | j                  j                  | j                        | _        y rh   )r   r   lenr   r   formatr   r   r   rF   r   r   )r2   rr   r   fmaps       r6   start_fieldzW3FieldWriter.start_field-  ss    ~~ ODMIDM"mmDO#!  ;;66t~~Fr8   c                     | j                   t        d      || _        | j                   j                  | j                  j
                  t                      y )Nz$Called start_term before start_field)r   r   r   r   r   r   r   )r2   btexts     r6   
start_termzW3FieldWriter.start_term=  sC    #BCC''(=(=
Mr8   c                 @    | j                   j                  ||||       y r.   )r   r   )r2   r   r   r   rV   s        r6   addzW3FieldWriter.addC  s    $$VVVVDr8   c                     | j                   j                         }t        | j                        | j                  z   }|j                         }| j                  j                  ||       y r.   )r   r   r   r   r   to_bytesr   r   )r2   rJ   keybytesvalbytess       r6   finish_termzW3FieldWriter.finish_termF  sQ    ##335 t}}-;$$&8,r8   c                 N    | j                   st        d      d| _         d | _        y )Nz&Called finish_field before start_fieldF)r   r   r   rj   s    r6   finish_fieldzW3FieldWriter.finish_fieldP  s$    }}DEEr8   c                 z    | j                   j                          | j                  j                          d| _        y rh   )r   r   r   r   rj   s    r6   r   zW3FieldWriter.closeV  s)    r8   N)r}   r~   r   r7   r   r   r   r   r   r   r   ri   r8   r6   r@   r@     s-    *=G NE- r8   r@   c                       e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd ZddZd Zd Zd Zd Zd Zd Zd Zd Zd Zy)rY   c                     || _         || _        |j                         | _        d | _        i | _        i | _        i | _        i | _        y r.   )	r   r   doc_count_allr   r   	_colfiles_readers_minlengths_maxlengthsr;   s      r6   r7   zW3PerDocReader.__init___  sE     ..0r8   c                     | j                   j                         D ]  \  }}}|j                           | j                  r| j                  j                          y y r.   )r  rS   r   r   )r2   colfile_s      r6   r   zW3PerDocReader.closej  sI    !^^224 	MGQMMO	??OO!!# r8   c                 P    | j                   | j                  j                         z
  S r.   )r   r   deleted_countrj   s    r6   	doc_countzW3PerDocReader.doc_countp  s    ~~ ; ; ===r8   c                     | j                   S r.   r   rj   s    r6   r  zW3PerDocReader.doc_count_alls      ~~r8   c                 6    | j                   j                         S r.   )r   has_deletionsrj   s    r6   r  zW3PerDocReader.has_deletionsx  s    }}**,,r8   c                 8    | j                   j                  |      S r.   )r   
is_deletedr   s     r6   r  zW3PerDocReader.is_deleted{  s    }}''//r8   c                 6    | j                   j                         S r.   )r   deleted_docsrj   s    r6   r  zW3PerDocReader.deleted_docs~  s    }}))++r8   c                 x    t         j                  | j                  |      }| j                  j	                  |      S r.   )r,   rt   r   r   file_exists)r2   rr   filenames      r6   
has_columnzW3PerDocReader.has_column  s-    **4==)D}}((22r8   c                     t         j                  | j                  |      }| j                  j	                  |      }| j                  j                  |      }|d|fS Nr   )r,   rt   r   r   r^   r_   )r2   rr   r  rV   r  s        r6   _get_column_filezW3PerDocReader._get_column_file  sM    **4==)D**84--))(36!!r8   c                     || j                   vr| j                  |      | j                   |<   | j                   |   \  }}}|j                  |||| j                        S r.   )r  r  readerr   )r2   rr   r   r  rU   rV   s         r6   column_readerzW3PerDocReader.column_reader  sU    DNN*(,(=(=i(HDNN9%"&..";}}WffdnnEEr8   c                     || j                   v r| j                   |   S | j                  |      sy | j                  ||      }|| j                   |<   |S r.   )r  r  r  )r2   rr   r   r  s       r6   _cached_readerzW3PerDocReader._cached_reader  sQ    %==++??9-''	6:F'-DMM)$Mr8   c                     || j                   kD  rt        d|| j                   fz        t        |      }| j                  |t              }||S ||   }|rt        |      S y )NzAsked for docnum %r of %d)r   
IndexErrorr   r   r   r%   )r2   r   rr   r(   r   r  lbytes          r6   doc_field_lengthzW3PerDocReader.doc_field_length  ss    DNN"8 &78 9 9 Y'$$X~>>Nv!%(( r8   c                 N    | j                   j                  j                  |d      S r  )r   r   getr   s     r6   field_lengthzW3PerDocReader.field_length  s    }}**..y!<<r8   c                     ||v r||   S t        |      }| j                  |t              }t         ||            }|||<   |S r.   )r   r   r   r%   )r2   rr   opcacher   r  rV   s          r6   _minmax_lengthzW3PerDocReader._minmax_length  sN    ##Y'$$X~>6
+!ir8   c                 D    | j                  |t        | j                        S r.   )r+  minr  r   s     r6   min_field_lengthzW3PerDocReader.min_field_length      ""9c43C3CDDr8   c                 D    | j                  |t        | j                        S r.   )r+  maxr  r   s     r6   max_field_lengthzW3PerDocReader.max_field_length  r/  r8   c                 z    | j                   j                  | j                  t        j                        }|| _        y r.   )r   r_   r   r,   r   r   )r2   r   s     r6   r   zW3PerDocReader._prep_vectors  s)    MM##DMM73E3EFr8   c                     || j                   kD  rt        d|| j                   fz        t        |      }| j                  |t              |   }| j                  |dz   t              }|r	||   }||fS d}||fS )NzAsked for document %r of %dr   )r   r"  r   r   r   )r2   r   rr   r   rU   lreaderrV   s          r6   _vector_extentzW3PerDocReader._vector_extent  s    DNN": &78 9 9Y' $$X}=fE %%hnmDV_F v~ Fv~r8   c                 l    | j                  t        |            r| j                  ||      \  }}|dk7  S y)Nr   F)r  r   r7  )r2   r   rr   rU   rV   s        r6   
has_vectorzW3PerDocReader.has_vector  s6    ??9Y/0!00CNFFQ;r8   c                     | j                   | j                          | j                  ||      \  }}|st        d|d|      t	        | j                   |||d      }|S )NzField z has no vector in docnum Tr   )r   r   r7  r   rO   )r2   r   rr   rP   rU   rV   rT   s          r6   r   zW3PerDocReader.vector  sf    ??" ,,VY?&0 1 1$//667"&(r8   c                 D    | j                  dt              }||   }|i }|S )Nr   )r   r   )r2   r   r  vs       r6   stored_fieldszW3PerDocReader.stored_fields  s,    $$Y>6N9Ar8   N)r   )r}   r~   r   r7   r   r  r  r  r  r  r  r  r  r   r$  r'  r+  r.  r2  r   r7  r9  r   r=  ri   r8   r6   rY   rY   ^  sn    	$>
-0,
3"F	)=EE
&	r8   rY   c                   6    e Zd Zd Zd Zd Zd Zd Zd Zd Z	y)	W3FieldCursorc                    || _         || _        || _        || _        || _         ||d      }| j                   j                  |      | _        | j                  | _        d | _        d | _	        d | _
        | j                          y )Nr8   )r   r   	_keycoder_keydecoderr   closest_key_pos	_startpos_pos_text_datapos_datalennext)r2   tindexrr   keycoder
keydecoderr   prefixbytess          r6   r7   zW3FieldCursor.__init__  sq    #!%!y#.55kBNN	
		r8   c                 D    | j                   | _        | j                         S r.   )rD  rE  rI  rj   s    r6   firstzW3FieldCursor.first  s    NN	yy{r8   c                     t        |t              s| j                  j                  |      }| j	                  | j
                  |      }| j                  j                  |      | _        | j                         S r.   )

isinstancer   r   r   rA  r   r   rC  rE  rI  )r2   rI   keys      r6   findzW3FieldCursor.find
  sV    $
+>>**40DnnT__d3LL005	yy{r8   c                 |   | j                   | j                  j                  | j                         }|m|\  }}}| j                  |      \  }}|| j                  k(  rD||z   | _         | j
                  j                  |      | _        || _        || _	        | j                  S d x| _        x| _         x| _        | _	        y r.   )
rE  r   key_and_range_atrB  r   r   
from_bytesrF  rG  rH  )r2   keyrngr   dataposdatalenfnamer   s          r6   rI  zW3FieldCursor.next  s    99 \\22499=F!-3*'7"..x8tDOO+ '' 1DI!%!:!:4!@DJ$+DM$+DM::%AEE
ETYEr8   c                     | j                   S r.   )rF  rj   s    r6   r   zW3FieldCursor.text!  s    zzr8   c                     | j                   y | j                  j                  j                  | j                  | j
                        }t        j                  |      S r.   )rE  r   rE   r&  rG  rH  r   rV  )r2   	databytess     r6   	term_infozW3FieldCursor.term_info$  sD    99LL''++DMM4==I	$$Y//r8   c                     | j                   d uS r.   )rE  rj   s    r6   is_validzW3FieldCursor.is_valid+  s    yy$$r8   N)
r}   r~   r   r7   rO  rS  rI  r   r^  r`  ri   r8   r6   r?  r?    s%      0%r8   r?  c                   n    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd ZddZd Zy)ra   c                 4   || _         || _        t        j                  ||      | _        | j                  j
                  d   | _        || _        d gt        | j                        z  | _	        t        | j                        D ]  \  }}|| j                  |<    y )Nr   )r   _dbfiler   OrderedHashReaderr   r   r   r   r   _fieldunmapr   )r2   r   rE   rV   re   rr   nums          r6   r7   zW3TermsReader.__init__0  s    !33FFC,,Z8! 6C$77'7 	.NIs$-DS!	.r8   c                     t        |t              s
J d|z         | j                  j                  |d      }t	        |      |z   S )Nz	tbytes=%ri  )rQ  r   r   r&  r   )r2   rr   tbytesfnums       r6   rA  zW3TermsReader._keycoder;  sB    &*-C{V/CC-~~!!)U34 6))r8   c                 \    t        |d t               d   }| j                  |   |t        d  fS r  )r   r   re  )r2   r   fieldids      r6   rB  zW3TermsReader._keydecoder@  s5    + 67:((;<*@@@r8   c                 X    | j                   j                  | j                  ||            S r.   )r   range_for_keyrA  )r2   rr   rh  s      r6   _range_for_keyzW3TermsReader._range_for_keyD  s"    ||))$..F*KLLr8   c                 8     | j                   | | j                  v S r.   )rA  r   )r2   rI   s     r6   __contains__zW3TermsReader.__contains__G  s    t~~t$44r8   c                 6    | j                   j                         S r.   )r   keysrj   s    r6   indexed_field_namesz!W3TermsReader.indexed_field_namesJ  s    ~~""$$r8   c                 h    | j                   }| j                  }| j                  }t        |||||      S r.   )r   rA  rB  r?  )r2   rr   r   rJ  coderdecoders         r6   cursorzW3TermsReader.cursorM  s2    ""VYwIIr8   c                 b    | j                   fd| j                  j                         D        S )Nc              3   .   K   | ]  } |        y wr.   ri   .0r   rL  s     r6   	<genexpr>z&W3TermsReader.terms.<locals>.<genexpr>U  s     I
8$I   )rB  r   rr  )r2   rL  s    @r6   termszW3TermsReader.termsS  s&    %%
IT\\5F5F5HIIr8   c                     | j                  ||      }| j                  fd| j                  j                  |      D        S )Nc              3   .   K   | ]  } |        y wr.   ri   rz  s     r6   r|  z+W3TermsReader.terms_from.<locals>.<genexpr>Z  s      8
8$ 8r}  )rA  rB  r   	keys_from)r2   rr   prefixrM  rL  s       @r6   
terms_fromzW3TermsReader.terms_fromW  s?    nnY7%%
8<<))+68 	8r8   c                     t         j                  | j                  fd| j                  j	                         D        S )Nc              3   D   K   | ]  \  }} |       |      f  y wr.   ri   r{  r   r   rL  	tidecoders      r6   r|  z&W3TermsReader.items.<locals>.<genexpr>`  s.      @&Hh H%y':; @    )r   rV  rB  r   r   )r2   rL  r  s    @@r6   r   zW3TermsReader.items]  s;    ))	%%
@*.,,*<*<*>@ 	@r8   c                     | j                  ||      }t        j                  | j                  fd| j                  j                  |      D        S )Nc              3   D   K   | ]  \  }} |       |      f  y wr.   ri   r  s      r6   r|  z+W3TermsReader.items_from.<locals>.<genexpr>g  s.      P&Hh H%y':; Pr  )rA  r   rV  rB  r   
items_from)r2   rr   r  rM  rL  r  s       @@r6   r  zW3TermsReader.items_fromc  sM    nnY7))	%%
P*.,,*A*A+*NP 	Pr8   c                     | j                  ||      }	 t        j                  | j                  |         S # t        $ r t        d|d|      w xY w)NzNo term :)rA  r   rV  r   KeyErrorr   )r2   rr   rh  rR  s       r6   r^  zW3TermsReader.term_infoj  sS    nnY/	F((c):;; 	F)VDEE	Fs	   !6 Ac                 l    | j                  ||      d   }t        j                  | j                  |      S r  )rn  r   read_weightrc  r2   rr   rh  rX  s       r6   	frequencyzW3TermsReader.frequencyq  s0    %%i8;%%dllG<<r8   c                 l    | j                  ||      d   }t        j                  | j                  |      S r  )rn  r   read_doc_freqrc  r  s       r6   doc_frequencyzW3TermsReader.doc_frequencyu  s0    %%i8;''g>>r8   Nc                     | j                  ||      }| j                  j                  | j                  ||||f|      }|S )NrK   )r^  r   rW   r   )r2   rr   rh  rP   rH   rJ   rT   s          r6   matcherzW3TermsReader.matchery  sE    >>)V4KK'''.7-@ ( Qr8   c                 l    | j                   j                          | j                  j                          y r.   )r   r   r   rj   s    r6   r   zW3TermsReader.close  s"    r8   r.   )r}   r~   r   r7   rA  rB  rn  rp  rs  rw  r~  r  r   r  r^  r  r  r  r   ri   r8   r6   ra   ra   /  sY    	.*
AM5%JJ8@PF=?r8   ra   c                   z    e Zd ZdZ	 	 ddZd Zd ZddZd Zd Z	dd	Z
d
 Zd Zd Zd Zd Zd Zd Zd Zd Zy)rD   zThis object writes posting lists to the postings file. It groups postings
    into blocks and tracks block level statistics to makes it easier to skip
    through the postings.
    c                 t    || _         || _        || _        || _        || _        d| _        d | _        d | _        y r  )r   r/   _byteidsr0   r1   _blockcountr   	_terminfo)r2   re   r3   rC   r4   r5   s         r6   r7   zW3PostingsWriter.__init__  s>    !%''r8   c                      | j                   dkD  S r  )r  rj   s    r6   writtenzW3PostingsWriter.written  s    !##r8   c                     | j                   rt        d      t        |t        j                        sJ || _        d| _        | j                          || _         | j                  j                         | _
        y )NzCalled start in a termr   )r  r   rQ  r   Formatr   r  
_new_blockr   tell_startoffset)r2   rP   rJ   s      r6   r   zW3PostingsWriter.start_postings  s^    >>455'7>>222! NN//1r8   Nc                    t        | j                        | j                  k\  r| j                          | j                  rt        |t              s$J d|z         t        |t              s
J d|z         t        |t        t        f      s
J d|z         t        |t              s
J d|z         |t        |t              sJ | j                  j                  |       | j                  j                  |       || j                  kD  r|| _        |r| j                  j                  |       |r2| j                  }|||k  r|| _        || j                   kD  r|| _        y y y )Nzid_=%rz	weight=%rz	vbytes=%r)r   _idsr/   _write_blockr  rQ  r	   r
   r   floatr   append_weights
_maxweight_values
_minlength
_maxlength)r2   id_r   r   rV   	minlengths         r6   r   zW3PostingsWriter.add_posting  s6   
 tyy>T--- ==c;/?C?/c=1A8c>A1&3,/Ev1EE/&*-C{V/CC-~FM!BBB		V$DOO#$DOLL'I FY$6"('"( (	 r8   c                    | j                   }| j                         s[t        |       | j                  k  rC|j	                  |        |j                  | j                  | j                  | j                         nY| j                  r| j                  d       | j                  }| j                  j                         |z
  }|j                  ||       d | _         |S )NT)last)r  r  r   r1   	add_block
set_inliner  r  r  r  r  r   r  
set_extent)r2   rJ   startoffsetrV   s       r6   r   z W3PostingsWriter.finish_postings  s    >> ||~#d)d.?.?"?t$		4==$,,G yy!!t!,++K^^((*[8FV4 r8   c                     | j                   rg n
t        d      | _        t        d      | _        g | _        d | _        d| _        d| _        y )Nr)   r   r   )r  r   r  r  r  r  r  r  rj   s    r6   r  zW3PostingsWriter._new_block  s>     --BU3Z	c
r8   c           
      &   | j                   s| j                  j                  t               | j                  j                  |        | j                         | j                         | j                         f}t        |d      }t        |      dk  rd}| j                  }|rt        j                  ||      }| j                  }t        t        |      |d   | j                  |t!        | j"                        t!        | j$                        fd      }| j                  }t        |      t        |      z   }|r|dz  }|j'                  |       |j                  |       |j                  |       | xj                   dz  c_         | j)                          y )N      r   r5  r{   )r  r   r   WHOOSH3_HEADER_MAGICr  r  	_mini_ids_mini_weights_mini_valuesr   r   r0   zlibcompressr  r  r$   r  r  	write_intr  )	r2   r  datar]  comprQ   	infobytesre   blocklengths	            r6   r  zW3PostingsWriter._write_block  sK    NN  !56 	  &  $"4"4"68I8I8KL$N	y>BD  i6I ii3s8SWdoot)$//:)$//:  !	 >>)ns9~52K;'y!y!Ar8   c                 ^    | j                   }| j                  st        |      }t        |      S r.   )r  r  r"   tupler2   rQ   s     r6   r  zW3PostingsWriter._mini_ids'  s(     ii}}s#CSzr8   c                     | j                   t        d D              ry t        fdD              rd   S t              S )Nc              3   &   K   | ]	  }|d k(    ywg      ?Nri   )r{  ws     r6   r|  z1W3PostingsWriter._mini_weights.<locals>.<genexpr>4  s     )AqCx)s   c              3   .   K   | ]  }|d    k(    yw)r   Nri   )r{  r  rR   s     r6   r|  z1W3PostingsWriter._mini_weights.<locals>.<genexpr>6  s     2Qgaj2r}  r   )r  allr  )r2   rR   s    @r6   r  zW3PostingsWriter._mini_weights/  sA     --)))2'221:>!r8   c                     | j                   j                         }| j                  }||dk  rt        |      }|S |dk(  rd }|S t	        j
                  |      }|S r  )r   fixed_value_sizer  r  r   ro   )r2   	fixedsizerS   vss       r6   r  zW3PostingsWriter._mini_values;  se     LL113		AvB
 		 !^B 	 (B	r8   c                 ,    t        | j                        S r.   )r   r  rj   s    r6   __len__zW3PostingsWriter.__len__K  s    499~r8   c                      | j                   d   S r  r  rj   s    r6   min_idzW3PostingsWriter.min_idO  s    yy|r8   c                      | j                   d   S )Nr5  r  rj   s    r6   max_idzW3PostingsWriter.max_idS  s    yy}r8   c                     | j                   S r.   r  rj   s    r6   
min_lengthzW3PostingsWriter.min_lengthW      r8   c                     | j                   S r.   r  rj   s    r6   
max_lengthzW3PostingsWriter.max_length[  r  r8   c                     | j                   S r.   r  rj   s    r6   
max_weightzW3PostingsWriter.max_weight_  r  r8   )Frz   r{   r.   r|   )r}   r~   r   __doc__r7   r  r   r   r   r  r  r  r  r  r  r  r  r  r  r  ri   r8   r6   rD   rD     sb    
 IJ
$2")>*5r
" r8   rD   c                       e Zd ZdZ	 	 ddZd Zd Zd Zd Zd Z	d	 Z
d
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy)rO   zsReads on-disk postings from the postings file and presents the
    :class:`whoosh.matching.Matcher` interface.
    Nc                     || _         || _        || _        || _        || _        || _        || _        | j                  j                         | _        | j                          | j                          y r.   )r   r  _lengthr   _termr  rH   r  
_fixedsize_read_headerreset)r2   re   r  rV   rP   rI   rC   rH   s           r6   r7   zW3LeafMatcher.__init__i  s]    !'
++668

r8   c                     | j                   }|j                  | j                         |j                  d      }|t        k7  rt        d|z        |j                         | _        y )N   zBlock tag error %r)r   seekr  readr  r   r  _baseoffset)r2   re   magics      r6   r  zW3LeafMatcher._read_headery  sU    >>d''(a ((05899 $==?r8   c                     d | _         d | _        d | _        d | _        d | _        d | _        d| _        d| _        | j                  | j                         y NF)
_blocklength_maxidr  r0   r  r  
_lastblock_atend_gotor  rj   s    r6   r  zW3LeafMatcher.reset  sN      

4##$r8   c                    | j                   }d | _        d | _        d | _        d | _        d| _        |j                  |       |j                         }|dk  rd| _        |dz  }|t        z   |z   | _
        |j                         }|j                         | _        |\  | _        | _        | _        | _        }}t%        |      | _        t%        |      | _        y )Nr   Tr5  )r   _datar  r  r  _ir  read_intr  r   _nextoffsetread_pickler  _dataoffsetr  r  r  r0   r%   r  r  )r2   positionre   rV   infomnlenmxlens          r6   r  zW3LeafMatcher._goto  s     >> 
	 	h""$A:"DObLF $i/&8##%#==? 		DK$:K	(/(/r8   c                     | j                   rt        d      | j                  rd| _         y | j                  | j                         y )NzNo next blockT)r  r   r  r  r  rj   s    r6   _next_blockzW3LeafMatcher._next_block  s8    ;; O,,__DK JJt''(r8   c                     d}| j                         r4 |       r-| j                          |dz  }| j                         r |       r-|S )Nr   r{   )	is_activer  )r2   	skipwhileskippeds      r6   _skip_to_blockzW3LeafMatcher._skip_to_block  sC     nn9;qLG nn9; r8   c                 R    | j                    xr | j                  | j                  k  S r.   )r  r  r  rj   s    r6   r  zW3LeafMatcher.is_active  s"    ;;>477T->->#>>r8   c                 l    | j                   | j                          | j                   | j                     S r.   )r  	_read_idsr  rj   s    r6   idzW3LeafMatcher.id  s+     99NNyy!!r8   c                 l    | j                   | j                          | j                   | j                     S r.   )r  _read_weightsr  rj   s    r6   r   zW3LeafMatcher.weight  s-     ==  }}TWW%%r8   c                 l    | j                   | j                          | j                   | j                     S r.   )r  _read_valuesr  rj   s    r6   r   zW3LeafMatcher.value  s-     <<||DGG$$r8   c                     | xj                   dz  c_         | j                   | j                  k(  r| j                          yy)Nr{   TF)r  r  r  rj   s    r6   rI  zW3LeafMatcher.next  s5     	177d'''r8   c                 f   | j                         st        | j                         k  ry | j                          kD  r| j	                  fd       | j                         rJ| j                         k  r6| j                          | j                         r| j                         k  r4y y y y )Nc                               kD  S r.   ri   )block_max_idtargetids   r6   <lambda>z'W3LeafMatcher.skip_to.<locals>.<lambda>  s    <>(A r8   )r  r   r  r  r
  rI  )r2   r  r  s    `@r6   skip_tozW3LeafMatcher.skip_to  s     ~~ twwy  ((ln$ AB nn4779x#7IIK nn4779x#7#7r8   c                 ^    | j                           kD  ry| j                  fd      S )Nr   c                               k  S r.   ri   )block_quality
minqualitys   r6   r  z/W3LeafMatcher.skip_to_quality.<locals>.<lambda>  s    =?j+H r8   )r  r
  )r2   r  r  s    `@r6   skip_to_qualityzW3LeafMatcher.skip_to_quality
  s3     ** ?Z' ""#HIIr8   c                 X    | j                   | j                          | j                   d   S r  )r  r  rj   s    r6   block_min_idzW3LeafMatcher.block_min_id  s$    99NNyy|r8   c                     | j                   S r.   )r  rj   s    r6   r  zW3LeafMatcher.block_max_id      {{r8   c                     | j                   S r.   r  rj   s    r6   block_min_lengthzW3LeafMatcher.block_min_length!      r8   c                     | j                   S r.   r  rj   s    r6   block_max_lengthzW3LeafMatcher.block_max_length$  r%  r8   c                     | j                   S r.   r  rj   s    r6   block_max_weightzW3LeafMatcher.block_max_weight'  r%  r8   c                     | j                   | j                  z
  }| j                  j                  | j                  |      }| j                  rt        j                  |      }t        |      | _        y r.   )	r  r  r   r&  r0   r  
decompressr   r  )r2   rY  r   s      r6   
_read_datazW3LeafMatcher._read_data*  s\     ""T%5%55NNt//9 "A 1X
r8   c                     | j                   | j                          | j                   d   }| j                  st        t	        |            }|| _        y r  )r  r,  r  r  r#   r  r  s     r6   r  zW3LeafMatcher._read_ids7  sB    ::OOjjm }}S)*C	r8   c                 2   | j                   | j                          | j                   d   | j                  }"t        dd t	        |      D              | _        y t        t              r$t        dfdt	        |      D              | _        y | _        y )Nr{   r   c              3       K   | ]  }d   ywr  ri   )r{  r  s     r6   r|  z.W3LeafMatcher._read_weights.<locals>.<genexpr>L  s     'G'Gs   c              3   "   K   | ]  }  y wr.   ri   )r{  r  rR   s     r6   r|  z.W3LeafMatcher._read_weights.<locals>.<genexpr>N  s     'KA'Ks   )r  r,  r  r   r   r  rQ  r  )r2   	postcountrR   s     @r6   r  zW3LeafMatcher._read_weightsC  sy    ::OO**Q- %%	?!#'GVI5F'GHDM'!#'K	9J'KLDM#DMr8   c           	      F   | j                   | j                          | j                  | j                   d   dk  r| _        y du rd| j                  z  | _        y t        t              sJ t        fdt        dt                    D              | _        y )Nr  r   r.   c              3   .   K   | ]  }||z      y wr.   ri   )r{  r*   r  r  s     r6   r|  z-W3LeafMatcher._read_values.<locals>.<genexpr>`  s&      !I%& "$Aa)m!4 !Ir}  )
r  r,  r  r  r  rQ  r   r  r   r   )r2   r  r  s    @@r6   r  zW3LeafMatcher._read_valuesR  s    ::OO OO	ZZ]	ADL!^"T%6%66DLb*---  !I*0CGY*G!I IDLr8   )NNN)r}   r~   r   r  r7   r  r  r  r  r
  r  r  r   r   rI  r  r  r   r  r$  r'  r)  r,  r  r  r  ri   r8   r6   rO   rO   d  s     EI&* 
+%"0H
)?"&%
(J

$Ir8   rO   c                       e Zd Z ej                  d      Zd Zd Zd Zd Z	d Z
d Zd Zd	 Zed
        Zed        Zed        Zed        Zed        Zy)r   z	!BfIBBfIIc                 `    t        j                  | g|i | d | _        d | _        d | _        y r.   )r   r7   _offsetr  _inlined)r2   argskwargss      r6   r7   zW3TermInfo.__init__q  s/    $000r8   c                    | xj                   t        |j                        z  c_         | xj                  t	        |      z  c_        |j                         }| j                  || _        nt        | j                  |      | _        t        | j                  |j                               | _	        t        | j                  |j                               | _        | j                  |j                         | _        |j                         | _        y r.   )_weightsumr  _dfr   r  r  r-  r1  r  r  r  r  _minidr  r  r  )r2   blockmls      r6   r  zW3TermInfo.add_blockw  s    ENN++CJ??" DO!$//26DOdoou/?/?/ABdoou/?/?/AB;;,,.DKllnr8   c                      || _         || _        y r.   r6  r  )r2   rU   rV   s      r6   r  zW3TermInfo.set_extent  s    r8   c                 2    | j                   | j                  fS r.   rB  rj   s    r6   rN   zW3TermInfo.extent  s    ||T\\))r8   c                 N    t        |      t        |      t        |      f| _        y r.   )r  r7  )r2   rQ   rR   rS   s       r6   set_inlinedzW3TermInfo.set_inlined  s    sU7^U6]Cr8   c                     | j                   d uS r.   r7  rj   s    r6   rL   zW3TermInfo.is_inlined  s    }}D((r8   c                     | j                   S r.   rG  rj   s    r6   rM   zW3TermInfo.inlined_postings  s    }}r8   c           
         | j                         }| j                  dnt        | j                        }t        | j                        }| j                  dn| j                  }| j
                  dn| j
                  }| j                  j                  || j                  | j                  ||| j                  ||      }|rt        | j                  d      }n+t        | j                        t        | j                         z   }||z  }|S )Nr       r  )rL   r  r$   r  r>  r  _structpackr;  r=  r  r   r7  r    r6  r   r  )r2   	isinlinedr  	maxlengthminidmaxidst	postbytess           r6   r   zW3TermInfo.to_bytes  s    OO%	 //1Q(9 	"4??3	 #kk1
t{{"kk1
t{{ \\y$,,()T__$e- dmmQ/I!$,,/(4<<2HHI
i	r8   c                 *   | j                   }|j                  |d |j                         } |        }|d   }|d   |_        |d   |_        t        |d         |_        t        |d         |_        |d   |_        |d   dk(  rd n|d   |_	        |d	   dk(  rd n|d	   |_
        |rt        ||j                  d        |_        |S |j                  }|j                  t        z   }t        |||       d   |_        t!        |||t"        z          |_        |S )
Nr   r{   r  rz   r        rJ     )rK  unpacksizer;  r=  r%   r  r  r  r>  r  r   r7  r   r!   r6  r   r   r  )rq   srQ  valsrJ   flagsoffposlenposs           r6   rV  zW3TermInfo.from_bytes  s   [[yy8BGG%5Q7Aw,T!W5,T!W5"1g"&q'Z"7$T!W"&q'Z"7$T!W %ak 2H  WWFWWz)F*1VF+;<Q?H)!F6I3E*FGHr8   c                 *    |j                  |dz         S Nr{   )	get_floatrq   rE   rX  s      r6   r  zW3TermInfo.read_weight  s    !,,r8   c                 8    |j                  |dz   t        z         S r_  )get_uintr   ra  s      r6   r  zW3TermInfo.read_doc_freq  s    w{[899r8   c                     |dz   t         z   t        z   }t        |j                  |            }t        |j                  |dz               }||fS r_  )r   r   r%   get_byte)rq   rE   rX  r]  r@  xls         r6   read_min_and_max_lengthz"W3TermInfo.read_min_and_max_length  sH    1{*Y6FOOF34FOOFQJ782vr8   c                 P    |dz   t         z   t        z   dz   }|j                  |      S )Nr{   r  )r   r   r`  )rq   rE   rX  
weightsposs       r6   read_max_weightzW3TermInfo.read_max_weight  s+    q[;.:Q>

++r8   N)r}   r~   r   structStructrK  r7   r  r  rN   rE  rL   rM   r   r   rV  r  r  rg  rj  ri   r8   r6   r   r   f  s     fmmK(G% *D)2  4 - - : :   , ,r8   r   c                   @    e Zd Zd
dZd Zd Zd Zd Zd ZddZ	d	 Z
y)rv   Nc                 |    || _         || j                         n|| _        || _        || _        || _        d| _        y r  )rw   
_random_idsegidr   r   _deletedr   )r2   r   rw   doccountrp  deleteds         r6   r7   zW3Segment.__init__  s9    "*/-T__&U
!r8   c                     | j                   S r.   )r   )r2   r9  s     r6   r   zW3Segment.codec  r"  r8   c                     || _         y r.   r  )r2   dcs     r6   set_doc_countzW3Segment.set_doc_count  s	    r8   c                     | j                   S r.   r  rj   s    r6   r  zW3Segment.doc_count_all  r  r8   c                 F    | j                   yt        | j                         S r  )rq  r   rj   s    r6   r
  zW3Segment.deleted_count  s    == 4==!!r8   c                 F    | j                   yt        | j                         S )Nri   )rq  iterrj   s    r6   r  zW3Segment.deleted_docs  s    == &&r8   c                     |r7| j                   t               | _         | j                   j                  |       y | j                   +|| j                   v r| j                   j                  |       y y y r.   )rq  setr   r   )r2   r   deletes      r6   delete_documentzW3Segment.delete_document   sZ    }}$ #MMf%]]&6T]]+BMM' ,C&r8   c                 8    | j                   y|| j                   v S r  )rq  r   s     r6   r  zW3Segment.is_deleted  s    == &&r8   )r   NN)T)r}   r~   r   r7   r   rw  r  r
  r  r  r  ri   r8   r6   rv   rv     s*    "
'('r8   rv   )Mr  rk  r   collectionsr   whooshr   r   whoosh.compatr   r   r	   r
   r   r   r   r   whoosh.codecr   whoosh.filedbr   r   whoosh.matchingr   r   r   whoosh.readingr   r   whoosh.systemr   r   r   r   r   r   r   r   r   r    r!   whoosh.util.numlistsr"   r#   whoosh.util.numericr$   r%   r  ImportErrorr  NumericColumnr   r   r   PickleColumnCompressedBytesColumnr   Codecr,   r   r   PerDocWriterWithColumnsr:   FieldWriterr@   PerDocumentReaderrY   FieldCursorr?  TermsReaderra   PostingsWriterrD   rO   r   Segmentrv   ri   r8   r6   <module>r     s  8   # # C C 9 9  . @ @ 1 $ I I 4 F F ; > y  '&&sA6%%%c*)G))#. $$$%BW%B%B%DEG*djj G*X!!uT11 upED$$ ETTT++ Tn7%D$$ 7%tRD$$ Rn[t** [|}IK }IDw, w,x)' )']%  Ds   
F4 4F?>F?