=wgbdZddlZddlZddlmZddlmZddlmZddl m Z m Z ddl m Z ddl mZdd lmZmZGd d eZd Zd ZdZeeefZej0dZej0dZej0dZdej8zZGddeZGddeZ GddeZ!Gdde Z"GddeZ#Gdde Z$y)zThis module defines writer and reader classes for a fast, immutable on-disk key-value database format. The current format is based heavily on D. J. Bernstein's CDB format (http://cr.yp.to/cdb.html). Ncrc32) bisect_left)md5)b bytes_type)xrange) GrowableArray) _INT_SIZE emptybytesc eZdZy)FileFormatErrorN)__name__ __module__ __qualname__O/var/www/horilla/myenv/lib/python3.12/site-packages/whoosh/filedb/filetables.pyrr.srrcFd}|D]}||dzzdzt|z }|S)Ni)ord)keyhcs rcdb_hashr4s6 A 1 !q&\Z '#a& 01 HrcNtt|jddzS)Nr)intr hexdigestrs rmd5_hashr";s! s3x!!#R (: 55rct|dzS)Nrrr!s rcrc_hashr$?s : ""rz!iiz!Iqz!qicReZdZdZeddfdZdZdZdZdZ d Z d Z d Z y ) HashWriteraImplements a fast on-disk key-value store. This hash uses a two-level hashing scheme, where a key is hashed, the low eight bits of the hash value are used to index into one of 256 hash tables. This is basically the CDB algorithm, but unlike CDB this object writes all data serially (it doesn't seek backwards to overwrite information at the end). Also unlike CDB, this format uses 64-bit file pointers, so the file length is essentially unlimited. However, each key and value must be less than 2 GB in length. HSH3rcx||_||_t|j|_i|_|j |_|j||j|j|jd|jdtdDcgc]}gc}|_ g|_ ycc}w)aR :param dbfile: a :class:`~whoosh.filedb.structfile.StructFile` object to write to. :param magic: the format tag bytes to write at the start of the file. :param hashtype: an integer indicating which hashing algorithm to use. Possible values are 0 (MD5), 1 (CRC32), or 2 (CDB hash). rr%N) dbfilehashtype_hash_functionshashfnextrastell startoffsetwrite write_byte write_intr buckets directory)selfr*magicr+_s r__init__zHashWriter.__init__`s   %dmm4  !;;= U$--(%+3K0q0 1s B7c6|jjSN)r*r/r6s rr/zHashWriter.tell}s{{!!rct|tsJt|tsJ|j}|j}|j t j t|t||j ||j ||j|}|j|dzj||fy)zAdds a key/value pair to the file. Note that keys DO NOT need to be unique. You can store multiple values under the same key and retrieve them using :meth:`HashReader.all`. N) isinstancerr*r/r1_lengthspacklenr-r4append)r6rvaluer*posrs raddzHashWriter.adds #z***%,,,kkm X]]3s8SZ89 S U KK  QW$$aX.rcB|j}|D]\}}|||y)zConvenience method to add a sequence of ``(key, value)`` pairs. This is the same as calling :meth:`HashWriter.add` on each pair in the sequence. N)rF)r6itemsrFrrDs radd_allzHashWriter.add_alls) hh JC UO rc|j}d}|jD]}|j}dt|z}|jj ||f|g|z}|D]-\}}|dz |z} || |k7r| dz|z} || |k7r||f|| </|D]*\}}|j tj||,y)N)rr) r*r4r/rBr5rCr1_pointerrA) r6r*nullentriesrEnumslots hashtablehashvalpositionslots r _write_hasheszHashWriter._write_hashess|| ?G++-C3w<'H NN ! !3/ 2)I%, 6!1 0o- 1H0D o-$+H"5 $ 6&/ ?! X]]7H=> ?) ?rc|j}|jD]*\}}|jtj ||,yr;)r*r5r1 _dir_entryrA)r6r*rTrQs r_write_directoryzHashWriter._write_directorys<"&.. > Hh LL8< = >rcN|jj|jyr;)r* write_pickler.r<s r _write_extraszHashWriter._write_extrass   -rc"|j}|j|j|j}|j |j |j|z |j}|j |Sr;)r*rVrYr/r\r3close)r6r*exposendposs rr^zHashWriter.closesn    ./  rN) rrr__doc__rr9r/rFrIrVrYr\r^rrrr'r'Ts: &'vY:"/(?>>.rr'ceZdZdZdeddfdZedZdZdZ d Z d Z d Z dd Z d ZdZdZdZdZdZddZdZdZdZy) HashReaderzTReader for the fast on-disk key-value files created by :class:`HashWriter`. Nr(rcx||_||_d|_|3|jdtj |j |z }|j||jd}||k7rtd|z|j|_ t|j|_ |j|j|j |_||ztz }|j!|}||z }|j||j#|j|t$z g|_t(j*} t(j,} t/dD]2} |j&j1| |j| 4|j&dd|_y)a :param dbfile: a :class:`~whoosh.filedb.structfile.StructFile` object to read from. :param length: the length of the file data. This is necessary since the hashing information is written at the end of the file. :param magic: the format tag bytes to look for at the start of the file. If the file's format tag does not match these bytes, the object raises a :class:`FileFormatError` exception. :param startoffset: the starting point of the file data. FNrzUnknown file header %rr%)r*r0 is_closedseekosSEEK_ENDr/readr read_byter+r,r-read_int startofdatar get_int _read_extras_directory_sizetablesrXsizeunpackr rC endofdata) r6r*lengthr7r0 filemagicexptrexlenr_ entrysize unpackentryr8s rr9zHashReader.__init__sl & > KK2;; '[[][0F K KKN  !":Y"FG G((* %dmm4 !;;=f$y0u%  E   EO+, OO  ''  DA KK  {6;;y+AB C DQ*rcX|j|}|j|}|||S)zConvenience method to open a hash file given a :class:`whoosh.filedb.filestore.Storage` object and a name. This takes care of opening the file and passing its length to the initializer. ) file_length open_file)clsstoragenamerur*s ropenzHashReader.opens1$$T*""4(66""rc|jSr;)r*r<s rfilezHashReader.files {{rcp |jj|_y#t$r i|_YywxYwr;)r* read_pickler.EOFErrorr<s rrozHashReader._read_extras"s1 ++113DK DK s "55cz|jrtd|z|jjd|_y)NzTried to close %r twiceT)rf Exceptionr*r^r<s rr^zHashReader.close(s1 >>5<= = rc|j}|j|}|j|tjz|Sr;)r*get_uintgetr@rr)r6rEr*keylens rkey_atzHashReader.key_at.s5%zz# -v66rc|j}tj}||jk\rytj |j ||\}}|j ||z|}||z|z}|||fSr;)r*r@rrrtrsr)r6rEr*lenssizerdatalenkeybytesdataposs rkey_and_range_atzHashReader.key_and_range_at5sq== $.. "//&**S(*CD::cHnf5.6)'))rc# K|j}|xs |j}|xs |j}tj}tj }||kr9||j ||\}}||z}||z} ||| |f| |z}||kr8yywr;)r*rmrtr@rrrsr) r6rEeodr*r unpacklensrrkeyposrs r_rangeszHashReader._rangesCs%T%%#T^^==__ Ci(C)BCOFG8^FvoG67G4 4G#C Cis B B BcJ|j|D]}|cSt|r;)allKeyErrorr6rrDs r __getitem__zHashReader.__getitem__Ss(XXc] EL smrc#K|j}|jD]1\}}}}|j||}|j||}||f3ywr;r*rr)r6r*rrrrrrDs r__iter__zHashReader.__iter__XsW04   ,FFGW**VV,CJJw0E,  AAc2|j|D]}yyNTF)ranges_for_key)r6rr8s r __contains__zHashReader.__contains___s"$$S) A rc#K|j}|jD]\}}}}|j||ywr;r)r6r*rrr8s rkeyszHashReader.keysds>$(LLN - FFAq**VV, , -<>c#K|j}|jD]\}}}}|j||ywr;r)r6r*r8rrs rvalueszHashReader.valuesis>&*lln / "Aq'7**Wg. . /rc#K|j}|jD]-\}}}}|j|||j||f/ywr;r)r6r*rrrrs rrHzHashReader.itemsnsP04  M ,FFGW::ff-vzz'7/KL L MsAAc8|j|D]}|cS|Sr;)r)r6rdefaultrDs rrzHashReader.getss#XXc] EL rc#~K|j}|j|D]\}}|j||yw)zCYields a sequence of values associated with the given key. N)r*rr)r6rr*rrs rrzHashReader.allxsA $ 3 3C 8 / GW**Wg. . /s;=c#tKt|tstd|z|j}|j |}|j |dz\}}|syt j}t j}tj}tj} ||dz |z|zz} t|D]} ||j| |\} } | sy| |k(rL| |j| |\}}|t|k(r#| |z}||j||k(r ||z|f| |z } | |||zzk(s|} yw)zeYields a sequence of ``(datapos, datalength)`` tuples associated with the given key. Key %r should be bytesr>NrL) r?r TypeErrorr*r-rqrNrrrsr@r rrB)r6rr*keyhash tablestartrQptrsize unpackptrrrslotposr8slothashitemposrrkeystarts rrzHashReader.ranges_for_keysJ #z*4s:; ;++c" ${{7S=9 H --OO ==__ 'Q,(!:g EF! %A )&**Wg*F G Hg7"",VZZ-J"KSX%&1Hfjj6::'&0':: w G*7(:;;$+ %s D0D83D8cJ|j|D]}|cSt|r;)rr)r6ritems r range_for_keyzHashReader.range_for_keys+'', DK smrNNr;)rrrrarr9 classmethodrrror^rrrrrrrrrHrrrrrrrrcrcs'+!F)3+j##  7 *$   - / M  //%brrcc"eZdZdZdZdZdZy)OrderedHashWriterzImplements an on-disk hash, but requires that keys be added in order. An :class:`OrderedHashReader` can then look up "nearest keys" based on the ordering. cftj||td|_t|_yNH)r'r9r indexr lastkeyr6r*s rr9zOrderedHashWriter.__init__s%D&)"3' ! rc||jkrtd|jd||jj|jj t j|||||_yNzKeys must increase: z..)r ValueErrorrrCr*r/r'rFrs rrFzOrderedHashWriter.adds_ $,,  $ c34 4 $++**,-tS%( rc|j}|j}|j|jd<t ||jd<t j ||j|y)N indextypeindexlen)r*rtypecoder.rBr'r\to_file)r6r*rs rr\zOrderedHashWriter._write_extrassT $)>> K "%e* J  & frN)rrrrar9rFr\rrrrrs " rrc0eZdZdZdZdZdZdZdZy)OrderedHashReadercL|j|}|y|j|S)zReturns the closest key equal to or greater than the given key. If there is no key in the file equal to or greater than the given key, returns None. N)closest_key_posr)r6rrEs r closest_keyzOrderedHashReader.closest_keys+ ""3' ;{{3rc#jK|j|}|y|j|D]}|yw)zYields a series of ``(keypos, keylen, datapos, datalen)`` tuples for the ordered series of keys equal or greater than the given key. N)rE)rr)r6rrErs r ranges_fromzOrderedHashReader.ranges_froms? ""3' ; LLSL) DJ s13c#K|j}|j|D]\}}}}|j||yw)zYYields an ordered series of keys equal to or greater than the given key. Nr*rr)r6rr*rrr8s r keys_fromzOrderedHashReader.keys_fromsE $($4$4S$9 - FFAq**VV, , -s=?c#K|j}|j|D]-\}}}}|j|||j||f/yw)zuYields an ordered series of ``(key, value)`` tuples for keys equal to or greater than the given key. Nr)r6rr*rrrrs r items_fromzOrderedHashReader.items_fromsW 040@0@0E M ,FFGW::ff-vzz'7/KL L MsAAc|j}tj||jd}|j |_|jd|_tj||_ |dk(r|j|_ y|dk(r|j|_ y|dk(r|j|_ y|dk(r|j|_ y|dk(r|j|_ yt!d|z) NrrBriIqUnknown index type %r)r*rcror.r/ indexbaserstructcalcsize indexsizeget_byte_get_pos get_ushortrnrget_longr)r6r*rs rrozOrderedHashReader._read_extrass %KK ,  J/ 3  "OODM # "--DM # "NNDM # "OODM # "OODM3i?@ @rc^t|tstd|z|j}|j}|j }|j }d}|j}||kr/||zdz}|||||zz} | |kr|dz}n|}||kr/||jk(ry||||zzS)NrrrKrM)r?rrrrrrr) r6rrrrrlohimidmidkeys rrz!OrderedHashReader.closest_key_poss#z*4s:; ;NN NN == ]]2g7q.CHYy%@ABF|1W 2g   BN233rN) rrrrrrrrorrrrrrs"   -MA24rrc(eZdZdZdZdZdZdZy)FieldedOrderedHashWriterzYImplements an on-disk hash, but writes separate position indexes for each field. crtj||ix|_|jd<t|_yNfieldmap)r'r9rr.r rrs rr9z!FieldedOrderedHashWriter.__init__Bs0D&)244  J/" rc|jj|_||_t d|_t |_yr)r*r/ fieldstart fieldnamer posesr rr6rs r start_fieldz$FieldedOrderedHashWriter.start_fieldJs0++**,""3' ! rc||jkrtd|jd||jj|jj |j z tj|||||_yr) rrrrCr*r/rr'rFrs rrFzFieldedOrderedHashWriter.addQsh $,,  $ c34 4 $++**,t>?tS%( rc|j}|j}|j}|j|j t ||j f|j|<|j|yr;) r*rrrr/rBrrr)r6r*rrs r end_fieldz"FieldedOrderedHashWriter.end_fieldYsUNN  $(OOV[[]CJ$)NN$4 i  frN)rrrrar9rrFrrrrrr=s""rrcdeZdZdZdZddZdZdZdZdZ d Z dd Z d Z d Z d ZdZdZy)FieldedOrderedHashReaderc tj|g|i||jd|_g|_t |jj D]4}|j|\}}}}|jj|||f6yr)rcr9r.r fieldlistsortedrrC)r6argskwargsrstartposixposixsizeixtypes rr9z!FieldedOrderedHashReader.__init__csD24262 J/  2 2 45 @I.2mmI.F +HeVV NN ! !9h"> ? @rc&|j|dS)Nr)rrs r field_startz$FieldedOrderedHashReader.field_startls}}Y'**rNc#K|j}d}||\}}}|j||D]#\}} } } ||k\r|dz }||\}}}||| | | f%yw)NrrM)rr) r6rErflistfposrstartendrrrrs rfielded_rangesz'FieldedOrderedHashReader.fielded_rangesosx %d  5#04 S#0F > ,FFGW} (-d % 5#VVWg= =  >sAAc#K|jj}|jD]\}}}}}||||fywr;r*rr)r6rrrrr8s r iter_termsz#FieldedOrderedHashReader.iter_termsysHkkoo/3/B/B/D 1 +Ivvq!S00 0 1sAAc#K|jj}|jD]}|\}}}}}|||||||f!ywr;r)r6rrrrrrrs riter_term_itemsz(FieldedOrderedHashReader.iter_term_items~sXkkoo'') HD:> 7IvvwS0#gw2GG G HsA A cH |j||}y#t$rYywxYwr)range_for_termr)r6rbtextxs r contains_termz&FieldedOrderedHashReader.contains_terms. ##Iu5A  s  !!c|j|\}}}}|j|D]\}}||cxkr|ksn||fcSt||fr;)rrr) r6rrr rrcoderrs rrz'FieldedOrderedHashReader.range_for_termsb%)]]9%="ufd $ 3 3E : ( GWw&&'' ( 5)**rcd|j||\}}|jj||Sr;)rr*r)r6rrrrs r term_dataz"FieldedOrderedHashReader.term_datas...y%@{{w00rcJ |j||S#t$r|cYSwxYwr;)rr)r6rrrs rterm_getz!FieldedOrderedHashReader.term_gets, >>)U3 3 N s  ""c t|tstd|z|j}|j}|j |\}}}}|dk(r |j } nV|dk(r |j} nD|dk(r |j} n2|dk(r |j} n |dk(r |j} ntd|zd} |} | | kr2| | zd z} ||| || |zzz} | |kr| d z} n| } | | kr2| |k(ry|| || |zzzS) NrrrrrrrrrKrM) r?rrr*rrrrrnrrr)r6rrr*rrrrrget_posrrrrs rclosest_term_posz)FieldedOrderedHashReader.closest_term_poss+#z*4s:; ;*.-- *B'% S=ooG s]''G s]nnG s]ooG s]ooG3f<= = 2g7q.CHwusV|/C'DDEF|1W 2g <'%"v+"5666rcN|j||}|y|j|Sr;)r!r)r6rrrEs r closest_termz%FieldedOrderedHashReader.closest_terms+##Iu5 ;{{3rc#K|j||}|y|j|\}}}}|j||D]}|ywr;)r!rr) r6rrrErrrrrs rterm_ranges_fromz)FieldedOrderedHashReader.term_ranges_fromsV##Iu5 ; *.-- *B'%LLe, DJ sAAc#K|j}|j||D]\}}}}|j||ywr;r*r%r)r6rrr*rrr8s r terms_fromz#FieldedOrderedHashReader.terms_fromsE$($9$9)U$K - FFAq**VV, , -s>Ac#K|j}|j||D]/}|\}}}}|j|||j||f1ywr;r') r6rrr*rrrrrs rterm_items_fromz(FieldedOrderedHashReader.term_items_froms\)))U; MD/3 ,FFGW::ff-vzz'7/KL L Mrrr;)rrrr9rrrrrrrrr!r#r%r(r*rrrrrbsL@+>1 H +1 &7P - Mrr)%rarhrbinasciirbisectrhashlibr whoosh.compatrrr whoosh.util.numlistsr whoosh.systemr r rrrr"r$r,Structr@rNrXrrrpobjectr'rcrrrrrrrr3s8 ' ./  i  6#Xx0 6==  6==  V]]5 !  ' CCLZZ~ D]4 ]4D"z"JzMzzMr