=wg/JddlZddlmZddlmZmZddlmZddlmZGddeZ Gdd eZ Gd d eZ Gd d eZ y)N)deque)u text_type)xrange)Filterc$eZdZdZddZdZdZy)CompoundWordFilteraGiven a set of words (or any object with a ``__contains__`` method), break any tokens in the stream that are composites of words in the word set into their individual parts. Given the correct set of words, this filter can break apart run-together words and trademarks (e.g. "turbosquid", "applescript"). It can also be useful for agglutinative languages such as German. The ``keep_compound`` argument lets you decide whether to keep the compound word in the token stream along with the word segments. >>> cwf = CompoundWordFilter(wordset, keep_compound=True) >>> analyzer = RegexTokenizer(r"\S+") | cwf >>> [t.text for t in analyzer("I do not like greeneggs and ham") ["I", "do", "not", "like", "greeneggs", "green", "eggs", "and", "ham"] >>> cwf.keep_compound = False >>> [t.text for t in analyzer("I do not like greeneggs and ham") ["I", "do", "not", "like", "green", "eggs", "and", "ham"] c ||_||_y)a) :param wordset: an object with a ``__contains__`` method, such as a set, containing strings to look for inside the tokens. :param keep_compound: if True (the default), the original compound token will be retained in the stream before the subwords. N)wordset keep_compound)selfr r s P/var/www/horilla/myenv/lib/python3.12/site-packages/whoosh/analysis/intraword.py__init__zCompoundWordFilter.__init__9s *c||jvr|gS||vr||Stdt|D]>}|d|}||jvs||d}|j||}|s1|g|z}|||<|cSyN)r rlensubwords)r smemoiprefixsuffix suffix_subsresults rrzCompoundWordFilter.subwordsDs  3J 97N3q6" "ArUF%12"mmFD9 $X 3F$DG!M "rc#K|j}i}|j}|D]B}||j|}|r't|dkDr|r||D] }||_|?|Dywr)r rtextr)r tokensr rrtsubssubwords r__call__zCompoundWordFilter.__call__Vsu** == AAFFD)Dt9q=]G#G$AFG sA#A%N)T)__name__ __module__ __qualname____doc__rrr#rrr r $s( +$ rr ceZdZdZddZdZy) BiWordFilteraMerges adjacent tokens into "bi-word" tokens, so that for example:: "the", "sign", "of", "four" becomes:: "the-sign", "sign-of", "of-four" This can be used to create fields for pseudo-phrase searching, where if all the terms match the document probably contains the phrase, but the searching is faster than actually doing a phrase search on individual word terms. The ``BiWordFilter`` is much faster than using the otherwise equivalent ``ShingleFilter(2)``. c||_yN)sep)r r-s rrzBiWordFilter.__init__xs rc#LK|j}d}d}d}d}|D]}|j}|j} | r |j} |j} | r |j } |1| r||_| r||_dj |||f|_|d}|}| r }| s }|syyw)NFT)r-r positionsposchars startcharjoin) r rr- prev_textprev_startcharprev_pos atleastonetokenrr0psr2scs rr#zBiWordFilter.__call__{shh   E::DIYYKKE__$ (EI&4EO WWid%;<  ! I!#A HKs BB$ B$N)-r$r%r&r'rr#r(rrr*r*fs",rr*ceZdZdZddZdZy) ShingleFilteraMerges a certain number of adjacent tokens into multi-word tokens, so that for example:: "better", "a", "witty", "fool", "than", "a", "foolish", "wit" with ``ShingleFilter(3, ' ')`` becomes:: 'better a witty', 'a witty fool', 'witty fool than', 'fool than a', 'than a foolish', 'a foolish wit' This can be used to create fields for pseudo-phrase searching, where if all the terms match the document probably contains the phrase, but the searching is faster than actually doing a phrase search on individual word terms. If you're using two-word shingles, you should use the functionally equivalent ``BiWordFilter`` instead because it's faster than ``ShingleFilter``. c ||_||_yr,)sizer-)r rAr-s rrzShingleFilter.__init__s rc#<K|j}|jtd}fd}|D]X}|jrj |j t |k(s>d}|jZ|s r |yyyw)NFcd}jDcgc]}|jc}|_|jrdj|_|Scc}w)Nr)r4rr2endchar)tkr bufr-s r make_tokenz*ShingleFilter.__call__..make_tokensKQBhh4145BGxx W__ I 5sAT)rAr-rstoppedappendcopyrpopleft)r rrAr8rHr9rGr-s @@rr#zShingleFilter.__call__syyhhg   "E== 5::<(s8t#!%J$,&KKM  "c, "zs `Wi`, `Fi`. * When splitwords=True, split on case transitions, e.g. `PowerShot` -> `Power`, `Shot`. * When splitnums=True, split on letter-number transitions, e.g. `SD500` -> `SD`, `500`. * Leading and trailing delimiter characters are ignored. * Trailing possesive "'s" removed from subwords, e.g. `O'Neil's` -> `O`, `Neil`. The mergewords and mergenums arguments turn on merging of subwords. When the merge arguments are false, subwords are not merged. * `PowerShot` -> `0`:`Power`, `1`:`Shot` (where `0` and `1` are token positions). When one or both of the merge arguments are true, consecutive runs of alphabetic and/or numeric subwords are merged into an additional token with the same position as the last sub-word. * `PowerShot` -> `0`:`Power`, `1`:`Shot`, `1`:`PowerShot` * `A's+B's&C's` -> `0`:`A`, `1`:`B`, `2`:`C`, `2`:`ABC` * `Super-Duper-XL500-42-AutoCoder!` -> `0`:`Super`, `1`:`Duper`, `2`:`XL`, `2`:`SuperDuperXL`, `3`:`500`, `4`:`42`, `4`:`50042`, `5`:`Auto`, `6`:`Coder`, `6`:`AutoCoder` When using this filter you should use a tokenizer that only splits on whitespace, so the tokenizer does not remove intra-word delimiters before this filter can see them, and put this filter before any use of LowercaseFilter. >>> rt = RegexTokenizer(r"\S+") >>> iwf = IntraWordFilter() >>> lcf = LowercaseFilter() >>> analyzer = rt | iwf | lcf One use for this filter is to help match different written representations of a concept. For example, if the source text contained `wi-fi`, you probably want `wifi`, `WiFi`, `wi-fi`, etc. to match. One way of doing this is to specify mergewords=True and/or mergenums=True in the analyzer used for indexing, and mergewords=False / mergenums=False in the analyzer used for querying. >>> iwf_i = IntraWordFilter(mergewords=True, mergenums=True) >>> iwf_q = IntraWordFilter(mergewords=False, mergenums=False) >>> iwf = MultiFilter(index=iwf_i, query=iwf_q) >>> analyzer = RegexTokenizer(r"\S+") | iwf | LowercaseFilter() (See :class:`MultiFilter`.) T)delims splitwords splitnums mergewords mergenumsz -_'"()!@#$%^&*[]{}<>\|;:,./?`~=+FcHddlm}m}m}t j ||_t jtd|j fztj|_ td|||j fz} t j| tj|_ td||fz} td|||fz} td|||fz} |r=|r;td| | | fz} t j| tj|_ np|r3t jt| tj|_ n;|r9td | | fz}t j|tj|_ |xs||_||_||_y ) aP :param delims: a string of delimiter characters. :param splitwords: if True, split at case transitions, e.g. `PowerShot` -> `Power`, `Shot` :param splitnums: if True, split at letter-number transitions, e.g. `SD500` -> `SD`, `500` :param mergewords: merge consecutive runs of alphabetic subwords into an additional token with the same position as the last subword. :param mergenums: merge consecutive runs of numeric subwords into an additional token with the same position as the last subword. r)digits lowercase uppercasez[^%s]+z(?<=[%s%s])'[Ss](?=$|[%s])z[%s][%s]z [%s%s][%s]z [%s][%s%s]z (%s|%s|%s)z(%s|%s)N)whoosh.support.unicoderVrWrXreescaperPcompilerUNICODEbetween possessiveboundaryr splittingrSrT)r rPrQrRrSrTrVrWrXdispat lower2upper letter2digit digit2lettersplitpatnumpats rrzIntraWordFilter.__init__sR HGii' zz!H+"> K /0Iy48KK4AA**VRZZ8 my)&<< )Y)GG &)Y)GG )+|*6*88HJJx>+%D"'..X"N% &  2#Um+$ %h#X..&x00+ 1 1s D/E2Ec |j}|j}d}d g} fd}tD]}|d}|d} |jrd} n|j rd} nd} |r| |cxk(rdk(rnn|s| |cxk(rdk(rnn|r|j |n%t |dkDr|| | dz  dz |g}| } dz t |dkDr||t  yy)Nrcdjd|D}|dd}|dd}j||||fy)Nr/c3&K|] }|d yw)rNr().0items r z>IntraWordFilter._merge..insert_item..s6$d1g6srrMrD)r4insert)rGatnewposnewtextnewscnewecinsertatpartss r insert_itemz+IntraWordFilter._merge..insert_itemsHgg6#66GF1IEGAJE LLGVUE#B CrrrM)rSrTlistisalphaisdigitrJr) r rrSrTlastrGrrrr1thisrs ` @r_mergezIntraWordFilter._merges__ NN  D K D7Dq'C||~**zD%A%) 4 s8a< XsQw7MHf MH; B s8a< SZ - rc #K|j}|j}d}|D]D}|j}||jr |j}nd}|j r |j s |js|jr||_||dz }~|j|}t|D cgc]\}\} } || | ||z| | f} } }} t| dkDr|s|r|j| |j} | r |j} | D]9\}}}}||_||_|jr |z|_| |z|_|;| s:| dddz}Gycc} } }ww)NrrrD)rSrTrr0r1rislowerisupperrr} enumeraterrr2r3rE)r rrSrTrr rrangesrr;rwrr2baser1r3rEs rr#zIntraWordFilter.__call__s}__ NN 0 .A66D~;;UUFFT\\^t||~<<>!  T*,5f,=??(QRr"+q6z2r:??u:>"Y E*;;D5:1D#y'!AFAEww&*Y&6 $(7N G "2Yq\A-Fa0 .0?sB;E/=E( BE/E/N)r$r%r&r'is_morphdictrbool __inittypes__rrrlr}rr#r(rrrOrOsU6pH dd$(D:M CD D!U*#X,11f5.n7.rrO) rZ collectionsr whoosh.compatrrrwhoosh.analysis.filtersrr r*r?rOr(rrrsM8 & *??DA6AH2F2jO.fO.r