=wgdddlmZGddeZdZ d dZGddeZGdd eZy ) ) iteritemsc eZdZy)CompositionErrorN)__name__ __module__ __qualname__L/var/www/horilla/myenv/lib/python3.12/site-packages/whoosh/analysis/acore.pyrr!sr rcd|DS)zCRemoves tokens from a token stream where token.stopped = True. c3:K|]}|jr|ywN)stopped).0ts r zunstopped..*s 4!!))A 4sr ) tokenstreams r unstoppedr's 5{ 44r c+K|}|}td||d|}|D]9} | |_|r ||_|dz }|r||_|t | z}||_|;yw)aMTakes a sequence of unicode strings and yields a series of Token objects (actually the same Token object over and over, for performance reasons), with the attributes filled in with reasonable values (for example, if ``positions`` or ``chars`` is True, the function assumes each token was separated by one space). ) positionscharsNr )Tokentextpos startcharlenendchar) textstreamrr start_pos start_charkwargsrcharrrs r entokenr$-ss C D 9 9&9A  AE 1HC AK#d)#DAI sAAc$eZdZdZddZdZdZy)ra Represents a "token" (usually a word) extracted from the source text being indexed. See "Advanced analysis" in the user guide for more information. Because object instantiation in Python is slow, tokenizers should create ONE SINGLE Token object and YIELD IT OVER AND OVER, changing the attributes each time. This trick means that consumers of tokens (i.e. filters) must never try to hold onto the token object between loop iterations, or convert the token generator into a list. Instead, save the attributes between iterations, not the object:: def RemoveDuplicatesFilter(self, stream): # Removes duplicate words. lasttext = None for token in stream: # Only yield the token if its text doesn't # match the previous token. if lasttext != token.text: yield token lasttext = token.text ...or, call token.copy() to get a copy of the token object. c ||_||_d|_d|_||_||_|j j|y)a :param positions: Whether tokens should have the token position in the 'pos' attribute. :param chars: Whether tokens should have character offsets in the 'startchar' and 'endchar' attributes. :param removestops: whether to remove stop words from the stream (if the tokens pass through a stop filter). :param mode: contains a string describing the purpose for which the analyzer is being called, i.e. 'index' or 'query'. Fg?N)rrrboost removestopsmode__dict__update)selfrrr(r)r"s r __init__zToken.__init__hsA#   &  V$r cdjdt|jD}|jjd|dS)N, c30K|]\}}|d|yw=Nr )rnamevalues r rz!Token.__repr__..~s#G)dE&*51G())joinrr* __class__r)r,parmss r __repr__zToken.__repr__}s> G-6t}}-EGG>>22E::r c,tdi|jS)Nr )rr*r,s r copyz Token.copys%t}}%%r N)FFT)rrr__doc__r-r;r>r r r rrKs8%*; &r rc"eZdZdZdZdZdZy) ComposableFc`ddlm}t|tst |d||||S)Nr)CompositeAnalyzerz is not composable with )whoosh.analysis.analyzersrD isinstancerB TypeError)r,otherrDs r __or__zComposable.__or__s,?%,dEJK K u--r cd}|jr+djdt|jD}|jjd|zzS)Nr?r/c30K|]\}}|d|ywr1r )rkeyr4s r rz&Composable.__repr__..s!;",#u*-e4;r5z(%s))r*r8rr9r)r,attrss r r;zComposable.__repr__sM ==II;!*4==!9;;E~~&&%77r c|jSr)is_morphr=s r has_morphzComposable.has_morphs }}r N)rrrrOrIr;rPr r r rBrBsH.8r rBN)FFrr) whoosh.compatr Exceptionrrr$objectrrBr r r rTsC8$  y 5 AB<9&F9&|r