=wg1ddlmZmZddlmZmZddlmZedZGddeZ Gdde Z Gd d e Z Gd d e Z d Z dZGdde Zy))u text_type) ComposableToken)rcompilez \w+(\.?\w+)*ceZdZdZdZy) TokenizerzBase class for Tokenizers. c:|xr|j|juSN) __class__selfothers Q/var/www/horilla/myenv/lib/python3.12/site-packages/whoosh/analysis/tokenizers.py__eq__zTokenizer.__eq__+s:5??::N)__name__ __module__ __qualname____doc__rrrr r 's ;rr ceZdZdZ ddZy) IDTokenizerzYields the entire input string as a single token. For use in indexed but untokenized fields, such as a document's path. >>> idt = IDTokenizer() >>> [token.text for token in idt("/a/b 123 alpha")] ["/a/b 123 alpha"] c +Kt|ts Jd|zt||f||d| } || _d| _|r|| _|r |dz| _|r|| _|t|z| _ | yw)N%r is not unicode removestopsmode?) isinstancerrtextboostoriginalpos startcharlenendchar) rvalue positionschars keeporiginalr start_pos start_charrkwargsts r__call__zIDTokenizer.__call__8s%+H-@5-HH+ )U  $   AJ MAE $AK"SZ/AIsA/A1N)FFFTrr)rrrrr1rrrrr/s6;1513rrc0eZdZdZedfdZdZ ddZy)RegexTokenizerz Uses a regular expression to extract tokens from text. >>> rex = RegexTokenizer() >>> [token.text for token in rex(u("hi there 3.141 big-time under_score"))] ["hi", "there", "3.141", "big", "time", "under_score"] Fc2t||_||_y)a :param expression: A regular expression object or string. Each match of the expression equals a token. Group 0 (the entire matched text) is used as the text of the token. If you require more complicated handling of the expression match, simply write your own tokenizer. :param gaps: If True, the tokenizer *splits* on the expression, rather than matching on the expression. N)r expressiongaps)rr6r7s r__init__zRegexTokenizer.__init__Ss#:. rc|j|jur.|jj|jjk(ryy)NTF)r r6patternr s rrzRegexTokenizer.__eq__`s6 >>U__ ,&&%*:*:*B*BBrc +RKt|tsJdt|zt||f|| d| } |s?|x| _| _d| _|r|| _|r|| _|t|z| _ | y|jst|jj|D]~\} } | jd| _d| _|r| j | _d| _|r || z| _|r0|| j#z| _|| j%z| _ | yd}|} |jj|D]{} |}| j#}|||}|rP|| _d| _|r| j | _d| _|r | | _| dz } |r||z| _||z| _ | | j%}}|t|krR||d| _d| _|r| j | _d| _|r| | _|r|| _t|| _ | yyw) :param value: The unicode string to tokenize. :param positions: Whether to record token positions in the token. :param chars: Whether to record character offsets in the token. :param start_pos: The position number of the first token. For example, if you set start_pos=2, the tokens will be numbered 2,3,4,... instead of 0,1,2,... :param start_char: The offset of the first character of the first token. For example, if you set start_char=2, the text "aaa bbb" will have chars (2,5),(6,9) instead (0,3),(4,7). :param tokenize: if True, the text should be tokenized. z%s is not unicoderrrFr N)r!rreprrr$r"r#r%r&r'r(r7 enumerater6finditergroupstoppedstartend)rr)r*r+r,rr-r.tokenizerr/r0r%matchprevendrBrCr"s rr1zRegexTokenizer.__call__fs$ %+N-@4;-NN+ )U  $  "' 'AJAG!( &U3 G'(@(@(GH  UQ!"AJ! %OAE",u{{}">> from whoosh.support.charset import charset_table_to_dict >>> from whoosh.support.charset import default_charset >>> charmap = charset_table_to_dict(default_charset) >>> chtokenizer = CharsetTokenizer(charmap) >>> [t.text for t in chtokenizer(u'Stra\xdfe ABC')] [u'strase', u'abc'] The Sphinx charset table format is described at http://www.sphinxsearch.com/docs/current.html#conf-charset-table. charmapc||_y)z :param charmap: a mapping from integer character numbers to unicode characters, as used by the unicode.translate() method. NrK)rrLs rr8zCharsetTokenizer.__init__s  rcp|xr3|j|juxr|j|jk(Sr )r rLr s rrzCharsetTokenizer.__eq__s42NNeoo52LLEMM1 3rc +Kt|ts Jd|zt||f|| d| } |s?|x| _| _d| _|r|| _|r|| _|t|z| _ | ytd} |j} |}|x}}|D]u}| t|}|r| |z } nX||kDrC| | _d| _|r| j| _|r || _|dz }|r|| _|| _ | |dz}td} |dz }w||kDrB|||| _d| _|r| j| _|r|| _|r|| _|| _ | yyw)r<rrrr2r N) r!rrr$r"r#r%r&r'r(rrLord)rr)r*r+r,rr-r.rDrr/r0r"rLr%r& currentcharchartchars rr1zCharsetTokenizer.__call__s %+H-@5-HH+ )U  $  "' 'AJAG!( &U3 GR5DllGC&0 0I  !D *EMD"Y.!%"%')*AJ$$'AE1HC *3AK(3AI +aIR5Dq ) !,Y&y5!"AJAE"+AK +AI'sE E NrG) rrrrdictstr __inittype__r8rr1rrrrJrJs0,$L3 JOGKBrrJctdS)zReturns a RegexTokenizer that splits tokens by whitespace. >>> sst = SpaceSeparatedTokenizer() >>> [token.text for token in sst("hi there big-time, what's up")] ["hi", "there", "big-time,", "what's", "up"] z [^ \t\r\n]+)r4rrrSpaceSeparatedTokenizerrX%s . ))rc4ddlm}td|zS)aSplits tokens by commas. Note that the tokenizer calls unicode.strip() on each match of the regular expression. >>> cst = CommaSeparatedTokenizer() >>> [token.text for token in cst("hi there, what's , up")] ["hi there", "what's", "up"] r StripFilterz[^,]+)whoosh.analysis.filtersr[r4rZs rCommaSeparatedTokenizerr]0s4 ( #km 33rc eZdZdZddZddZy) PathTokenizerzhA simple tokenizer that given a string ``"/a/b/c"`` yields tokens ``["/a", "/a/b", "/a/b/c"]``. c$t||_yr )rexpr)rr6s rr8zPathTokenizer.__init__EsZ( rc+Kt|ts Jd|zt|fi|}|}|jj |D],}|d|j |_|r ||_|dz }|.yw)Nrr )r!rrrar?rCr"r%)rr)r*r-r/tokenr%rEs rr1zPathTokenizer.__call__Hs5), I.AE.I I,y+F+ii((/ U -UZ [  sA4A6N)z[^/]+)Fr)rrrrr8r1rrrr_r_@s) rr_N) whoosh.compatrrwhoosh.analysis.acorerrwhoosh.util.textrrHr rr4rJrXr]r_rrrrgsh8'3%?+ ; ;)6nYnbgygT* 4 Ir