=wg-dZddlmZddlZddlmZddlmZddlm Z m Z Gdde Z Gd d e Z Gd d e ZGd de ZGdde ZddZddZdZddZddZdZdZdZy)zQClasses and functions for classifying and extracting information from documents. )divisionN) defaultdict)log)xrange iteritemsceZdZdZdZdZy)ExpansionModelc||_||_|jr|j|jz |_yd|_y)Nr)Ncollection_total mean_length)self doc_count field_lengths F/var/www/horilla/myenv/lib/python3.12/site-packages/whoosh/classify.py__init__zExpansionModel.__init__+s7 , 66#44tvv=D  D ctNNotImplementedErrorr maxweight top_totals r normalizerzExpansionModel.normalizer4!!rctrr)r weight_in_topweight_in_collectionrs rscorezExpansionModel.score7rrN)__name__ __module__ __qualname__rrr rrr r *s!""rr ceZdZdZdZy)Bo1Modelc~||jz }|td|z|z ztd|zztdz S)N?@r rrrrfs rrzBo1Model.normalizer<s>  CqA ..S1W=SIIrcj||jz }|td|z|z dztd|zdzSNr(r*rrrrr,s rr zBo1Model.score@s9 466 )sC!Gq=!44s37AFFrNr!r"r#rr r$rrr&r&;sJGrr&ceZdZdZdZy)Bo2Modelc||jz|jz }|td|z|z dztd|zdzSr.)r r rr+s rrzBo2Model.normalizerFsD  !6!6 63a1}a003sQw?BBrcp||z|jz }|td|z|z dztd|zdzSr.r rr0s rr zBo2Model.scoreJs@ I %(=(= =sC!Gq=!44s37AFFrNr1r$rrr3r3EsCGrr3ceZdZdZdZy)KLModelcV|t|j|z ztdz |zS)Nr))rr rs rrzKLModel.normalizerPs0C 5 5 ABBSXM rct||z }||jz }||kry|t|||jz z dzS)Nrr/r6)rrrr wit_over_tt wic_over_cts rr z KLModel.scoreTsT#i/ *T-B-BB  $[(58M8M(M&O%&"(( (rNr1r$rrr8r8Os  (rr8c4eZdZdZefdZdZdZdZddZ y) ExpanderzeUses an ExpansionModel to expand the set of query terms based on the top N result documents. c||_||_|jj}|jj|}t |tur |||}||_t t|_d|_ y)aT :param reader: A :class:whoosh.reading.IndexReader object. :param fieldname: The name of the field in which to search. :param model: (classify.ExpansionModel) The model to use for expanding the query terms. If you omit this parameter, the expander uses :class:`Bo1Model` by default. rN) ixreader fieldname doc_count_allrtypemodelrfloat topN_weightr)rr@rArDdoccountfieldlens rrzExpander.__init__eso! "MM//1==--i8 ;$ (H-E 'u-rcd}|j}|D]\}}||z }||xx|z cc<|xj|z c_y)zAdds forward-index information about one of the "top N" documents. :param vector: A series of (text, weight) tuples, such as is returned by Reader.vector_as("weight", docnum, fieldname). rN)rFr)rvector total_weightrFwordweights raddz Expander.add}sQ && " (LD& F "L   '  ( ,&rc|j}|jj||jr-|j|j d||jy|jj |jj r:|j|j|j|jytd|jd|d)NrMzField z in document z is not vectored or stored) r@ has_vectorrArN vector_asschemastoredadd_text stored_fieldsget Exception)rdocnumr@s r add_documentzExpander.add_documents== == # #FDNN ; HHX''&$..I J ]] ! !$.. 1 8 8 MM(008<z$Expander.add_text..s()0Ba*T"F+)s)r@rRrAr_rNindex)rstringfieldr_s @rrTzExpander.add_textsJ  $$T^^4%%  )KK') )rcn|j}|j}|j}|j|}g}d}|jsgSt |jD]f\} } |j | } || f|vs|j|| } |j| | |j} | |kDr| }|j| | fh|r|j||j}n|}|D cgc] \} }| |z |f}} }|jd|d|D cgc] \} }|| f c}} Scc}} wcc}} w)zReturns the N most important terms in the vectors added so far. :param number: The number of terms to return. :param normalize: Whether to normalize the weights. :returns: A list of ("term", weight) tuples. rcd|dz |dfSNrr$xs rz)Expander.expanded_terms..s!ad(AaD!1rkeyN) rDrAr@rRrFrto_bytes frequencyr rappendrsort)rnumber normalizerDrAr@rctlistrrLrMbtextcfr normts rexpanded_termszExpander.expanded_termss= NN == * I%d&6&67 ,LD&NN4(E5!X-'' 59 FB?9$ %I eT]+ , ##It~~>DD5:; &4-#;; 1 2-27F^< F <<<=s *D+D1N)T) r!r"r#__doc__r&rrNrYrTrxr$rrr>r>`s%3;0' 8 )#=rr>ctt}fdttdz z DD]}||xxdz cc<t |S)Nc3.K|] }||zywrr$)r\iinputsizes rr`zshingles..s$>!AH%>srg)rintrlenr)r}r~dshingles`` rshinglesrsQCA>#CJ$($;<> ' a  Q<rcdk(rt}nfd}dgz}|D]@\}}||}tD]%}|d|zzr||xx|z cc<||xx|zcc<'Bd}t|D]\}} | dkDs |d|zz}|S)N ct|Sr)_hash)shashbitss rrjzsimhash..s5H-rrrg)hashr enumerate) featuresrhashfnvsfeaturerMhr|outvs ` rsimhashrs2~- xB#  7O! AAF|11    C" 1 q5 16MC Jrc|dk(ryt|ddz}d}d|zdz }|D]}||zt|z |z}|t|z}|dk(rd}|S) NriCBr/rg)ordr)rrrimmaskcs rrrstBw !IN H}q  *Aa%3q6!T)A * SV  7ArcJ||z d|zdz z}d}|r|dz }||dz z}|r|S)Nrgrr$) first_hash other_hashrritots rhamming_distancersD j a8mq%89A C  q QU   Jrcd}t|}|}dg|z} dg|z} dg|z} |r|} ntj||} d} |}d}t|D] }d| |<d| |<t|D]W}|}t|D]}||| |z dz}||ks|| |<|}| | |xx||z cc<| | |xxdz cc<||z }Yt|D]}| |r | || |z n| || |<| dz } t ||z |ks| |kDr | | fS)ay One-dimensional K-means clustering function. :param data: list of data points. :param k: number of clusters. :param t: tolerance; stop if changes between iterations are smaller than this value. :param distfun: a distance function. :param centers: a list of centroids to start with. :param maxiter: maximum number of iterations to run. g`rr/rg)rrandomsamplerabs)datakrwdistfunmaxitercenters DOUBLE_MAXnerrorcountslabelsc1rniter old_errorr|r min_distancedistances rkmeansrs J D A ES1WFS1WF qB  MM$ " E   AF1IBqE  "A%LAY , GadNq0l* !F1I#+L  , vayMT!W $M 6!9  "  \ !E " =A(.q 2a56!9$r!uAaD =    ! "Q &EGO  19A rcxd}d}d}|D] }|dz }||z}||z }|D]}|||z ||z zz }||dz z }|Srfr$)rrsum1sum2rimeanvariances rtwo_pass_variancerKsw A D D  Qax !8D ( Ta$h''(q1u~H Orc~d}d}d}|D])\}}||z}||z }||z|z }|||z|zz }||z }|}+||dz z } | Srfr$) data_weight_pairsrS sumweightrirMtempQRVariances rweighted_incremental_variancer]s{ D AI& 6 ! H J  Y]Q    IM"H OrcDg}t|D]|\}}|}||}|t|dz kr,||z |kr$|dz }||}|t|dz kr ||z |kr$d}||z dkDrt|||dz}|j||||z |f~|j d|S)Nrgicd|dz |dfS)Nrr/r$rhs rrjzswin..xsQqT1Q4 0rrk)rrrrorp)rr~clustersr|leftjrightrs rswinrlsHT? 14 Q#d)a-EDL4$7 FAGE#d)a-EDL4$7  q519!$qQ-0Aua!eQ/0 1 MM0M1 Or)r/)r)g-C6?N2N)ry __future__rr collectionsrmathr whoosh.compatrrobjectr r&r3r8r>rrrrrrrrr$rrrs8 #+ "V""G~GG~G(n("e=ve=T,  CP$  r