$i@dZddlmZmZmZmZmZmZmZ dZ dZ dZ dZ dZdZd Zd Zd Zd ZiZiZiZiZd ZdZe[[dZdZdZdZdddZdZdZ dZ!dZ"dZ#e"e e#e!dZ$dZ%e&dk(rddl'Z'e'jPyy)aCore functions for Unicode normalization. This module provides the main functions for applying the four Unicode normalization forms: NFC, NFD, NFKC, and NFKD. It relies on data files from the Unicode character database (UCD) associated with version 17.0 of the Unicode Standard. As a result, the module can only fully handle the characters defined in version 17.0 and produces results consistent with the definitions and rules of that version. )_COMPOSITION_EXCLUSIONS_DECOMP_BY_CHARACTER_NFC__QC_NO_OR_MAYBE _NFD__QC_NO_NFKC_QC_NO_OR_MAYBE _NFKD_QC_NO_NON_ZERO_CCC_TABLEiiiiiaiuiic|D]t}|g}|r_g}|D]M}|tvr|jt|$||vr|j||=|j|O||k(rn|}|r_|x||<t|<vyN)_SHARED_FULL_DECOMP_CACHEextendappend) decomp_dictkey decomposition next_decompks K/var/www/br/venv/lib/python3.12/site-packages/pyunormalize/normalization.py_compute_full_decompositionsr=sJ K" *11&&'@'CD+%&&{1~6&&q)  *m+'M$=JI C4S9+Jc2tjD]b\}}t|dtr>t |dk(r|dt vr|t t|<|xt|<t|<W|ddt|<dtttty)Nr) ritems isinstanceintlen_CCC_COMPOSITE_BY_CDECOMPtuple_FULL_CDECOMP_BY_CHAR_FULL_KDECOMP_BY_CHARr)rvals r_init_normalization_tablesr&^s(..01S c!fc "3x1}Qt!347%eCj1FI I !# &)>s)C*-QR !# &1!!67!!67rcnd}|D].}t|}||vry|tvrt|x}|kry|}0y)NrFT)ordr )stringquick_check_setprev_ccccharcpcurr_cccs r _quick_checkr/ysT H  Y   T> R HH ,  rc|tz }|tz}||z tz}t|tzz}t|tzz}|r||t dz |zfS||fSNr)_SB_TCOUNT_VB_VCOUNT_LB_TB)r-sindextindexqVLs r_decompose_hangul_syllabler=sc#XF g F &W$A qG|A qG|A 1cAg&'' q6MrcBt|cxkr tkrAnn>t|cxkr tkr+nn(t|tz t z|ztz t zzSt|cxkr tkr8ny|tz t zs%t|cxkr tkrny||ztdz z Syr1) r6_LLr4_VLr2r5r3_SLr7_TL)r-next_cps r_compose_hangul_syllablerDs bCC71c1SG+w6<GGG rS cW$ 7 !c ! G|sQw'' rcBt|}|dkDrqd}d}||kr`||}|tvr|dz }||dz }|tvst|t|kr|dz }A||||dz c||dz <||<|}|dz }||kr`|}|dkDrqdjtt|S)Nrrr)rr joinmapchr) codepointssizeswap_posicurrprevs r_reorderrPs z?D ( $ha=D4Qa!e$D44:d#;Q0:!}jQ>O ,Jq1u z!}H FA-$h2; (> 773sJ' ((rF compatibilitycg}|rtnt}tt|D]]}||vr|j ||t |cxkr t krnn|j t|M|j|_|Sr ) r$r#rHr(rr2rAr=r)r)rRrJdecompr-s r _decomposerUszJ&3 "9NF#v" <   fRj ) B #    8< =   b ! " rc gtt|}t|D]\}}||tvrd}d}t||dzd|dzD]}\}}|tvrd}nd}|r|r||dz }||tvst|t|ks>||f} | tvr t| } nt | } | r| t vrd||<| x||<}|rd}wd}z|s}djtttd|S)NFrTrF) rHr( enumerater r!rDrrGrIfilter) r)rJrMr- is_blockedlast_is_combiningjrCprev_cppairprecomposed_chars r_composer_sL %3sF#$J:&=2 :t  !$Jq1uw$7Q?2 JAw$$(!" / Q'G$&=4=0G}00'>> string = "élève" >>> nfd = NFD(string) >>> nfd 'élève' >>> nfd != string # binary content differs True >>> " ".join([f"{ord(c):04X}" for c in string]) '00E9 006C 00E8 0076 0065' >>> " ".join([f"{ord(c):04X}" for c in nfd]) '0065 0301 006C 0065 0300 0076 0065' # Decomposing Hangul syllables >>> string = "한국" >>> nfd = NFD(string) >>> nfd '한국' >>> " ".join([f"{ord(c):04X}" for c in string]) 'D55C AD6D' >>> " ".join([f"{ord(c):04X}" for c in nfd]) '1112 1161 11AB 1100 116E 11A8' # Compatibility characters are unaffected >>> strings = ["ffi", "⑴", "²", "ｱｲｳｴｵ"] >>> all(NFD(s) == s for s in strings) True )r/rrPrUr)s rNFDrbAs$XFK( Jv& ''rcRt|tr|Stt|dS)u Return the normalization form KD of the input string. Replaces composed characters with their canonically equivalent decomposed forms in canonical order and converts compatibility characters into their nominal counterparts. The function first checks if the input is already in NFKD. If so, it returns the string unchanged to avoid unnecessary processing. Args: string (str): The string to be normalized. Returns: str: The NFKD string. Examples: # NFKD decomposes canonically composed forms >>> NFKD("élève") == "élève" True # NFKD converts compatibility characters >>> [NFKD(s) for s in ["ffi", "⑴", "²", "ｱｲｳｴｵ"]] ['ffi', '(1)', '2', 'アイウエオ'] TrQ)r/rrPrUras rNFKDrdrs%2FK( JvT: ;;rcNt|tr|Stt|S)uDReturn the normalization form C of the input string. Replaces character sequences with their canonically equivalent composed forms whenever possible, while leaving compatibility characters unaffected. The function first checks if the input is already in NFC. If so, it returns the string unchanged to avoid unnecessary processing. Args: string (str): The string to be normalized. Returns: str: The NFC string. Examples: # Composing accented characters >>> string = "élève" >>> nfc = NFC(string) >>> nfc 'élève' >>> nfc != string # binary content differs True >>> " ".join([f"{ord(c):04X}" for c in string]) '0065 0301 006C 0065 0300 0076 0065' >>> " ".join([f"{ord(c):04X}" for c in nfc]) '00E9 006C 00E8 0076 0065' # Composing Hangul syllables >>> string = "한국" >>> nfc = NFC(string) >>> nfc '한국' >>> " ".join([f"{ord(c):04X}" for c in string]) '1112 1161 11AB 1100 116E 11A8' >>> " ".join([f"{ord(c):04X}" for c in nfc]) 'D55C AD6D' # Compatibility characters are unaffected >>> strings = ["ffi", "⑴", "²", "ｱｲｳｴｵ"] >>> all(NFC(s) == s for s in strings) True )r/rr_rbras rNFCrfs$VF01 CK  rcNt|tr|Stt|S)uReturn the normalization form KC of the input string. Replaces character sequences with their canonically equivalent composed forms whenever possible and converts compatibility characters into their nominal counterparts. The function first checks if the input is already in NFKC. If so, it returns the string unchanged to avoid unnecessary processing. Args: string (str): The string to be normalized. Returns: str: The NFKC string. Examples: # NFKC composes canonically decomposed forms >>> NFKC("élève") == "élève" True # NFKC converts compatibility characters >>> [NFKC(s) for s in ["ffi", "⑴", "²", "ｱｲｳｴｵ"]] ['ffi', '(1)', '2', 'アイウエオ'] )r/rr_rdras rNFKCrhs#2F01 DL !!r)rfrbrhrdct||S)uEReturn the normalized form of the input string as specified by `form`. This function transforms the input string according to the normalization form given in `form`. Supported values are "NFC", "NFD", "NFKC", and "NFKD". Args: form (str): The normalization form to apply, one of "NFC", "NFD", "NFKC", or "NFKD". string (str): The string to be normalized. Returns: str: The normalized string. Examples: >>> normalize("NFKD", "fluffiness") 'fluffiness' >>> forms = ["NFC", "NFD", "NFKC", "NFKD"] >>> string = "ẛ̣" >>> def hexpoints(s): ... return " ".join([f"{ord(c):04X}" for c in s]) >>> [hexpoints(normalize(f, string)) for f in forms] ['1E9B 0323', '017F 0323 0307', '1E69', '0073 0323 0307'] )_NORMALIZATION_FORMS)formr)s r normalizerls6  %f --r__main__N))__doc__pyunormalize._unicode_datarrrrrrr r r2rAr6r?r4r@r7rBr5r3rr!r#r$rr&r/r=rDrPrUr_rbrdrfrhrjrl__name__doctesttestmodrrrts          JB8*30  4')T).$G7T.(b<<-!`"> #3dK.< zGOOr