+
    öäûj€  ã            	      óÈ  a € 0 t $ R t^ RIHt ^ RIt^ RIHt ^ RIHt	 ^ RIH
t ^ RIHt ^ RIHt ^ RIHt ^ R	IHtHtHtHtHt ^ R
IHt ^ RIHtHt . RIOtRtRtRt Rt!R]"R&   R R lt# ^ RI$t$]PJ                  PM                  4       '       g   Rt!MRt]'! ]$R4      t]'! ]$R4      t  ];'       d6    ]'! ]$R4      ;'       d%    ]'! ]$R4      ;'       d    ]#! ]$PV                  R4      t,];'       d%    ]'! ]$R4      ;'       d    ]#! ]$PZ                  R4      t.].;'       d
    ]'! ]$R4      t/].'       d(   ^RI0H1t1H2t2H3t3H4t4H5t5H6t6H7t7H8t8H9t9H:t:H;t;H<t< ]. RJO,          t]/'       d(   ^RI0H=t=H>t>H?t?H@t@HAtAHBtBHCtCHDtDHEtEHFtFHGtGHHtH ]. RKO,          t^ ]P’                  ^]P”                  ^]P–                  /tL^tM^€tNRtO^tP^@tQ]P¤                  ! ]P¦                  4      P¨                  ^8,          tUR  R! ltVR" R# ltWR$ R% ltXR& R' ltYR( R) ltZR* R+ lt[]M3R, R- llt\R. R/ lt]R0 R1 lt^RLR2 R3 llt_RMR4 R5 llt`RMR6 R7 lltaR8 R9 ltbR: R; ltcR]P–                  RR<RRR=RR3	R> R? lltdRR<]QR@3RA RB llteRC RD ltf]'       d%   ]PÎ                  ! RE]h! ]i]RF7      ]f! 4       RG7       R# ]PÔ                  ! RE]!;'       g    RH4       R#   ]( d   t)R]) 2t! Rt)A)EL"Rt)A)i]* d   t)R]) 2t! Rt)A)EL7Rt)A)ii ; i)Nz¦Huawei Ascend NPU backend.

The backend is intentionally optional: importing :mod:`comfy_kitchen` must not
require torch-npu on CPU, CUDA, HIP, or XPU installations.
)ÚannotationsN)ÚINPUT_ACT_TO_CODE)Úapply_input_act)Úapply_residual)Úinput_act_width)Ú&_quantize_signed_int4_rowwise_unpacked)Ú_unpack_int4_row_major)Ú	ExactDimsÚFunctionConstraintsÚMinDimsÚParamConstraintÚValidationResult)Úregistry)Ú_build_hadamardÚ_rotate_activationFú
str | NoneÚ_ASCEND_ERRORc               ó$   € V ^8„  d   QhRRRRRR/# )é   ÚoperatorÚobjectÚ	parameterÚstrÚreturnÚbool© )Úformats   "ÚbC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\backends\ascend\__init__.pyÚ__annotate__r   3   s&   € ÷ Eñ E fð E¸ð EÀñ Eó    c                óê   a€  V P                   P                  P                  p\
        ;QJ d    T3R lT 4       F  '       g   K   R# 	  R# ! T3R lT 4       4      #   \        \        3 d     R# i ; i)zFReturn whether the default torch operator schema contains a parameter.Fc              3  ó@   <"  € T F  qP                   S8H  x € K  	  R # 5i)N)Úname)Ú.0Úargumentr   s   & €r   Ú	<genexpr>Ú*_operator_has_parameter.<locals>.<genexpr>9   s   øé € ÐD¹)¨h�}‰} 	Ö)»)ùs   ƒT)ÚdefaultÚ_schemaÚ	argumentsÚAttributeErrorÚRuntimeErrorÚany)r   r   r)   s   &f r   Ú_operator_has_parameterr-   3   sc   ø€ ðØ×$Ñ$×,Ñ,×6Ñ6ˆ	÷ ‹3ÔD¹)ÓD�3Œ3ÐDŠ3ÐDˆ3ÔD¹)ÓDÓDÐDøô œLÐ)ô Úðús   ƒ A ÁA2Á1A2z@torch-npu is installed, but no Huawei Ascend device is availableTÚnpu_quant_matmulÚnpu_rotate_quantztorch-npu is not installed: z!torch-npu initialization failed: Únpu_dynamic_quantÚnpu_quantizeÚdiv_modeÚnpu_rotary_mulÚrotary_modeÚnpu_rms_norm)Ú
apply_ropeÚapply_rope1Úapply_rope1_Úapply_rope_Úapply_rope_split_halfÚapply_rope_split_half1Úapply_rope_split_half1_Úapply_rope_split_half_Úvalidate_apply_ropeÚvalidate_apply_rope1Úvalidate_apply_rope_split_halfÚvalidate_apply_rope_split_half1)Úrms_ropeÚ	rms_rope1Ú
rms_rope1_Ú	rms_rope_Úrms_rope_split_halfÚrms_rope_split_half1Úrms_rope_split_half1_Úrms_rope_split_half_Úvalidate_rms_ropeÚvalidate_rms_rope1Úvalidate_rms_rope_split_halfÚvalidate_rms_rope_split_half1i€>  c               ó   € V ^8„  d   QhRR/# ©r   r   r   r   )r   s   "r   r   r   ž   s   € ÷ !ñ !Ð4Dñ !r   c                ó’   € V P                  R 4      pVe   V^ 8”  d   \        P                  ! R R4      # \        P                  ! 4       # )Ústochastic_roundingz2not supported by the Ascend quantization operators)Úgetr   ÚfailÚok)ÚkwargsrQ   s   & r   Ú$_validate_deterministic_quantizationrV   ž   sJ   € Ø Ÿ*™*Ð%:Ó;ÐØÒ&Ð+>ÀÔ+BÜ×$Ò$Ø!Ð#Wó
ð 	
ô ×ÒÓ Ð r   c               ó   € V ^8„  d   QhRR/# rO   r   )r   s   "r   r   r   §   s   € ÷ 
!ñ 
!Ð*:ñ 
!r   c                óŒ  € \        V 4      pVP                  '       g   V# V P                  R 4      p\        V\        4      '       d   VR8w  d   \
        P                  ! R R4      # \        V\        P                  4      '       d-   VP                  4       ^8w  d   \
        P                  ! R R4      # \
        P                  ! 4       # )ÚscaleÚrecalculatez"string value must be 'recalculate'z must contain exactly one element)rV   ÚsuccessrR   Ú
isinstancer   r   rS   ÚtorchÚTensorÚnumelrT   )rU   ÚresultrY   s   &  r   Ú_validate_tensorwise_scalera   §   s‘   € Ü1°&Ó9€FØ�>�>ˆ>Øˆà�J‰J�wÓ€EÜ�%œ×Ò %¨=Ô"8Ü×$Ò$ WÐ.RÓSÐSÜ�%œŸ™×&Ò&¨5¯;©;«=¸AÔ+=Ü×$Ò$ WÐ.PÓQÐQÜ×ÒÓ Ð r   c               ó   € V ^8„  d   QhRR/# rO   r   )r   s   "r   r   r   ´   s   € ÷ !ñ !Ð&6ñ !r   c                ó’   € V P                  R 4      pV\        9  d   \        P                  ! R R4      # \        P                  ! 4       # )Úoutput_dtype_codez)must select float32, float16, or bfloat16)rR   Ú_DTYPE_CODE_TO_DTYPEr   rS   rT   )rU   rd   s   & r   Ú_validate_output_dtyperf   ´   sE   € ØŸ
™
Ð#6Ó7ÐØÔ 4Ô4Ü×$Ò$ØÐ!Ló
ð 	
ô ×ÒÓ Ð r   c               ó    € V ^8„  d   QhRRRR/# )r   rY   útorch.Tensorr   r   )r   s   "r   r   r   ½   s   € ÷ #ñ #�|ð #¨ñ #r   c                ó$   € V P                  R 4      # )ç ÂëþKH´9)Ú
clamp_min_)rY   s   &r   Ú_safe_scalerl   ½   s   € ð ×Ñ˜EÓ"Ð"r   c               ó   € V ^8„  d   QhRR/# rO   r   )r   s   "r   r   r   Ã   s   € ÷ )!ñ )!Ð&6ñ )!r   c                óŒ  € \        V 4      pVP                  '       g   V# V P                  R 4      pV P                  RV P                  R4      4      pV P                  R4      p\        V\        4      '       d   V\
        8  d   \        P                  ! RR\
         24      # V^,          ^ 8w  d   \        P                  ! RR4      # \        V\        P                  4      '       d!   \        V\        P                  4      '       g   \        P                  ! 4       # VP                  4       ^ 8X  d   \        P                  ! R R4      # VP                  R,          \        8  d   \        P                  ! R R\         24      # VP                  R,          \        8”  d   \        P                  ! R R\         24      # VP                  R,          V,          ^ 8w  d.   \        P                  ! RV R	VP                  R,           24      # VP                  WD38w  d4   \        P                  ! RR
V RV R\        VP                  4       24      # VP                   VP                   8w  d   \        P                  ! RR4      # VP"                  VP"                  8w  d   \        P                  ! RR4      # \        P                  ! 4       # )ÚxÚHÚhÚ
group_sizez,must be an integer greater than or equal to zmust be divisible by 8z3empty tensors are not supported by npu_rotate_quantú last dimension must be at least zlast dimension must not exceed ú  does not divide input features zmust have shape (z, z), got zmust be on the same device as xzmust have the same dtype as xéÿÿÿÿ)rV   r[   rR   r\   ÚintÚ_ROTATE_QUANT_MIN_GROUP_SIZEr   rS   r]   r^   rT   r_   ÚshapeÚ_ROTATE_QUANT_MIN_FEATURESÚ_ROTATE_QUANT_MAX_FEATURESÚtupleÚdeviceÚdtype)rU   r`   ro   rq   rr   s   &    r   Ú_validate_rotate_quantr~   Ã   sB  € Ü1°&Ó9€FØ�>�>ˆ>Øˆà�
‰
�3‹€AØ�
‰
�3˜Ÿ
™
 3›Ó(€AØ—‘˜LÓ)€JÜ�j¤#×&Ò&¨*Ô7SÔ*SÜ×$Ò$ØØ:Ô;WÐ:XÐYó
ð 	
ð �A…~˜ÔÜ×$Ò$ \Ð3KÓLÐLÜ�aœŸ™×&Ò&¬j¸¼E¿L¹L×.IÒ.IÜ×"Ò"Ó$Ð$Ø‡w�wƒy�A„~Ü×$Ò$ SÐ*_Ó`Ð`Ø‡w�wˆr…{Ô/Ô/Ü×$Ò$ØØ.Ô/IÐ.JÐKó
ð 	
ð 	‡w�wˆr…{Ô/Ô/Ü×$Ò$ØØ-Ô.HÐ-IÐJó
ð 	
ð 	‡w�wˆr…{�ZÕ 1Ô$Ü×$Ò$Ø˜Z˜LÐ(HÈÏÉÐQSÍÈÐVó
ð 	
ð 	‡w�w�:Ð*Ô*Ü×$Ò$ØÐ$ Z L°°:°,¸gÄeÈAÏGÉGÃnÐEUÐVó
ð 	
ð 	‡x�x�1—8‘8ÔÜ×$Ò$ SÐ*KÓLÐLØ‡w�w�!—'‘'ÔÜ×$Ò$ SÐ*IÓJÐJÜ×ÒÓ Ð r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   ro   rh   rq   rr   rv   r   r   r   )r   s   "r   r   r   ï   s)   € ÷ 
ñ 
˜\ð 
¨lð 
Èð 
ÐPTñ 
r   c           
     óX   € \         '       g   R # \        RV RVRVR^ /4      P                  # )Fro   rp   rr   rQ   )Ú_ASCEND_ROTATE_QUANT_AVAILABLEr~   r[   )ro   rq   rr   s   &&&r   Ú_can_use_rotate_quantr‚   ï   s8   € ß)Ó)ÙÜ!à�Ø�Ø˜*Ø! 1ð		
ó÷ �gðr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   ro   rh   rq   Ú	dst_dtyperv   r   ú!tuple[torch.Tensor, torch.Tensor]r   )r   s   "r   r   r   ü   s0   € ÷ ñ Øðàðð ðð 'ñ	r   c                ó>   € \         P                  ! V VVRR^ RRR7      # )é   Úrintç        F)r„   ÚaxisÚ
round_modeÚ	scale_algÚdst_type_maxÚtranspose_yru   )Ú	torch_npur/   )ro   rq   r„   s   &&&r   Ú_npu_rotate_quantr�   ü   s/   € ô
 ×%Ò%Ø	Ø	ØØØØØØô	ð 	r   c               ó   € V ^8„  d   QhRR/# rO   r   )r   s   "r   r   r     s   € ÷ ;!ñ ;!Ð%5ñ ;!r   c                ó²  € V P                  R 4      pV P                  R4      pV P                  R4      pV P                  R4      pV P                  R4      pV\        9  d   \        P                  ! RRV: 24      # \	        V\
        P                  4      '       d!   \	        V\
        P                  4      '       g   \        P                  ! 4       # VP                  4       ^ 8X  g   VP                  4       ^ 8X  d   \        P                  ! R R4      # VP                  R,          pVR8X  d[   V P                  R4      p\	        V\
        P                  4      '       d   VP                  V38w  d   \        P                  ! RR	4      # V P                  R
4      e+   V P                  R4      f   \        P                  ! RR4      # \        V4      pWh,          ^ 8w  d   \        P                  ! R RV RV 24      # Wh,          p	W’P                  R,          8w  d/   \        P                  ! RRV	 RVP                  R,           24      # VeU   VP                  4       ^VP                  ^ ,          39  d-   \        P                  ! RRVP                  ^ ,           R24      # VeS   VP                  4       VP                  ^ ,          8w  d-   \        P                  ! RRVP                  ^ ,           R24      # V P                  R4      '       d»   V P                  RR4      p
\	        V
\        4      ;'       dI    V
^8¬  ;'       d<    Wª^,
          ,          ^ 8H  ;'       d"    V
P                  4       ^,
          ^,          ^ 8H  pV'       g   \        P                  ! RR4      # Wš,          ^ 8w  d   \        P                  ! RV
 RV	 24      # \        P                  ! 4       # )ro   ÚweightÚweight_scaleÚbiasÚ	input_actzunsupported value z3empty tensors are not supported by npu_quant_matmulÚrms_normÚinput_act_weightz8rms_norm requires a one-dimensional input-channel weightÚresidualÚresidual_scalez"required when residual is providedzlast dimension z& is not divisible by activation width úinput features z do not match weight features zmust be scalar or contain ú per-output-channel valuesúmust contain ú output-channel valuesÚconvrotÚconvrot_groupsizeé   z must be a positive power of fourrt   ru   )rR   Ú_INPUT_ACT_TO_CODEr   rS   r\   r]   r^   rT   r_   rx   Ú_input_act_widthrv   Ú
bit_length)rU   ro   r“   r”   r•   r–   Úinput_featuresÚnorm_weightÚwidthÚactivated_featuresrr   Úis_power_of_fours   &           r   Ú_validate_int8_linearrª     sd  € Ø�
‰
�3‹€AØ�Z‰Z˜Ó!€FØ—:‘:˜nÓ-€LØ�:‰:�fÓ€DØ—
‘
˜;Ó'€IàÔ*Ô*Ü×$Ò$ [Ð4FÀyÁmÐ2TÓUÐUÜ�aœŸ™×&Ò&¬j¸ÄÇÁ×.NÒ.NÜ×"Ò"Ó$Ð$Ø‡w�wƒy�A„~˜Ÿ™›¨1Ô,Ü×$Ò$ SÐ*_Ó`Ð`à—W‘W˜R•[€NØ�JÔØ—j‘jÐ!3Ó4ˆÜ˜+¤u§|¡|×4Ò4¸×8IÑ8IÈnÐM^Ô8^Ü#×(Ò(Ø"Ð$^óð ð ‡z�z�*ÓÒ)¨f¯j©jÐ9IÓ.JÒ.RÜ×$Ò$Ð%5Ð7[Ó\Ð\Ü˜YÓ'€EØÕ Ô"Ü×$Ò$Ø�? >Ð"2Ð2XÐY^ÐX_Ð`ó
ð 	
ð (Õ0ÐØŸ\™\¨"Õ-Ô-Ü×$Ò$ØØÐ0Ð1Ð1OÐPV×P\ÑP\Ð]_ÕP`ÐOaÐbó
ð 	
ð Ò L×$6Ñ$6Ó$8ÀÀFÇLÁLÐQRÅOÐ@TÔ$TÜ×$Ò$ØØ(¨¯©°a­Ð(9Ð9SÐTó
ð 	
ð Ò˜DŸJ™J›L¨F¯L©L¸­OÔ;Ü×$Ò$Ø�m F§L¡L°¥OÐ#4Ð4JÐKó
ð 	
ð ‡z�z�)×ÒØ—Z‘ZÐ 3°SÓ9ˆ
ä�z¤3Ó'÷ 7ð 7Ø˜a‘÷7ð 7à¨A�~Õ.°1Ñ4÷7ð 7ð ×&Ñ&Ó(¨1Õ,°Õ1°QÑ6ð	 	÷  Ü#×(Ò(Ð)<Ð>`ÓaÐaØÕ*¨aÔ/Ü#×(Ò(Ø#Ø�,Ð>Ð?QÐ>RÐSóð ô ×ÒÓ Ð r   c               ó   € V ^8„  d   QhRR/# rO   r   )r   s   "r   r   r   K  s   € ÷ >!ñ >!Ð-=ñ >!r   c                óø  € V P                  R 4      pV P                  R4      pV P                  R4      pV P                  R4      pV P                  RR4      pV P                  R\        4      pV P                  RR4      pVR8w  d   \        P                  ! RR	4      # V\        8w  d   \        P                  ! RR
\         24      # \	        V\
        4      ;'       dM    V\        8¬  ;'       d<    WU^,
          ,          ^ 8H  ;'       d"    VP                  4       ^,
          ^,          ^ 8H  pV'       g   \        P                  ! RR4      # \	        V\        P                  4      '       d!   \	        V\        P                  4      '       g   \        P                  ! 4       # VP                  4       ^ 8X  g   VP                  4       ^ 8X  d   \        P                  ! R R4      # VP                  R,          p	VP                  ^ ,          p
V	\        8”  d   \        P                  ! R R4      # W’P                  R,          ^,          8w  d6   \        P                  ! RRV	 RVP                  R,          ^,           24      # V	\        8  d   \        P                  ! R R\         24      # W•,          ^ 8w  d   \        P                  ! RV RV	 24      # V	^,          ^ 8w  g   V
^,          ^ 8w  d   \        P                  ! RR4      # Ve1   VP                  4       V
8w  d   \        P                  ! RRV
 R24      # Ve1   VP                  4       V
8w  d   \        P                  ! RRV
 R24      # \        P                  ! 4       # )ro   ÚqweightÚwscalesr•   r    r¡   Úquant_group_sizeÚlinear_dtypeÚint4zAscend A4W4 requires 'int4'zAscend A4W4 requires z3must be a power of four greater than or equal to 16z.empty tensors are not supported by Ascend A4W4z0feature width can overflow the INT32 accumulatorr›   z% do not match packed weight features rs   rt   z=Ascend A4W4 requires input and output features divisible by 8r�   rœ   rž   ru   )rR   Ú_INT4_QUANT_GROUP_SIZEr   rS   r\   rv   rw   r¤   r]   r^   rT   r_   rx   Ú_INT4_MAX_ACCUMULATION_FEATURESry   )rU   ro   r­   r®   r•   r    r¯   r°   r©   r¥   Úoutput_featuress   &          r   Ú_validate_convrot_w4a4_linearrµ   K  s7  € Ø�
‰
�3‹€AØ�j‰j˜Ó#€GØ�j‰j˜Ó#€GØ�:‰:�fÓ€DØŸ
™
Ð#6¸Ó<ÐØ—z‘zÐ"4Ô6LÓMÐØ—:‘:˜n¨fÓ5€Là�vÔÜ×$Ò$ ^Ð5RÓSÐSØÔ1Ô1Ü×$Ò$ØØ#Ô$:Ð#;Ð<ó
ð 	
ô
 	Ð$¤cÓ*÷ 	:ð 	:ØÔ!=Ñ=÷	:ð 	:à°aÕ"7Õ8¸QÑ>÷	:ð 	:ð ×)Ñ)Ó+¨aÕ/°1Õ4¸Ñ9ð	 ÷ Ü×$Ò$ØÐ!Vó
ð 	
ô �aœŸ™×&Ò&¬j¸Ä%Ç,Á,×.OÒ.OÜ×"Ò"Ó$Ð$Ø‡w�wƒy�A„~˜Ÿ™›¨AÔ-Ü×$Ò$ SÐ*ZÓ[Ð[à—W‘W˜R•[€NØ—m‘m AÕ&€OØÔ7Ô7Ü×$Ò$ SÐ*\Ó]Ð]ØŸ™ rÕ*¨QÕ.Ô.Ü×$Ò$ØØ˜nÐ-ð .ØŸ™ bÕ)¨AÕ-Ð.ð0ó
ð 	
ð
 Ô2Ô2Ü×$Ò$ØÐ3Ô4NÐ3OÐPó
ð 	
ð Õ)¨QÔ.Ü×$Ò$ØØ Ð!Ð!AÀ.ÐAQÐRó
ð 	
ð ˜Õ˜QÔ /°AÕ"5¸Ô":Ü×$Ò$ØÐVó
ð 	
ð Ò˜wŸ}™}›°/ÔAÜ×$Ò$Ø˜ Ð&7Ð7QÐRó
ð 	
ð Ò˜DŸJ™J›L¨OÔ;Ü×$Ò$Ø�m OÐ#4Ð4JÐKó
ð 	
ô ×ÒÓ Ð r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   ro   rh   rY   z!torch.Tensor | float | str | NonerQ   ú
int | Noner   r…   r   )r   s   "r   r   r   Œ  s0   € ÷ #ñ #Øð#à,ð#ð $ð#ð 'ñ	#r   c           	     ób  € ?Ve   \        V\        4      '       dP   VR8X  dI   V P                  4       P                  4       pVP	                  4       R,          P                  RR7      pTpMŠ\        P                  ! V\        P                  V P                  R7      p\        P                  ! V P                  4      P                  p\        P                  ! V^ 8H  \        P                  ! WF4      V4      p\        P                   ! V VP#                  ^4      R\        P$                  R	RR7      pWt3# )
zGQuantize an Ascend tensor using one float32 scale for the whole tensor.NrZ   g     À_@rj   )Úmin)r}   r|   T)Úzero_pointsr}   rŠ   r2   ru   )r\   r   ÚabsÚmaxÚfloatÚclampr]   Ú	as_tensorÚfloat32r|   Úfinfor}   ÚtinyÚwhereÚ	full_liker�   r1   ÚreshapeÚqint8)ro   rY   rQ   Úabs_maxÚoutput_scaleÚquantization_scaleÚ	scale_minÚ	quantizeds   &&&     r   Úquantize_int8_tensorwiserÌ   Œ  sê   € ð 	à‚}œ E¬3×/Ò/°E¸]Ô4JØ—%‘%“'—+‘+“-ˆØŸ™›¨%Õ/×6Ñ6¸5Ð6ÓAˆØ)Ñä—’ u´E·M±MÈ!Ï(É(ÔSˆÜ—K’K §¡Ó(×-Ñ-ˆ	Ü"Ÿ[š[Ø˜AÑÜ�OŠO˜LÓ4Øó
Ðô
 ×&Ò&Ø	Ø×"Ñ" 1Ó%ØÜ�k‰kØØô€Ið Ð"Ð"r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   ro   rh   rQ   r·   r   r…   r   )r   s   "r   r   r   «  s&   € ÷ 7ñ 7Øð7à#ð7ð 'ñ7r   c                ól   € ?\         P                  ! V 4      w  r#V\        V4      P                  R4      3# )z>Quantize an Ascend tensor per row using ``npu_dynamic_quant``.ru   )r�   r0   rl   Ú	unsqueeze)ro   rQ   rË   rY   s   &&  r   Úquantize_int8_rowwiserÐ   «  s5   € ð
 	Ü ×2Ò2°1Ó5Ñ€IØ”k %Ó(×2Ñ2°2Ó6Ð6Ð6r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r   ro   rh   rq   rr   rv   rQ   r·   r   r…   r   )r   s   "r   r   r   µ  s:   € ÷ ñ Øðàðð ðð $ð	ð
 'ñr   c                ó  € ?V P                   pV P                  RV P                   R,          4      P                  4       p\        WQP                  4       4      w  rg\	        V4      pVP                  V4      VP                  ! . VRR O^N5!  3# )z@Fuse grouped activation rotation and row-wise INT8 quantization.Nru   )rx   rÅ   Ú
contiguousr�   rl   )ro   rq   rr   rQ   Úoriginal_shapeÚx_2drË   rY   s   &&&&    r   Úquantize_and_rotate_rowwiserÖ   µ  s�   € ð 	Ø—W‘W€NØ�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DÜ(¨¯|©|«~Ó>Ñ€IÜ˜Ó€Eà×Ñ˜.Ó)Ø�ŠÐ.�~ c rÐ*Ð.¨AÓ.ðð r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úqrh   rY   r   r   )r   s   "r   r   r   Ç  s!   € ÷ ñ ˜lð °<ð ÀLñ r   c                ó0   € V P                  4       V,          # )zCDequantize INT8 data on Ascend without transferring it to the host.)r½   )rØ   rY   s   &&r   Údequantize_int8_simplerÚ   Ç  s   € à�7‰7‹9�uÕÐr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   rØ   rh   rY   rd   rv   r   r   )r   s   "r   r   r   Ì  s2   € ÷ Xñ XØðXØ(ðXØ=@ðXàñXr   c                óL   € \        W4      P                  \        V,          4      # )z;Dequantize INT8 data to the requested floating-point dtype.)rÚ   Útore   )rØ   rY   rd   s   &&&r   Údequantize_int8_simple_dtyperÞ   Ì  s!   € ô " !Ó+×.Ñ.Ô/CÐDUÕ/VÓWÐWr   r¡   r‰   c               óL   € V ^8„  d   QhRRRRRRRRRRR	R
RRRRRRRRRRRRRR/# )r   ro   rh   r“   r”   r•   útorch.Tensor | NoneÚ	out_dtypeztorch.dtyperŸ   r   r    rv   r–   r   r˜   Úinput_act_epsr½   r™   rš   r   r   )r   s   "r   r   r   Ó  s™   € ÷ @=ñ @=Øð@=àð@=ð ð@=ð ð	@=ð
 ð@=ð ð@=ð ð@=ð ð@=ð *ð@=ð ð@=ð "ð@=ð (ð@=ð ñ@=r   c           	     ó  € V P                   p\        WW‰4      p V P                   R,          VP                   R,          8w  d4   \        RV P                   R,           RVP                   R,           24      hVP                  V P                  R7      P                  4       pVP                  V P                  \        P                  R7      P                  R4      pVP                  4       ^VP                   ^ ,          39  d8   \        R\        VP                   4       R\        VP                   4       24      hV P                  RV P                   R,          4      P                  4       pV'       d¸   V P                   R,          V,          ^ 8w  d#   \        RV RV P                   R,           24      h\        W`P                  V P                  R7      p\        WÞV4      '       d   \        WÞP                  4       4      w  ppM?\        WÞV4      p\         P"                  ! V4      w  ppM\         P"                  ! V4      w  ppR	pVe;   VP                  V P                  VR7      P                  R4      P                  4       p\         P$                  ! VVP'                  4       VP                  4       VP                  R4      P                  4       VVR
7      pVP                  ! . VR	R OVP                   ^ ,          N5!  p\)        VW«4      # )zóRun dynamically quantized INT8 linear on Ascend NPU.

Comfy Kitchen stores weights as ``[N, K]``. ``npu_quant_matmul`` accepts
the resulting non-contiguous ``[K, N]`` transpose view directly, avoiding
a full weight copy on every forward pass.
z2Input and weight inner dimensions must match, got z and ©r|   ©r|   r}   z<INT8 weight scale must be scalar or per-output-channel, got z for weight shape zConvRot group size rt   N)Úpertoken_scaler•   Úoutput_dtyperu   )rx   Ú_apply_input_actÚ
ValueErrorrÝ   r|   rÓ   r]   rÀ   rÅ   r_   r{   r   r}   r‚   r�   r   r�   r0   r.   ÚtÚ_apply_residual)ro   r“   r”   r•   rá   rŸ   r    r–   r˜   râ   r™   rš   Ú
orig_shaperÕ   ÚhadamardÚquantized_xÚactivation_scaleÚnpu_biasr`   s   &&&&&&&&&&&&       r   Úint8_linearrñ   Ó  s‘  € ð( —‘€JÜ˜Ð'7ÓG€AØ‡w�wˆr…{�f—l‘l 2Õ&Ô&ÜðØ—7‘7˜2•;�-˜u V§\¡\°"Õ%5Ð$6ð8ó
ð 	
ð
 �Y‰Y˜aŸh™hˆYÓ'×2Ñ2Ó4€FØ—?‘?¨!¯(©(¼%¿-¹-�?ÓH×PÑPÐQSÓT€LØ×ÑÓ A v§|¡|°A¥Ð#7Ô7ÜðÜ˜×+Ñ+Ó,Ð-Ð-?ÄÀfÇlÁlÓ@SÐ?TðVó
ð 	
ð
 �9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DßØ�7‰7�2�;Ð*Õ*¨aÔ/ÜØ%Ð&7Ð%8ð 9"Ø"#§'¡'¨"¥+ ð0óð ô #Ð#4¿X¹XÈQÏWÉWÔUˆÜ  Ð1B×CÒCÜ,=¸d×DWÑDWÓDYÓ,ZÑ)ˆKÑ)ä% dÐ6GÓHˆDÜ,5×,GÒ,GÈÓ,MÑ)ˆKÑ)ä(1×(CÒ(CÀDÓ(IÑ%ˆÐ%Ø€HØÒØ—7‘7 !§(¡(°)�7Ó<×DÑDÀRÓH×SÑSÓUˆä×'Ò'ØØ�‰‹
Ø×ÑÓ!Ø'×/Ñ/°Ó3×>Ñ>Ó@ØØô€Fð �^Š^Ð>˜Z¨¨˜_Ð>¨f¯l©l¸1­oÓ>€FÜ˜6 8Ó<Ð<r   r±   c               ó8   € V ^8„  d   QhRRRRRRRRRRR	RR
RRR/# )r   ro   rh   r­   r®   r•   rà   r    rv   r¯   r°   r   r   r   )r   s   "r   r   r     sb   € ÷ 1Bñ 1BØð1Bàð1Bð ð1Bð ð	1Bð
 ð1Bð ð1Bð ð1Bð ñ1Br   c           
     óÐ  € VR8w  d   \        RV: 24      hV\        8w  d   \        R\         24      hV P                  pV P                  RV P                  R,          4      P	                  4       pVP                  V P                  R7      P	                  4       p\        VV P                  V P                  R7      p	\        W‰V4      p
\        V
4      w  r¼VP	                  4       p\        V4      P	                  4       p\        P                  ! VVP                  4       \        P                   ! VP                  ^ ,          V P                  \        P"                  R7      \        P$                  R7      pVP                  V P                  4      pWìP                  V P                  4      P                  R^4      ,          pWâP                  V P                  V P                  R7      P                  ^R4      ,          pVe>   WãP                  V P                  V P                  R7      P                  ^R4      ,           pVP                  ! . VRR OVP                  ^ ,          N5!  # )	aL  Run ConvRot W4A4 with reference preprocessing and integer accumulation.

Do not cast FP32 activations before rotation/quantization: even small
perturbations change the rounded A4 codes. The scaled packed-A4W4 kernel
also narrows output to FP16/BF16. Instead, retain the same A4/W4 codes in
INT8 containers and request an INT32 accumulator without per-token scaling, then
reproduce eager's cast, scale and bias order. This intentionally trades
packed-kernel performance for the reference's numerical behavior. Activation
codes stay in INT8 storage; only the packed weights need unpacking.
r±   z.Ascend A4W4 requires linear_dtype='int4', got z&Ascend A4W4 requires quant_group_size rä   rå   )rç   Nru   )ré   r²   rx   rÅ   rÓ   rÝ   r|   r   r}   r   r   r   r�   r.   rê   r]   ÚonesrÀ   Úint32)ro   r­   r®   r•   r    r¯   r°   rÔ   rÕ   rí   Úrotatedrî   rï   Úquantized_weightr`   s   &&&&&&&        r   Úconvrot_w4a4_linearrø     sá  € ð& �vÔÜÐIÈ,ÑIYÐZÓ[Ð[ØÔ1Ô1ÜÐAÔBXÐAYÐZÓ[Ð[à—W‘W€NØ�9‰9�R˜Ÿ™ �Ó%×0Ñ0Ó2€DØ�j‰j §¡ˆjÓ)×4Ñ4Ó6€GÜØØ�x‰xØ�g‰gô€Hô
 ! Ð1BÓC€Gô %KÈ7Ó$SÑ!€KØ×(Ñ(Ó*€KÜ-¨gÓ6×AÑAÓCÐÜ×'Ò'ØØ×ÑÓÜ�
Š
�7—=‘= Õ#¨A¯H©H¼E¿M¹MÔJÜ—[‘[ô	€Fð �Y‰Y�q—w‘wÓ€FØ×)Ñ)¨!¯'©'Ó2×:Ñ:¸2¸qÓAÕA€FØ—j‘j¨¯©¸¿¹�jÓ@×HÑHÈÈBÓOÕO€FØÒØŸ'™'¨¯©¸¿¹˜'ÓA×IÑIÈ!ÈRÓPÕPˆØ�>Š>ÐA˜>¨#¨2Ð.ÐA°·±¸aÕ0@ÓAÐAr   c               ó   € V ^8„  d   QhRR/# )r   r   zdict[str, FunctionConstraints]r   )r   s   "r   r   r   J  s   € ÷ Tñ TÐ:ñ Tr   c            "     óÈ  € \        R 04      p \        \        P                  \        P                  04      p\        \        P                  \        P                  \        P                  04      p\        \        P                  \        P                  \        P                  \
        \        \        04      pR\        R\        \        \        P                  04      R7      R\        \        \        P                  04      R7      /V R7      R\        R\        \        \        P                  04      R7      R\        \        \        P                  04      R7      R\        \        \        04      R7      /V \        3R7      R	\        R
\        V\        ^4      3R7      R\        \        \        P                  04      \        ^4      3R7      R\        \        \        P                  04      R7      R\        \        \        P                  \        P                  \        P                  04      R7      R\        VR7      R\        \        \        04      R7      R\        \        \        04      R7      R\        \        \        04      R7      R\        VR7      R\        \        \
        \        04      R7      R\        VR7      R\        VR7      /V \        3R7      /p\         '       dy   \        R
\        V\        ^4      3R7      R\        V\        ^4      3R7      R\        \        \        04      R7      R\        \        \        04      R7      /V \"        3R7      VR&   \$        '       d   \'        \(        R4      '       g   VP+                  R	4       \,        '       d›   VP/                  R\        R
\        VR7      R\        VR7      R\        \        \        04      R7      /V \0        3R7      R\        R
\        V\        ^4      3R7      R\        \        \        04      R7      /V \2        3R7      /4       R\        \        \        P                  \        P                  \        P                  04      \        ^4      3R7      /p\4        '       dð   VP/                  R\        R
\        V\        ^4      3R7      /VCV \6        3R7      R \        R!\        V\        ^4      3R7      R"\        V\        ^4      3R7      /VCV \8        3R7      R#\        R
\        V\        ^4      3R7      /VCV \:        3R7      R$\        R!\        V\        ^4      3R7      R"\        V\        ^4      3R7      /VCV \<        3R7      /4       \>        '       Edk   \        \        \        P                  \        P                  \        P                  04      \        ^4      3R7      pVP/                  R%\        R
\        V\        ^4      3R7      /VCRV/CV \@        3R7      R&\        R\        V\        ^4      3R7      R'\        V\        ^4      3R7      /VCR(VR)V/CV \B        3R7      R*\        R
\        V\        ^4      3R7      /VCRV/CV \D        3R7      R+\        R\        V\        ^4      3R7      R'\        V\        ^4      3R7      /VCR(VR)VR,\        \        \        04      R7      /CV \F        3R7      /4       R-R R.RR/R$R0R#R1R&R2R%R3R+R4R*/PI                  4        F  w  rxW„9   g   K  WH,          WG&   K  	  \$        '       Ed    \        R
\        V\        ^4      3R7      R5\        \        \        P                  \        PJ                  04      \        ^4      3R7      R6\        \        \        P                  04      R7      R\        \        \        P                  \        P                  \        P                  04      R7      R\        \        \        04      R7      R7\        \        \        04      R7      R8\        \        \        04      R7      /V \L        3R7      VR9&   V# ):ÚnpurÚ   rØ   )ÚdtypesrY   )ÚparamsÚdefault_devicesrÞ   rd   )rý   rþ   Ú
call_rulesrñ   ro   )rü   Úshape_rulesr“   r”   r•   rá   rŸ   r    r–   r˜   râ   r™   rš   rp   rr   rQ   rÖ   r0   rÌ   rÐ   Ú	freqs_cisr7   r6   ÚxqÚxkr;   r:   rC   rB   ÚkÚq_scaleÚk_scalerG   rF   Úrot_dimr9   r8   r=   r<   rE   rD   rI   rH   r­   r®   r¯   r°   rø   )'Ú	frozensetr]   Úfloat16Úbfloat16rÀ   r½   rv   r   r
   r   Úint8rf   r   r	   r   rª   r�   r~   Ú_ASCEND_QUANT_MATMUL_AVAILABLEÚhasattrr�   ÚpopÚ_ASCEND_QUANT_AVAILABLEÚupdatera   rV   Ú_ASCEND_ROPE_AVAILABLEr?   r>   rA   r@   Ú_ASCEND_RMS_ROPE_AVAILABLErK   rJ   rM   rL   ÚitemsÚuint8rµ   )	Úascend_devicesÚascend_floatsÚascend_linear_floatsÚscale_valuesÚconstraintsÚrope_tensorsÚscale_constraintÚinplace_nameÚfunctional_names	            r   Ú_build_constraintsr  J  s‘  € Ü ˜wÓ'€NÜœuŸ}™}¬e¯n©nÐ=Ó>€MÜ$¤e§m¡m´U·^±^ÄUÇ]Á]Ð%SÓTÐÜœeŸm™m¬U¯^©^¼U¿]¹]ÌEÔSVÔX[Ð\Ó]€Lð 	!Ô"5à”_¬I´u·z±z°lÓ,CÔDØœ´	¼5¿=¹=¸/Ó0JÔKðð +ô#
ð 	'Ô(;à”_¬I´u·z±z°lÓ,CÔDØœ´	¼5¿=¹=¸/Ó0JÔKØ#¤_¼IÄsÀeÓ<LÔ%Mðð
 +Ü.Ð0ô)
ð 	Ô*à”_¨MÌÐPQË
À}ÔUØœ/Ü$¤e§j¡j \Ó2ÄÈ1ÃÀôð ¤´yÄ%Ç-Á-ÀÓ7QÔ RØœÜ$¤e§m¡m´U·^±^ÄUÇ]Á]Ð%SÓTôð œ_°MÔBØœ?´)¼T¸FÓ2CÔDØ#¤_¼IÄsÀeÓ<LÔ%MØœ_´I¼s¸eÓ4DÔEØ"¤OÐ;OÔ$PØ¤¼	Ä5Ì#À,Ó8OÔ!PØœOÐ3GÔHØ ¤/Ð9MÔ"Nð!ð$ +Ü-Ð/ô)
ð#'€K÷R &Ó%Ü5Hà”_¨MÌÐPQË
À}ÔUØ”_¨MÌ	ÐRSËÀÔWØœo´YÄ¸uÓ5EÔFØ%¤¼iÌÈÓ>NÔ'Oð	ð +Ü.Ð0ô	6
ˆÐ1Ñ2÷ +Ó*¬w´yÐBU×/VÒ/VØ�‰˜Ô&çÓØ×Ñà*Ô,?àœ_°MÔBØ¤¸Ô!EØ-¬ÄiÔQTÐPUÓFVÔ/Wðð
 %3Ü :Ð<ô-ð (Ô)<àœ_°MÔPWÐXYÓPZÈ}Ô]Ø-¬ÄiÔQTÐPUÓFVÔ/Wðð %3Ü DÐFô*ðô	
ð. 	”_ÜœeŸm™m¬U¯^©^¼U¿]¹]ÐKÓLÜ" 1›˜ô
ð€L÷ ÓØ×ÑàÔ2àœ_°MÔPYÐZ[ÓP\ÈÔ_ðà&ðð %3Ü 4Ð6ô ð Ô1àœo°]ÔQZÐ[\ÓQ]ÐP_Ô`Øœo°]ÔQZÐ[\ÓQ]ÐP_Ô`ðð 'ðð
 %3Ü 3Ð5ôð )Ô*=àœ_°MÔPYÐZ[ÓP\ÈÔ_ðà&ðð %3Ü ?ÐAô+ð (Ô)<àœo°]ÔQZÐ[\ÓQ]ÐP_Ô`Øœo°]ÔQZÐ[\ÓQ]ÐP_Ô`ðð 'ðð
 %3Ü >Ð@ô*ð5#ô%	
÷N "Ô!Ü*ÜœeŸm™m¬U¯^©^¼U¿]¹]ÐKÓLÜ" 1›˜ô
Ðð 	×ÑàÔ0àœ_°MÔPYÐZ[ÓP\ÈÔ_ðà&ðð  Ð!1ñð
 %3Ü 2Ð4ôð Ô/àœ_°MÔPYÐZ[ÓP\ÈÔ_Øœ_°MÔPYÐZ[ÓP\ÈÔ_ðð 'ðð "Ð#3Ø!Ð#3ñð %3Ü 1Ð3ô
ð 'Ô(;àœ_°MÔPYÐZ[ÓP\ÈÔ_ðà&ðð  Ð!1ñð
 %3Ü =Ð?ô)ð &Ô':àœ_°MÔPYÐZ[ÓP\ÈÔ_Øœ_°MÔPYÐZ[ÓP\ÈÔ_ðð 'ðð "Ð#3Ø!Ð#3Ø!¤?¼)ÄSÀEÓ:JÔ#Kñð %3Ü <Ð>ô(ð=*ô,	
ð^ 	�|Ø˜Ø Ð"9Ø!Ð#;Ø�ZØ�kØÐ 5ØÐ!7ð	*÷ �eƒgò	*Ñ%ˆð Ö)Ø(3Õ(DˆKÓ%ñ	*÷ &Ô%Ü-@à”_Ð,@ÌwÐWXËzÈmÔ\Øœ?Ü$¤e§j¡j´%·+±+Ð%>Ó?Ü!*¨1£ ôð œ?´)¼U¿]¹]¸OÓ2LÔMØœÜ$¤e§m¡m´U·^±^ÄUÇ]Á]Ð%SÓTôð $¤_¼IÄsÀeÓ<LÔ%MØ"¤O¼9ÄcÀUÓ;KÔ$LØ¤´yÄ#ÀÓ7GÔ Hðð +Ü5Ð7ô!.
ˆÐ)Ñ*ð$ Ðr   Úascend)Úfromlist)r"   ÚmoduleÚcapabilitiesz$Huawei Ascend backend is unavailable)rø   rÚ   rÞ   rÖ   rÐ   rÌ   rñ   )r6   r7   r8   r9   r:   r;   r<   r=   )rB   rC   rD   rE   rF   rG   rH   rI   )Né    )r#  )kÚ__conditional_annotations__Ú__doc__Ú
__future__r   r]   Ú#comfy_kitchen.backends._activationsr   r¢   r   rè   r   rë   r   r£   Ú)comfy_kitchen.backends.eager.convrot_w4a4r   Ú%comfy_kitchen.backends.eager.svdquantr   Úcomfy_kitchen.constraintsr	   r
   r   r   r   Úcomfy_kitchen.registryr   Úcomfy_kitchen.tensor.int8_utilsr   r   Ú__all__Ú_ASCEND_DEVICE_AVAILABLEr  r�   r   Ú__annotations__r-   r�   rû   Úis_availabler  ÚImportErrorÚexcÚ	Exceptionr1   r  r3   r  r  Úroper6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rÀ   r	  r
  re   Ú_ROTATE_QUANT_DST_DTYPE_INT8ry   rz   rw   r²   Úiinforõ   r¼   r³   rV   ra   rf   rl   r~   r‚   r�   rª   rµ   rÌ   rÐ   rÖ   rÚ   rÞ   rñ   rø   r  ÚregisterÚ
__import__Ú__name__Úmark_unavailable)r$  s   @r   Ú<module>r;     s  øðôõ #ã å Wõõõõ ]Ý H÷õ õ ,ß Oò€ð !Ð Ø!&Ð Ø!&Ð Ø €ˆzÓ  õEð>Ûà�9‰9×!Ñ!×#Ò#ØZ‰à#'Ð Ù)0°Ð<NÓ)OÐ&Ù)0°Ð<NÓ)OÑ&ð ÷ Dð DÙ�	Ð.Ó/÷Dð Dá�	˜>Ó*÷Dð Dñ 	  	× 6Ñ 6¸
ÓCð	 ð ÷ Ið IÙ�	Ð+Ó,÷Ið Iá 	× 8Ñ 8¸-ÓHð ð
 4×ZÐZ¹À	È>Ó8ZÐ ç÷÷ ÷ ó ð ò 	õ 	€G÷ ÷÷ ÷ ó ð ò 	õ 	€Gð €u‡}�}Ø€u‡}�}Ø€u‡~�~ðÐ ð  !Ð Ø Ð Ø"Ð Ø!Ð ØÐ à"'§+¢+¨e¯k©kÓ":×">Ñ">À5Õ"IÐ õ!õ
!õ!õ#õ)!õX
ð  2÷õ";!õ|>!÷B#÷>7÷õ$õ
Xð !%Ø"Ÿ^™^ØØ Ø Ø,0ØØ$(Ø*.÷@=ðN !%Ø Ø2Ø÷1BõhT÷n Ø×ÒØÙ˜(¨WÔ5Ù'Ó)÷ð ×Ò˜h¨×(_Ð(_Ð9_Ö`øðG ô 9Ø2°3°%Ð8†MûØô >Ø7¸°uÐ=†Mûð>ús0   Á)&J5 ÂJ5 Ê5K!Ê<KËK!ËK!ËKËK!