+
    ƒ°ñjÏO  ã                   óp  € ^ RI HtHtHtHt ^ RIt^ RIHt ^ RIHu H	t
 ^ RIHt ^ RIHtHtHt ^RIHtHtHt ^RIHtHt ^RIHt ^RIHtHt ^ RIt^ RItRR	 lt RR
 lt! ! R R]PD                  4      t# ! R R]PD                  4      t$ ! R R]PD                  4      t% ! R R]PD                  4      t&R# )é    )ÚDictÚListÚOptionalÚTupleN)Ú	rearrange)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention)ÚFeedForwardÚ
PatchEmbedÚTimestepEmbedder)Úcompute_mixed_rotationÚcreate_position_matrix)Úapply_rotary_emb_qk_real)ÚAttentionPoolÚmodulatec                 ó˜   € \         P                  P                  P                  WR 7      pV^VP	                  ^4      ,           ,          pV# )©Úeps)ÚcomfyÚldmÚ
common_ditÚrms_normÚ	unsqueeze)ÚxÚscaler   Úx_normedÚx_modulateds   &&&  ÚRC:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\genmo\joint_model\asymm_models_joint.pyÚmodulated_rmsnormr    !   s=   € ä�y‰y×#Ñ#×,Ñ,¨QÐ,Ó8€HØ˜a %§/¡/°!Ó"4Õ4Õ5€KàÐó    c                 óÂ   € \         P                  ! V4      P                  ^4      p\        P                  P
                  P                  WR7      V,          pW,           pV# )é   r   )ÚtorchÚtanhr   r   r   r   r   )r   Úx_resÚgater   Ú	tanh_gater   Úoutputs   &&&&   r   Úresidual_tanh_gated_rmsnormr*   )   sM   € ä—
’
˜4Ó ×*Ñ*¨1Ó-€Iô �y‰y×#Ñ#×,Ñ,¨UÐ,Ó<¸yÕH€Hð �\€Fà€Mr!   c                   óZ   a a€ ] tR t^5t oRV3R lV 3R lllt/ 3V3R lR lltRtVtV ;t# )ÚAsymmetricAttentionc                ó�   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[RS[RS[RS[R	S[R
S[S[,          RS[S[P
                  ,          /# )é   Údim_xÚdim_yÚ	num_headsÚqkv_biasÚqk_normÚ	attn_dropÚupdate_yÚout_biasÚattend_to_paddingÚsoftmax_scaleÚdevice)ÚintÚboolÚfloatr   r$   r9   )ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú AsymmetricAttention.__annotate__6   s†   ø€ ÷ 2
ñ 2
áð2
ñ ð2
ñ ð	2
ñ
 ð2
ñ ð2
ñ ð2
ñ ð2
ñ ð2
ñ  ð2
ñ  ¡•ð2
ñ ™Ÿ™Õ&ñ2
r!   c                óT  <€ \         SV `  4        \        4       V n        Wn        W n        W0n        W,          V n        W`n        Wpn	        W�n
        W n        W,          ^ 8w  d   \        RV RV 24      hW@n        VP                  V^V,          WKVR7      V n        VP                  V^V,          WKVR7      V n        V'       g   Q hVP#                  V P                  RW¼R7      V n        VP#                  V P                  RW¼R7      V n        VP#                  V P                  RW¼R7      V n        VP#                  V P                  RW¼R7      V n        VP                  WW‹VR7      V n        V'       d   VP                  WW‹VR7      V n        R# \.        P0                  ! 4       V n        R# )r   zdim_x=z" should be divisible by num_heads=)Úbiasr9   Údtypegñhãˆµøä>)r   r9   rC   N)ÚsuperÚ__init__r	   Úcomfy_attentionr/   r0   r1   Úhead_dimr4   r5   r7   r8   Ú
ValueErrorr2   ÚLinearÚqkv_xÚqkv_yÚRMSNormÚq_norm_xÚk_norm_xÚq_norm_yÚk_norm_yÚproj_xÚnnÚIdentityÚproj_y)Úselfr/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   rC   Ú
operationsÚ	__class__s   &&&&&&&&&&&&&&€r   rE   ÚAsymmetricAttention.__init__6   s‰  ø€ ô  	‰ÑÔÜ-Ó/ˆÔØŒ
ØŒ
Ø"ŒØÕ*ˆŒØ"ŒØ ŒØ!2ÔØ*ÔØÕ Ô!ÜØ˜˜ÐAÀ)ÀÐMóð ð
 !ŒØ×&Ñ& u¨a°%­i¸hÐ]bÐ&ÓcˆŒ
à×&Ñ& u¨a°%­i¸hÐ]bÐ&ÓcˆŒ
÷ ˆˆwØ"×*Ñ*¨4¯=©=¸dÈ6Ð*Ó_ˆŒØ"×*Ñ*¨4¯=©=¸dÈ6Ð*Ó_ˆŒØ"×*Ñ*¨4¯=©=¸dÈ6Ð*Ó_ˆŒØ"×*Ñ*¨4¯=©=¸dÈ6Ð*Ó_ˆŒð !×'Ñ'¨¸8ÐZ_Ð'Ó`ˆŒ÷ ð ×Ñ˜e°ÐPUÐÓVð 	Žô —’“ð 	Žr!   c                óÆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[S[ P                  S[ P                  3,          /# )r.   r   ÚyÚscale_xÚscale_yÚreturn)r$   ÚTensorr   )r=   r>   s   "€r   r?   r@   j   s_   ø€ ÷ 7ñ 7á�<‰<ð7ñ �<‰<ð7ñ —‘ð	7ñ
 —‘ð7ñ 
‰u�|‰|™UŸ\™\Ð)Õ	*ñ7r!   c           
     ó(  € VP                  R 4      pVP                  R4      p	\        W4      p\        W$4      pV P                  V4      P                  VP                  ^ ,          VP                  ^,          ^V P
                  R4      P                  ^4      w  r«pV P                  V
4      p
V P                  V4      pV P                  V4      P                  VP                  ^ ,          VP                  ^,          ^V P
                  R4      P                  ^4      w  rÞpV P                  V4      p\        WØV	4      pV P                  V4      p\        WèV	4      p\        P                  ! WÚRRV13,          .^R7      P                  ^^4      p\        P                  ! WëRRV13,          .^R7      P                  ^^4      p\        P                  ! WüRRV13,          .^R7      P                  ^^4      pVP                  ^,          V
P                  ^,          pp????
??\!        V4      \!        V4      \!        V4      ppp\#        VVVV P
                  RV P$                  VR7      p\        P&                  ! VV3^R7      w  rV P)                  V4      p\        P*                  ! VP                  ^ ,          VVP                  R,          VP,                  VP.                  R7      pVVRRVP                  ^,          13&   V P1                  V4      pW3# )	Úrope_cosÚrope_sin:NNNN©ÚdimT)Úskip_reshapeÚpreferred_attentionÚtransformer_options©r9   rC   éÿÿÿÿ)Úgetr    rK   ÚviewÚshaper1   ÚunbindrO   rP   rJ   rM   r   rN   r$   ÚcatÚ	transposer   r
   rF   Útensor_splitrQ   Úzerosr9   rC   rT   )rU   r   rZ   r[   r\   Úcrop_yrf   Úrope_rotationr`   ra   Úq_yÚk_yÚv_yÚq_xÚk_xÚv_xÚqÚkÚvÚseq_xÚseq_yÚxyÚos   &&&&&&&,               r   ÚforwardÚAsymmetricAttention.forwardj   s�  € ð !×$Ñ$ ZÓ0ˆØ ×$Ñ$ ZÓ0ˆä˜aÓ)ˆô ˜aÓ)ˆØŸ
™
 1›×*Ñ*¨1¯7©7°1­:°q·w±w¸qµzÀ1ÀdÇnÁnÐVXÓY×`Ñ`ÐabÓc‰ˆ�#à�m‰m˜CÓ ˆØ�m‰m˜CÓ ˆð Ÿ
™
 1›×*Ñ*¨1¯7©7°1­:°q·w±w¸qµzÀ1ÀdÇnÁnÐVXÓY×`Ñ`ÐabÓc‰ˆ�#Ø�m‰m˜CÓ ˆÜ& s°hÓ?ˆØ�m‰m˜CÓ ˆÜ& s°hÓ?ˆä�IŠI�s  7 F 7 
�OÐ,°!Ô4×>Ñ>¸qÀ!ÓDˆÜ�IŠI�s  7 F 7 
�OÐ,°!Ô4×>Ñ>¸qÀ!ÓDˆÜ�IŠI�s  7 F 7 
�OÐ,°!Ô4×>Ñ>¸qÀ!ÓDˆà—y‘y •| S§Y¡Y¨q¥\ˆuˆØ��c˜3  SÜ*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆ1ˆÜ  Ø!"Ø!" D§N¡NÀÐ[_×[oÑ[oð  FYôZˆô ×!Ò! " u h°AÔ6‰ˆØ�K‰K˜‹NˆÜ�KŠK˜Ÿ™ �
 E¨1¯7©7°2­;¸q¿x¹xÈqÏwÉwÔWˆØˆˆ!ˆ[ˆa�g‰g�a�jˆ[ˆ.Ñà�K‰K˜‹Nˆð ˆtˆr!   )r7   r4   rF   r/   r0   rG   rN   rP   r1   rQ   rT   rM   rO   r2   rJ   rK   r8   r5   )é   TFg        TTFNNNN)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rE   r€   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rW   r>   s   @@r   r,   r,   5   s$   ù‡ € ÷2
õ 2
ðv ÷7÷ 7ò 7r!   r,   c                   ó~   a a€ ] tR t^¤t oRRRRRRRRR	RR
R/V3R lV 3R lllt/ 3V3R lR lltR tR tRtVt	V ;t
# )ÚAsymmetricJointBlockÚmlp_ratio_xç       @Úmlp_ratio_yç      @r5   Tr9   NrC   rV   c                óh   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RS[RS[RS[S[P
                  ,          /# )r.   Úhidden_size_xÚhidden_size_yr1   r�   r�   r5   r9   )r:   r<   r;   r   r$   r9   )r=   r>   s   "€r   r?   Ú!AsymmetricJointBlock.__annotate__¥   sZ   ø€ ÷ <ñ <áð<ñ ð<ñ ð	<ñ ð<ñ ð<ñ ð<ñ ™Ÿ™Õ&ñ<r!   c               óB  <€ \         SV `  4        W`n        Wn        W n        V	P                  V^V,          WxR7      V n        V P                  '       d"   V	P                  V^V,          WxR7      V n        MV	P                  WWxR7      V n        \        VV3RVRVRVRVRV	/V
B V n	        \        W,          4      pV\        R
4      8X  g   Q h\        VVRRVVV	R	7      V n        V P                  '       d+   \        W%,          4      p\        VVRRVVV	R	7      V n        R# R# )é   rg   r1   r5   r9   rC   rV   é   N)Úin_featuresÚhidden_sizeÚmultiple_ofÚffn_dim_multiplierr9   rC   rV   i 0  )rD   rE   r5   r’   r“   rI   Úmod_xÚmod_yr,   Úattnr:   r   Úmlp_xÚmlp_y)rU   r’   r“   r1   r�   r�   r5   r9   rC   rV   Úblock_kwargsÚmlp_hidden_dim_xÚmlp_hidden_dim_yrW   s   &&&&$$$$$$,  €r   rE   ÚAsymmetricJointBlock.__init__¥   sR  ø€ ô 	‰ÑÔØ ŒØ*ÔØ*ÔØ×&Ñ& }°a¸-Õ6GÐPVÐ&ÓdˆŒ
Ø�=�=ˆ=Ø#×*Ñ*¨=¸!¸mÕ:KÐTZÐ*ÓhˆD�Jà#×*Ñ*¨=ÐPVÐ*ÓdˆDŒJô (ØØñ	
ð  ð	
ð ð		
ð
 ð	
ð ð	
ð "ð	
ð ñ	
ˆŒ	ô ˜}Õ:Ó;ÐØ¤3 x£=Ô0Ð0Ð0Ü Ø%Ø(ØØ#ØØØ!ô
ˆŒ
ð �=�=ˆ=Ü" =Õ#>Ó?ÐÜ$Ø)Ø,ØØ#'ØØØ%ôˆDŽJñ r!   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r.   r   ÚcrZ   ©r$   r^   )r=   r>   s   "€r   r?   r”   ã   s5   ø€ ÷ 4ñ 4á�<‰<ð4ñ �<‰<ð4ñ �<‰<ñ	4r!   c           	     óN  € VP                  ^4      p\        P                  ! V4      pV P                  V4      pVP	                  ^^R7      w  r‰r«V P                  V4      pV P                  '       d   VP	                  ^^R7      w  rÞppMTpV P                  ! VV3RVRVRV/VB w  ppVP                  ^4      V8X  g   Q h\        VVV	4      pV P                  '       d   \        VVX4      pV P                  WV4      pV P                  '       d   V P                  VXX4      pW3# )al  Forward pass of a block.

Args:
    x: (B, N, dim) tensor of visual tokens
    c: (B, dim) tensor of conditioned features
    y: (B, L, dim) tensor of text tokens
    num_frames: Number of frames in the video. N = num_frames * num_spatial_tokens

Returns:
    x: (B, N, dim) tensor of visual tokens after block
    y: (B, L, dim) tensor of text tokens after block
rb   r[   r\   rf   )ÚsizeÚFÚsilurœ   Úchunkr�   r5   rž   r*   Ú
ff_block_xÚ
ff_block_y)rU   r   r¦   rZ   rf   Úattn_kwargsÚNrœ   Úscale_msa_xÚ
gate_msa_xÚscale_mlp_xÚ
gate_mlp_xr�   Úscale_msa_yÚ
gate_msa_yÚscale_mlp_yÚ
gate_mlp_yÚx_attnÚy_attns   &&&&&,             r   r€   ÚAsymmetricJointBlock.forwardã   s*  € ð( �F‰F�1‹Iˆä�FŠF�1‹IˆØ—
‘
˜1“ˆØ;@¿;¹;ÀqÈa¸;Ó;PÑ8ˆ à—
‘
˜1“ˆØ�=�=ˆ=Ø?D¿{¹{È1ÐRS¸{Ó?TÑ<ˆK [±*àˆKð ŸšØØñ
ð  ð
ð  ð	
ð
 !4ð
ð ñ
‰ˆ�ð �{‰{˜1‹~ Ô"Ð"Ð"Ü'¨¨6°:Ó>ˆØ�=�=ˆ=Ü+¨A¨v°zÓBˆAð �O‰O˜A¨JÓ7ˆØ�=�=ˆ=Ø—‘  ;°
Ó;ˆAàˆtˆr!   c                óV   € \        W4      pV P                  V4      p\        WV4      pV# ©N)r    rŸ   r*   )rU   r   r[   Úgate_xÚx_modr&   s   &&&&  r   r­   ÚAsymmetricJointBlock.ff_block_x  ó+   € Ü! !Ó-ˆØ—
‘
˜5Ó!ˆÜ'¨°&Ó9ˆØˆr!   c                óV   € \        W4      pV P                  V4      p\        WV4      pV# r½   )r    r    r*   )rU   rZ   r\   Úgate_yÚy_modÚy_ress   &&&&  r   r®   ÚAsymmetricJointBlock.ff_block_y  rÁ   r!   )rž   r’   r“   rŸ   r    rœ   r�   r5   )rƒ   r„   r…   r†   rE   r€   r­   r®   r‡   rˆ   r‰   rŠ   s   @@r   rŒ   rŒ   ¤   sk   ù‡ € ð<ð !ð<ð !ð<ð ð<ð *.ð<ð ð<ð ÷<õ <ðF ÷4ò 4òl÷ò r!   rŒ   c                   óL   a a€ ] tR tRt oRtRV3R lV 3R llltR tRtVtV ;t	# )Ú
FinalLayeri&  z
The final layer of DiT.
c                óD   <€ V ^8„  d   QhRS[ S[P                  ,          /# )r.   r9   )r   r$   r9   )r=   r>   s   "€r   r?   ÚFinalLayer.__annotate__+  s    ø€ ÷ 
ñ 
ñ
 ™Ÿ™Õ&ñ
r!   c                óä   <€ \         SV `  4        VP                  VR RWER7      V n        VP	                  V^V,          WER7      V n        VP	                  WV,          V,          WER7      V n        R# )Fç�íµ ÷Æ°>)Úelementwise_affiner   r9   rC   rg   N)rD   rE   Ú	LayerNormÚ
norm_finalrI   ÚmodÚlinear)rU   r™   Ú
patch_sizeÚout_channelsr9   rC   rV   rW   s   &&&&&&&€r   rE   ÚFinalLayer.__init__+  su   ø€ ô 	‰ÑÔØ$×.Ñ.Ø¨E°tÀFð /ó 
ˆŒð ×$Ñ$ [°!°kµ/È&Ð$Ó^ˆŒØ ×'Ñ'Ø jÕ0°<Õ?Èð (ó 
ˆŽr!   c                óÒ   € \         P                  ! V4      pV P                  V4      P                  ^^R7      w  r4\	        V P                  V4      W44      pV P                  V4      pV# )r.   rb   )rª   r«   rÐ   r¬   r   rÏ   rÑ   )rU   r   r¦   Úshiftr   s   &&&  r   r€   ÚFinalLayer.forward=  sU   € Ü�FŠF�1‹IˆØ—x‘x “{×(Ñ(¨°Ð(Ó2‰ˆÜ�T—_‘_ QÓ'¨Ó6ˆØ�K‰K˜‹NˆØˆr!   )rÑ   rÐ   rÏ   )NNN)
rƒ   r„   r…   r†   Ú__doc__rE   r€   r‡   rˆ   r‰   rŠ   s   @@r   rÈ   rÈ   &  s   ù‡ € ñ÷
õ 
÷$ò r!   rÈ   c            .       óè   a a€ ] tR tRt oRtR^R^RRRRR^0R	^R
RRRRRRRRRRRRRRRRRRRRRRRRRRRR RR!RR"R/V3R# lV 3R$ llltV3R% lR& ltV3R' lR( ltRRRRR/ 3V3R) lR* lltR+t	Vt
V ;t# ),ÚAsymmDiTJointiE  z[
Diffusion model with a Transformer backbone.

Ingests text embeddings instead of a label.
rÒ   Úin_channelsr’   i€  r“   Údepthr1   r�   rŽ   r�   r�   Úuse_t5FÚt5_feat_dimi   Út5_token_lengthr—   Úlearn_sigmaTÚpatch_embed_biasÚtimestep_mlp_biasr7   Útimestep_scaleNÚuse_extended_posencÚposenc_preserve_areaÚ
rope_thetag     ˆÃ@Úimage_modelr9   rC   rV   c                ó�   <€ V ^8„  d   QhRS[ RS[RS[RS[ RS[ RS[ RS[S[,          RS[ R	S[ R
S[RS[S[P
                  ,          /# )r.   rÝ   rÞ   rß   rá   râ   r7   rã   rä   rå   ræ   r9   )r;   r:   r   r<   r$   r9   )r=   r>   s   "€r   r?   ÚAsymmDiTJoint.__annotate__L  s•   ø€ ÷ q
ñ q
ñ ðq
ñ ðq
ñ ðq
ñ ðq
ñ   ð!q
ñ"  ð#q
ñ$ !¡�ð%q
ñ& "ð'q
ñ( #ð)q
ñ* ð+q
ñ. ™Ÿ™Õ&ñ/q
r!   c               óÎ  <€ \         SV `  4        VV n        WÀn        W n        V'       d
   V^,          MTV n        Wn        W`n        W0n        W@n	        W6,          V n
        Wðn        VV n        VV n        W�n        W°n        W n        VV n        \%        VVVVVVVR7      V n        \)        W>VVVVR7      V n        V P                  '       d1   \-        V
^VVVVR7      V n        VP1                  W¤RVVR7      V n        \4        P6                  ! \8        P:                  ! ^V P                  V P                  ^,          VVR7      4      V n        V P                  '       d   Q h. p\?        V4       F<  pVV^,
          8  p\A        VVV3RVRVR	VR
VRVRVRV/VB pVPC                  V4       K>  	  \4        PD                  ! V4      V n#        \I        W1V P
                  VVVR7      V n%        R# )r.   )rÒ   Úin_chansÚ	embed_dimrB   rC   r9   rV   )rB   rã   rC   r9   rV   )r1   Ú
output_dimrC   r9   rV   T)rB   rC   r9   ©rC   r9   r�   r�   r5   r7   r9   rC   rV   )rC   r9   rV   N)&rD   rE   rC   rà   rÛ   rÓ   rÒ   r1   r’   r“   rG   r7   rä   rå   rÝ   rß   rÞ   ræ   r   Ú
x_embedderr   Ú
t_embedderr   Út5_y_embedderrI   Út5_yprojrR   Ú	Parameterr$   ÚemptyÚpos_frequenciesÚrangerŒ   ÚappendÚ
ModuleListÚblocksrÈ   Úfinal_layer)rU   rÒ   rÛ   r’   r“   rÜ   r1   r�   r�   rÝ   rÞ   rß   rà   rá   râ   r7   rã   rä   rå   ræ   rç   r9   rC   rV   r¡   rù   Úbr5   ÚblockrW   s   &$$$$$$$$$$$$$$$$$$$$$$$,    €r   rE   ÚAsymmDiTJoint.__init__L  s"  ø€ ô8 	‰ÑÔàˆŒ
Ø&ÔØ&Ôß/:˜K¨!žOÀˆÔØ$ŒØ"ŒØ*ÔØ*ÔàÕ&ð 	Œð "3ÔØ#6ˆÔ Ø$8ˆÔ!ØŒØ.ÔØ&Ôàð 	Œô %Ø!Ø Ø#Ø!ØØØ!ô
ˆŒô +ØÀ.ÐX]Ðflð  zDô
ˆŒð �;�;ˆ;ä!.Ø q°]È%ÐX^Ðkuô"ˆDÔð
 '×-Ñ-Ø°¸UÈ6ð .ó ˆDŒMô
  "Ÿ|š|Ü�KŠK˜˜4Ÿ>™>¨4¯=©=¸AÕ+=ÀUÐSYÔZó 
ˆÔð ×)×)Ð)Ð)Ð)ð ˆÜ�u–ˆAà˜5 1�9‘}ˆHÜ(ØØØñð (ð	ð
 (ðð "ðð #4ðð ðð ðð &ðð ñˆEð �M‰M˜%Ö ñ# ô$ —m’m FÓ+ˆŒä%Ø t×'8Ñ'8ÀÈfÐakô
ˆÖr!   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r.   r   r]   r§   )r=   r>   s   "€r   r?   ré   ¿  s#   ø€ ÷ "ñ "™Ÿ™ð "©%¯,©,ñ "r!   c                ó$   € V P                  V4      # )zŒ
Args:
    x: (B, C=12, T, H, W) tensor of visual tokens

Returns:
    x: (B, C=3072, N) tensor of visual tokens with positional embedding.
)rï   )rU   r   s   &&r   Úembed_xÚAsymmDiTJoint.embed_x¿  s   € ð �‰˜qÓ!Ð!r!   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r.   r   ÚsigmaÚt5_featÚt5_maskr§   )r=   r>   s   "€r   r?   ré   É  sC   ø€ ÷  0ñ  0á�<‰<ð 0ñ �|‰|ð 0ñ —‘ð	 0ñ
 —‘ñ 0r!   c                óÖ  € VP                   RR w  rVpW`P                  ,          WpP                  ,          r˜V P                  V4      pVP                  ^8X  g   Q hW`P                  ,          WpP                  ,          r˜WX,          V	,          p
VP	                  ^4      V
8X  g   Q h\        WXW‘P                  \        P                  R7      p\        \        P                  P                  V P                  VP                  VP                  R7      VR7      w  rÍV P                  ^V,
          VP                  R7      pV P!                  W44      pWï,           pV P#                  V4      pVVVWÍ3# )z*Prepare input and conditioning embeddings.N)ÚpHÚpWr9   rC   rî   )ÚfreqsÚpos)Ú	out_dtypeéýÿÿÿ)rk   rÒ   r   Úndimr©   r   r9   r$   Úfloat32r   r   ÚopsÚcast_torõ   rC   rð   rñ   rò   )rU   r   r  r  r  ÚTÚHÚWr  r  r°   r
  r`   ra   Úc_tÚ	t5_y_poolr¦   Úy_feats   &&&&&             r   ÚprepareÚAsymmDiTJoint.prepareÉ  s"  € ð —'‘'˜"˜#�,‰ˆˆaØ—o‘oÕ% q¯O©OÕ';ˆBØ�L‰L˜‹OˆØ�v‰v˜Œ{Ðˆ{à—o‘oÕ% q¯O©OÕ';ˆBØ�F�R�KˆØ�v‰v�a‹y˜AŒ~Ðˆ~Ü$Ø˜§H¡H´E·M±Mô
ˆô 4Ü—)‘)×#Ñ# D×$8Ñ$8ÀÇÁÐPQ×PXÑPXÐ#ÓYÐ_bô
Ñˆð �o‰o˜a %�i°1·7±7ˆoÓ;ˆà×&Ñ& wÓ8ˆ	à�Oˆà—‘˜wÓ'ˆà�!�V˜XÐ/Ð/r!   c                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[S[ P                  3,          RS[ P                  RS[ P                  /# )r.   r   ÚtimestepÚcontextÚattention_maskÚpacked_indicesr`   ra   )r$   r^   r   r   Ústr)r=   r>   s   "€r   r?   ré   ë  sŠ   ø€ ÷ Jñ Já�<‰<ðJñ —,‘,ðJñ ‘e—l‘lÕ#ð	Jñ
 ™UŸ\™\Õ*ðJñ ™S¡%§,¡,Ð.Õ/ðJñ —,‘,ðJñ —,‘,ñJr!   c                ój  a€ V
P                  R / 4      pTpTpTp VP                  w  pppppV P                  WWÞ4      w  ppr×p?VP                  R/ 4      p\        V P                  4       F\  w  poRV3V9   d=   V3R lpVRV3,          ! RVRVRVRVRVR	VR
V
/RV/4      pVR,          pVR,          pKK  S! VVVVVVV
R7      w  rK^  	  ?V P                  VV4      p\        VRVVV P                  ,          VV P                  ,          V P                  V P                  V P                  R7      pV) # )Úpatches_replaceÚditÚdouble_blockc           
      óž   <€ / pS! V R ,          V R,          V R,          V R,          V R,          V R,          V R,          R7      w  VR &   VR&   V# )ÚimgÚvecÚtxtr`   ra   Ú
num_tokensrf   ©r`   ra   rq   rf   © )ÚargsÚoutrü   s   & €r   Ú
block_wrapÚ)AsymmDiTJoint.forward.<locals>.block_wrap  s\   ø€ Ø�CÙ-2Ø48¸µKØ48¸µKØ48¸µKØ=AÀ*Õ=MØ=AÀ*Õ=MØ;?ÀÕ;MØHLÐMbÕHcô.6Ñ*�C˜‘J  E¡
ð �Jr!   r$  r&  r%  r`   ra   r'  rf   Úoriginal_blockr(  z.B (T hp wp) (p1 p2 c) -> B c T (hp p1) (wp p2))r  ÚhpÚwpÚp1Úp2r¦   )	ri   rk   r  Ú	enumeraterù   rú   r   rÒ   rÓ   )rU   r   r  r  r  r'  r  r`   ra   Úcontrolrf   Úkwargsr   r  Úy_maskr  ÚBÚ_r  r  r  r¦   Úblocks_replaceÚir,  r+  rü   s   &&&&&&&&&&&,              @r   r€   ÚAsymmDiTJoint.forwardë  s   ø€ ð .×1Ñ1Ð2CÀRÓHˆØˆØˆØˆð	ð Ÿ™‰ˆˆ1ˆa��Aà+/¯<©<Ø�fó,
Ñ(ˆˆ1ˆf ð à(×,Ñ,¨U°BÓ7ˆÜ! $§+¡+Ö.‰HˆAˆuØ Ð" nÔ4õð % n°aÐ%8Ö9¸5À!ÀUÈFÐTYÐ[\Ð^hÐjrÐt~ð  AIð  KWð  Ycð  ezð  |Oð  ;Pð  Scð  eoð  Rpó  q�Ø˜U��Ø˜•J’á!ØØØØ%Ø%Ø%Ø(;ô‘	�‘6ñ% /ð6 à×Ñ˜Q Ó"ˆÜØØ<ØØ�D—O‘OÕ#Ø�D—O‘OÕ#Ø�‰Ø�‰Ø×Ñô	
ˆð ˆrˆ	r!   )r7   rù   rC   rú   rG   r’   r“   rÛ   rà   r1   rÓ   rÒ   rõ   rå   ræ   rÞ   rß   rñ   rò   rð   rä   rÝ   rï   )rƒ   r„   r…   r†   rØ   rE   r   r  r€   r‡   rˆ   r‰   rŠ   s   @@r   rÚ   rÚ   E  sN  ù‡ € ñðq
ð ðq
ð ð	q
ð
 ðq
ð ðq
ð ðq
ð ðq
ð ðq
ð ðq
ð ðq
ð  ðq
ð  #ðq
ð ðq
ð "&ðq
ð  #'ð!q
ð" #(ð#q
ð$ +/ð%q
ð& %*ð'q
ð( &+ð)q
ð* $ð+q
ð, ð-q
ð. *.ð/q
ð0 ð1q
ð2 ÷3q
õ q
÷f"ð "÷ 0ð  0ðP Ø26Ø!%Ø!%Ø¨"÷J÷ Jò Jr!   rÚ   )rÌ   )'Útypingr   r   r   r   r$   Útorch.nnrR   Útorch.nn.functionalÚ
functionalrª   Úeinopsr   Úcomfy.ldm.modules.attentionr   r	   r
   Úlayersr   r   r   Ú
rope_mixedr   r   Útemporal_roper   Úutilsr   r   Úcomfy.ldm.common_ditr   Ú	comfy.opsr    r*   ÚModuler,   rŒ   rÈ   rÚ   r)  r!   r   Ú<module>rI     s•   ð÷ /Ó .ã Ý ß Ð Ý ç eÑ e÷ñ ÷õ 4÷ó
 Û ôô
ôl˜"Ÿ)™)ô lô^˜2Ÿ9™9ô ôD�—‘ô ô>p�B—I‘Iö pr!   