+
    ƒ°ñjs0  ã                   ó  € ^ RI t ^ RIHt ^ RIHu Ht ^ RIt^ RIt^ RI	H
t
Ht ^ RIHt ^ RIHtHtHt ^ RIHt ^RIHtHtHtHtHtHt  ! R R]P6                  4      t ! R	 R
]P6                  4      t ! R R]P6                  4      tR# )é    N)Ú
apply_ropeÚrope)ÚFeedForwardSwiGLU)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attention)ÚTimestepEmbedder)Ú
FinalLayerÚPatchTokenEmbedderÚPiTBlockÚPixelTokenEmbedderÚapply_adaln_Úprecompute_freqs_cis_2dc                   óJ   a a€ ] tR t^t oRtRV 3R lltRR/ 3R ltRtVtV ;t	# )ÚMMDiTJointAttentionzãJoint MMDiT attention with separate Q/K/V/proj for image and text streams.

RoPE is applied to each stream before concatenation so each stream uses its own
2D/1D positional encoding. Concat order is [text, image] (text first).
Nc                ó~  <€ \         SV `  4        \        4       V n        W,          ^ 8X  g   Q hW n        W,          V n        VP                  W^,          W4VR7      V n        VP                  W^,          W4VR7      V n        VP                  V P
                  RWER7      V n
        VP                  V P
                  RWER7      V n        VP                  V P
                  RWER7      V n        VP                  V P
                  RWER7      V n        VP                  WWER7      V n        VP                  WWER7      V n        R# )r   ©ÚbiasÚdtypeÚdeviceç�íµ ÷Æ°>©Úepsr   r   ©r   r   N)ÚsuperÚ__init__r   Úcomfy_attentionÚ	num_headsÚhead_dimÚLinearÚqkv_xÚqkv_yÚRMSNormÚq_norm_xÚk_norm_xÚq_norm_yÚk_norm_yÚproj_xÚproj_y)ÚselfÚdimr   Úqkv_biasr   r   Ú
operationsÚ	__class__s   &&&&&&&€Ú<C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\pixeldit\model.pyr   ÚMMDiTJointAttention.__init__   s  ø€ Ü‰ÑÔÜ-Ó/ˆÔØ� !Ô#Ð#Ð#Ø"ŒØÕ(ˆŒà×&Ñ& s°!­G¸(ÐX^Ð&Ó_ˆŒ
Ø×&Ñ& s°!­G¸(ÐX^Ð&Ó_ˆŒ
à"×*Ñ*¨4¯=©=¸dÈ%Ð*Ó_ˆŒØ"×*Ñ*¨4¯=©=¸dÈ%Ð*Ó_ˆŒØ"×*Ñ*¨4¯=©=¸dÈ%Ð*Ó_ˆŒØ"×*Ñ*¨4¯=©=¸dÈ%Ð*Ó_ˆŒà ×'Ñ'¨¸Ð'ÓMˆŒØ ×'Ñ'¨¸Ð'ÓMˆŽó    c                ó¼  € VP                   w  rxp	VP                   w  ršp	V P                  pV P                  pV P                  V4      P	                  Wx^W¼4      P                  ^^ ^^^4      pVP                  ^ 4      w  rïpV P                  V4      pV P                  V4      pV P                  V4      P	                  Wz^W¼4      P                  ^^ ^^^4      pVP                  ^ 4      w  pppV P                  V4      pV P                  V4      p\        WïVR,          4      w  rïVe   \        VVVR,          4      w  pp\        P                  ! VV.^R7      p\        P                  ! VV.^R7      p\        P                  ! VV.^R7      p????????\        V4      \        V4      \        V4      ppp\!        VVVVVRRW`P"                  R7	      pVRRRV
1R3,          P%                  ^^4      P	                  WzW¼,          4      pVRRV
R1R3,          P%                  ^^4      P	                  WxW¼,          4      pV P'                  V4      V P)                  V4      3# )é   N©r+   T)ÚmaskÚskip_reshapeÚskip_output_reshapeÚtransformer_optionsÚpreferred_attentionºNNN)NN)Úshaper   r   r!   ÚreshapeÚpermuteÚunbindr$   r%   r"   r&   r'   r   ÚtorchÚcatr   r   r   Ú	transposer(   r)   )r*   ÚxÚyÚpos_imgÚpos_txtÚ	attn_maskr8   ÚBÚNxÚ_ÚNyÚHÚDr!   ÚqxÚkxÚvxr"   ÚqyÚkyÚvyÚq_jointÚk_jointÚv_jointÚ	out_jointÚout_yÚout_xs   &&&&&&&                    r/   ÚforwardÚMMDiTJointAttention.forward.   s5  € Ø—7‘7‰ˆˆqØ—7‘7‰ˆˆqØ�N‰NˆØ�M‰Mˆà—
‘
˜1“×%Ñ% a¨Q°Ó5×=Ñ=¸aÀÀAÀqÈ!ÓLˆØ—\‘\ !“_‰
ˆ�Ø�]‰]˜2ÓˆØ�]‰]˜2Óˆà—
‘
˜1“×%Ñ% a¨Q°Ó5×=Ñ=¸aÀÀAÀqÈ!ÓLˆØ—\‘\ !“_‰
ˆˆB�Ø�]‰]˜2ÓˆØ�]‰]˜2Óˆä˜B G¨JÕ$7Ó8‰ˆØÒÜ  B¨°
Õ(;Ó<‰FˆB�ä—)’)˜R ˜H¨!Ô,ˆÜ—)’)˜R ˜H¨!Ô,ˆÜ—)’)˜R ˜H¨!Ô,ˆà�5˜"˜b " b¨"¨bÜ$<¸WÓ$EÔG_Ð`gÓGhô  kCð  DKó  kL˜'�ˆÜ'Ø�W˜g qØ¨À4Ø 3×I]ÑI]ô
ˆ	ð ˜!˜Q    Q˜,Õ'×1Ñ1°!°QÓ7×?Ñ?ÀÀqÅuÓMˆØ˜!˜Q ¡ Q˜,Õ'×1Ñ1°!°QÓ7×?Ñ?ÀÀqÅuÓMˆà�{‰{˜5Ó! 4§;¡;¨uÓ#5Ð5Ð5r1   )r   r   r%   r'   r   r(   r)   r$   r&   r!   r"   )é   FNNN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   rY   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r.   Ú__classdict__s   @@r/   r   r      s$   ù‡ € ñ÷
Nð$ .2¸TÐWY÷ #6ô #6r1   r   c                   óF   a a€ ] tR t^Tt oRV 3R lltRR/ 3R ltRtVtV ;t# )ÚMMDiTBlockT2INc           
     óZ  <€ \         SV `  4        VP                  VR WER7      V n        VP                  VR WER7      V n        \        WRWEVR7      V n        VP                  VR WER7      V n        VP                  VR WER7      V n        \        W,          4      p\        W^WEVR7      V n        \        W^WEVR7      V n        \        P                  ! VP                  V^V,          RWER7      4      V n        \        P                  ! VP                  V^V,          RWER7      4      V n        R# )r   r   F)r   r,   r   r   r-   )Úmultiple_ofr   r   r-   Tr   N)r   r   r#   Únorm_x1Únorm_y1r   ÚattnÚnorm_x2Únorm_y2Úintr   Úmlp_xÚmlp_yÚnnÚ
Sequentialr    ÚadaLN_modulation_imgÚadaLN_modulation_txt)	r*   Úhidden_sizeÚgroupsÚ	mlp_ratior   r   r-   Úmlp_hidden_dimr.   s	   &&&&&&& €r/   r   ÚMMDiTBlockT2I.__init__U   s;  ø€ Ü‰ÑÔØ!×)Ñ)¨+¸4ÀuÐ)Ó\ˆŒØ!×)Ñ)¨+¸4ÀuÐ)Ó\ˆŒÜ'¨ÐPUÐ]bð  Iô  JˆŒ	Ø!×)Ñ)¨+¸4ÀuÐ)Ó\ˆŒØ!×)Ñ)¨+¸4ÀuÐ)Ó\ˆŒÜ˜[Õ4Ó5ˆÜ& {ÐPQÐY^ð  {Eô  FˆŒ
Ü& {ÐPQÐY^ð  {Eô  FˆŒ
Ü$&§M¢M°*×2CÑ2CÀKÐQRÐU`ÕQ`ÐgkÐsxÐ2Có  3Ió  %JˆÔ!Ü$&§M¢M°*×2CÑ2CÀKÐQRÐU`ÕQ`ÐgkÐsxÐ2Có  3Ió  %JˆÖ!r1   c                óž  € V P                  V4      P                  ^RR7      w  r‰r«rÍV P                  V4      P                  ^RR7      w  rïpppp\        V P	                  V4      W‰4      p\        V P                  V4      Wï4      pV P                  VVWEWgR7      w  pp\        P                  ! WV4      p\        P                  ! VVV4      p\        P                  ! WV P                  \        V P                  V4      W¼4      4      4      p\        P                  ! VVV P                  \        V P                  V4      VV4      4      4      pW3# )é   r4   ©r8   éÿÿÿÿ)rt   Úchunkru   r   rj   rk   rl   r?   Úaddcmulrp   rm   rq   rn   )r*   rB   rC   ÚcrD   rE   rF   r8   Úshift_msa_xÚscale_msa_xÚ
gate_msa_xÚshift_mlp_xÚscale_mlp_xÚ
gate_mlp_xÚshift_msa_yÚscale_msa_yÚ
gate_msa_yÚshift_mlp_yÚscale_mlp_yÚ
gate_mlp_yÚx_normÚy_normÚattn_xÚattn_ys   &&&&&&&&                r/   rY   ÚMMDiTBlockT2I.forwardb   s,  € ØUY×UnÑUnÐopÓUq×UwÑUwÐxyð  @BÐUwó  VCÑRˆ *¸;ØUY×UnÑUnÐopÓUq×UwÑUwÐxyð  @BÐUwó  VCÑRˆ *¨k¸;È
ä˜dŸl™l¨1›o¨{ÓHˆÜ˜dŸl™l¨1›o¨{ÓHˆØŸ™ 6¨6°7ÀY˜Óx‰ˆ�Ü�MŠM˜!¨Ó0ˆÜ�MŠM˜!˜Z¨Ó0ˆä�MŠM˜!¨¯©´LÀÇÁÈaÃÐR]Ó4kÓ)lÓmˆÜ�MŠM˜!˜Z¨¯©´LÀÇÁÈaÃÐR]Ð_jÓ4kÓ)lÓmˆØˆtˆr1   )	rt   ru   rl   rp   rq   rj   rm   rk   rn   )g      @NNN)	r\   r]   r^   r_   r   rY   ra   rb   rc   rd   s   @@r/   rg   rg   T   s   ù‡ € ÷Jð 15ÀÐZ\÷ ô r1   rg   c                   ór   a a€ ] tR t^qt oRtRV 3R lltR tR tRR/ 3R ltR t	R t
RR/ 3R	 ltR
tVtV ;t# )Ú
PixDiT_T2Ia�  PixelDiT T2I model. Hardcoded for the released 1024px Stage-3 checkpoint
(also runs at 512px when fed the appropriate latent size and flow_shift).

Forward:
  x:        [B, 3, H, W] pixel-space input (no VAE)
  timesteps:[B] in [0, 1000] (ComfyUI flow sampling convention)
  context:  [B, Ltxt, 2304] Gemma-2-2b-it hidden states (chi_prompt prepended)
Returns flow-matching velocity [B, 3, H, W].
Nc                ó  <€ \         SV `  4        Wðn        Wn        Wn        W0n        W n        Wpn        W€n        W�n	        W@n
        WPn        W`n        W n        W°n        WÀn        WÐn        \#        V P                  V P                  VVVR 7      V n        \'        V P                  V P                  ^,          ,          V P
                  RVVVR7      V n        \+        V P
                  VVV^
R7      V n        \'        V P                  V P
                  RRVVVR7      V n        \0        P2                  ! \4        P6                  ! ^V P                  V P
                  VVR7      4      V n        \0        P:                  ! \=        V P                  4       Uu. uF'  p\?        V P
                  V P                  VVVR 7      NK)  	  up4      V n         \0        P:                  ! \=        V P                  4       Uu. uFT  p\C        V P                  V P
                  V P                  V P                  V P                  V P                  VVVVR7
      NKV  	  up4      V n"        \G        V P                  V P                  VVVR 7      V n$        R# u upi u upi ))r   r   r-   T)r   r   r   r-   )r   r   r-   Ú
max_period)r   Úuse_normr   r   r-   r   )Ú
patch_sizer   Úattn_hidden_sizeÚattn_num_headsr   r   r-   Ú
mlp_chunksN)%r   r   r   Úin_channelsÚout_channelsrv   Ú
num_groupsÚpatch_depthÚpixel_depthr˜   Úpixel_hidden_sizeÚpixel_attn_hidden_sizeÚpixel_num_groupsÚtxt_embed_dimÚtxt_max_lengthÚuse_text_ropeÚtext_rope_thetar   Úpixel_embedderr   Ú
s_embedderr	   Ú
t_embedderÚ
y_embedderrr   Ú	Parameterr?   ÚemptyÚy_pos_embeddingÚ
ModuleListÚrangerg   Úpatch_blocksr   Úpixel_blocksr
   Úfinal_layer)r*   rœ   rž   rv   r¡   r¢   r£   rŸ   r    r˜   r¤   r¥   r¦   r§   Úimage_modelr   r   r-   Úpixel_mlp_chunksrI   r.   s   &&&&&&&&&&&&&&&&&&& €r/   r   ÚPixDiT_T2I.__init__{   sb  ø€ ô* 	‰ÑÔØŒ
Ø&ÔØ'ÔØ&ÔØ$ŒØ&ÔØ&ÔØ$ŒØ!2ÔØ&<Ô#Ø 0ÔØ*ÔØ,ÔØ*ÔØ.Ôä0°×1AÑ1AÀ4×CYÑCYÐafÐouð  CMô  NˆÔÜ,¨T×-=Ñ-=ÀÇÁÐSTÕ@TÕ-TÐVZ×VfÑVfÐmqÐy~ð  HNð  [eô  fˆŒÜ*¨4×+;Ñ+;À5ÐQWÐdnÐ{}Ô~ˆŒÜ,¨T×-?Ñ-?À×AQÑAQÐX\ÐgkÐsxð  BHð  U_ô  `ˆŒÜ!Ÿ|š|¬E¯KªK¸¸4×;NÑ;NÐPT×P`ÑP`ÐhmÐv|Ô,}Ó~ˆÔäŸMšMô ˜4×+Ñ+Ô,ó+
ñ -�ô ˜$×*Ñ*¨D¯O©OØ %¨fÀ÷Má,ñ+
ó ˆÔô
 ŸMšMô ˜4×+Ñ+Ô,ó+
ñ -�ô Ø×&Ñ&Ø× Ñ ØŸ?™?ØŸ/™/Ø!%×!<Ñ!<Ø#×4Ñ4Ø F°zØ+÷	ñ -ñ+
ó ˆÔô & d×&<Ñ&<¸d×>OÑ>OÐW\Ðekð  yCô  DˆÖùò'+
ùò
+
s   Æ-I?Ç.AJc                óZ   € \        V P                  V P                  ,          W3R VRV/VB # )r   r   )r   rv   rž   )r*   ÚheightÚwidthr   r   Ú	rope_optss   &&&&&,r/   Ú_fetch_patch_posÚPixDiT_T2I._fetch_patch_pos¼   sC   € Ü& t×'7Ñ'7¸4¿?¹?Õ'JÈFñ  DÐbhð  DÐpuð  Dð  zCñ  Dð  	Dr1   c                ó  € \        \        P                  ! V\        P                  VR 7      P	                  ^R4      V P
                  V P                  ,          V P                  4      P                  ^ 4      P                  VR7      # )r   )r   r~   )
r   r?   ÚarangeÚfloat32r<   rv   rž   r§   ÚsqueezeÚto)r*   Úlengthr   r   s   &&&&r/   Ú_fetch_text_posÚPixDiT_T2I._fetch_text_pos¿   s¡   € Ü”E—L’L ¬u¯}©}ÀVÔL×TÑTÐUVÐXZÓ[Ð]a×]mÑ]mÐqu÷  rAñ  rAõ  ^Að  CG÷  CWñ  CWó  X÷  `ñ  `ð  abó  c÷  fñ  fð  mrð  fó  sð  	sr1   c                ó  € \         P                  P                  P                  V P                  V \         P                  P                  \         P                  P                  P                  V4      4      P                  ! WW4V3/ VB # ©N)	ÚcomfyÚpatcher_extensionÚWrapperExecutorÚnew_class_executorÚ_forwardÚget_all_wrappersÚ
WrappersMPÚDIFFUSION_MODELÚexecute)r*   rB   Ú	timestepsÚcontextÚattention_maskr8   Úkwargss   &&&&&&,r/   rY   ÚPixDiT_T2I.forwardÂ   s{   € Ü×&Ñ&×6Ñ6×IÑIØ�M‰M˜4¤×!8Ñ!8×!IÑ!IÌ%×JaÑJa×JlÑJl×J|ÑJ|ð  Ró  "Só
ç
‰'ðà Ð9LñXàPVñXð 	Xr1   c                ó   € V# )zYHook for subclasses to inject per-block state into the patch stream (e.g. PiD's LQ gate).© )r*   ÚsÚirÓ   s   &&&,r/   Ú_pre_patch_blockÚPixDiT_T2I._pre_patch_blockÇ   s   € àˆr1   c                ó   € V# rÆ   rÖ   )r*   r×   rÓ   s   &&,r/   Ú_pre_pixel_blocksÚPixDiT_T2I._pre_pixel_blocksË   s   € Øˆr1   c                ó  € VP                   ^,          VP                   ^,          r‡\        P                  P                  P	                  WP
                  V P
                  34      pVP                   w  ršr¼W°P
                  ,          pWÀP
                  ,          pWÞ,          pV P                  ! WÞVP                  VP                  3/ VP                  R4      ;'       g    / B p\        P                  ! WP
                  V P
                  R7      P                  ^^4      pV P                  VP                  R	4      VP                  4      P                  V	R	V P                  4      pVe   VP!                  4       ^8w  d   \#        R4      h\%        VP                   ^,          V P&                  4      pVRRV1R3,          pV P)                  V4      P                  V	VV P                  4      pVV P*                  RRV1R3,          P-                  V4      ,           p\        P.                  ! V4      pV P0                  '       d(   V P3                  VVP                  VP                  4      MRpV P5                  V4      p\7        V P8                  4       F,  w  ppV P:                  ! VV3/ VB pV! VVVVVRVR7      w  ppK.  	  \        P.                  ! VV,           4      pV P<                  ! V3/ VB pVP                  WŸ,          V P                  4      pV P?                  WP
                  R7      pV P@                   F  pV! VVW¼V P
                  RVR7      pK  	  V PC                  V4      pV PD                  pV P
                  V P
                  ,          pVP                  WŸVV4      PG                  ^ ^^^4      PI                  V	VV,          V4      p\        PJ                  ! VW¼3V P
                  V P
                  R7      pVRRRV1RV13,          # )
é   Úrope_options)Úkernel_sizeÚstrideNz@PixDiT_T2I requires context (text embeddings) of shape [B, L, D]r:   r}   )r˜   )r5   r8   r~   )&r;   rÇ   ÚldmÚ
common_ditÚpad_to_patch_sizer˜   r»   r   r   ÚgetÚFÚunfoldrA   rª   Úviewrv   r+   Ú
ValueErrorÚminr¥   r«   r®   rÁ   Úsilur¦   rÃ   r©   Ú	enumerater±   rÙ   rÜ   r¨   r²   r³   r�   r=   r<   Úfold) r*   rB   rÐ   rÑ   rÒ   r8   rÓ   ÚH_origÚW_origrG   rI   rK   ÚWÚHsÚWsÚLrD   Ú	x_patchesÚt_embÚLtxtrC   Úy_embÚ	conditionrE   r×   rØ   ÚblkÚs_condÚx_pixelsÚC_outÚP2Úouts    &&&&&&,                         r/   rË   ÚPixDiT_T2I._forwardÎ   sJ  € ØŸ™ � Q§W¡W¨Q¥Z�Ü�I‰I× Ñ ×2Ñ2°1·±ÈÏÉÐ6XÓYˆØ—W‘W‰
ˆˆaØ—/‘/Õ!ˆØ—/‘/Õ!ˆØ�Gˆà×'Ò'¨°·±¸!¿'¹'ÑuÐFY×F]ÑF]Ð^lÓFm×FsÐFsÐqsÑuˆÜ—H’H˜Q¯O©OÀDÇOÁOÔT×^Ñ^Ð_`ÐbcÓdˆ	à—‘ 	§¡¨rÓ 2°A·G±GÓ<×AÑAÀ!ÀRÈ×IYÑIYÓZˆàŠ?˜gŸk™k›m¨qÔ0ÜÐ_Ó`Ð`Ü�7—=‘= Õ# T×%8Ñ%8Ó9ˆØ�A�u˜�u˜a�KÕ ˆØ—‘ Ó"×'Ñ'¨¨4°×1AÑ1AÓBˆØ˜×,Ñ,¨Q°°°°q¨[Õ9×<Ñ<¸UÓCÕCˆä—F’F˜5“Mˆ	ØCG×CU×CUÐCU�$×&Ñ& t¨Q¯X©X°q·w±wÔ?Ð[_ˆà�O‰O˜IÓ&ˆÜ × 1Ñ 1Ö2‰FˆAˆsØ×%Ò% a¨Ñ5¨fÑ5ˆAÙ˜1˜e Y°¸À$Ð\oÔp‰HˆAŠuñ 3ô �FŠF�5˜1•9Óˆà×"Ò" 1Ñ/¨Ñ/ˆØ—‘˜�˜t×/Ñ/Ó0ˆØ×&Ñ& q·_±_Ð&ÓEˆØ×$Ô$ˆCÙ˜8 V¨Q°4·?±?ÈÐcvÔwŠHñ %ð ×#Ñ# HÓ-ˆØ×!Ñ!ˆØ�_‰_˜tŸ™Õ.ˆØ—=‘=  r¨5Ó1×9Ñ9¸!¸QÀÀ1ÓE×MÑMÈaÐQVÐY[ÕQ[Ð]^Ó_ˆÜ�fŠf�X ˜v°4·?±?È4Ï?É?Ô[ˆØ�1�a˜˜&˜ ' 6 'Ð)Õ*Ð*r1   )r   r³   rv   rœ   rž   r�   r±   rŸ   r˜   r¢   r²   r    r¨   r¡   r£   r©   rª   r§   r¤   r¥   r¦   r«   r®   )r3   é   i   é   i€  r  é   rß   r  i 	  i,  Tg     ˆÃ@NNNNrß   )r\   r]   r^   r_   r`   r   r»   rÃ   rY   rÙ   rÜ   rË   ra   rb   rc   rd   s   @@r/   r”   r”   q   sK   ù‡ € ñ÷?DòBDòsð -1ÀÐ[]ô Xò
òð .2À$Ð\^÷ (+ô (+r1   r”   )r?   Útorch.nnrr   Útorch.nn.functionalÚ
functionalrç   Úcomfy.ldm.common_ditrÇ   Úcomfy.patcher_extensionÚcomfy.ldm.flux.mathr   r   Úcomfy.ldm.hidream.modelr   Úcomfy.ldm.modules.attentionr   r   r   Ú(comfy.ldm.modules.diffusionmodules.mmditr	   Úmodulesr
   r   r   r   r   r   ÚModuler   rg   r”   rÖ   r1   r/   Ú<module>r     sc   ðÛ Ý ß Ð ã Û ß 0Ý 5ß eÑ eÝ E÷÷ ô;6˜"Ÿ)™)ô ;6ô|�B—I‘Iô ô:E+�—‘ö E+r1   