+
    ƒ°ñj¢)  ã                   óÌ   € ^ RI t ^ RIHt ^RIHtHtHtHtHtH	t	 ^ RI
HtHtHtHt R^R] P                  3R lt ! R R]P"                  4      t ! R R	]P"                  4      tR# )
é    N)Út2i_modulateÚCaptionEmbedderÚAttentionKVCompressÚMultiHeadCrossAttentionÚT2IFinalLayerÚSizeEmbedder)ÚTimestepEmbedderÚ
PatchEmbedÚMlpÚ'get_1d_sincos_pos_embed_from_grid_torchg      ð?c           	      ój  € \         P                  ! \         P                  ! W%VR 7      W$,          ,          V,          \         P                  ! WVR 7      W,          ,          V,          RR7      w  rx\        V ^,          WuVR 7      p	\        V ^,          W…VR 7      p
\         P                  ! W©.^R7      pV# ))ÚdeviceÚdtypeÚij)Úindexing©Údim)ÚtorchÚmeshgridÚaranger   Úcat)Ú	embed_dimÚwÚhÚpe_interpolationÚ	base_sizer   r   Úgrid_hÚgrid_wÚemb_hÚemb_wÚembs   &&&&&&&     Ú=C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\pixart\pixartms.pyÚget_2d_sincos_pos_embed_torchr#      sŠ   € Ü—^’^Ü�Š�Q¨UÔ3°qµ{ÕCÐFVÕVÜ�Š�Q¨UÔ3°qµ{ÕCÐFVÕVØô�N€Fô
 4°IÀµNÀFÐafÔg€EÜ3°IÀµNÀFÐafÔg€EÜ
�)Š)�U�N¨Ô
*€CØ€Jó    c                   óD   a a€ ] tR t^t oRtRV 3R lltRR ltRtVtV ;t	# )ÚPixArtMSBlockzI
A PixArt block with adaptive layer norm zero (adaLN-Zero) conditioning.
c                óÒ  <€ \         SV `  4        Wn        VP                  VR RWšR7      V n        \        V3RVRRRVRVRVR	V	R
V
RV/VB V n        \        W3R	V	R
V
RV/VB V n        VP                  VR RWšR7      V n	        R p\        V\        W,          4      VWšVR7      V n        \        P                  ! \        P                   ! ^V4      VR,          ,          4      V n        R# )Fg�íµ ÷Æ°>)Úelementwise_affineÚepsr   r   Ú	num_headsÚqkv_biasTÚsamplingÚsr_ratioÚqk_normr   r   Ú
operationsc                  ó0   € \         P                  ! R R7      # ©Útanh)Úapproximate©ÚnnÚGELU© r$   r"   Ú<lambda>Ú(PixArtMSBlock.__init__.<locals>.<lambda>/   ó   € œbŸgšg°&Õ9r$   )Úin_featuresÚhidden_featuresÚ	act_layerr   r   r/   g      à?N)ÚsuperÚ__init__Úhidden_sizeÚ	LayerNormÚnorm1r   Úattnr   Ú
cross_attnÚnorm2r   ÚintÚmlpr5   Ú	Parameterr   ÚrandnÚscale_shift_table)Úselfr@   r*   Ú	mlp_ratioÚ	drop_pathÚ
input_sizer,   r-   r.   r   r   r/   Úblock_kwargsÚapprox_geluÚ	__class__s   &&&&&&&&&&&&, €r"   r?   ÚPixArtMSBlock.__init__!   s'  ø€ ä‰ÑÔØ&ÔØ×)Ñ)¨+È%ÐUYÐafÐ)ÓvˆŒ
Ü'Øñ
Ø#,ð
Ø7;ð
ØFNð
ØYað
àð
à#(ð
à17ð
àDNð
àR^ñ
ˆŒ	ô 2Øñ
Ø*/ð
Ø8>ð
ØKUð
ØYeñ
ˆŒð  ×)Ñ)¨+È%ÐUYÐafÐ)ÓvˆŒ
á9ˆÜØ#´S¸Õ9PÓ5QÐ]hØ°:ô
ˆŒô "$§¢¬e¯kªk¸!¸[Ó.IÈKÐ[^ÕL^Õ.^Ó!_ˆÖr$   c           	     óþ  € VP                   w  rxp	V P                  R ,          P                  VP                  VP                  R7      VP                  V^R4      ,           P                  ^^R7      w  r«rÍrïWV P                  \        V P                  V4      W«4      VR7      ,          ,           pWP                  WV4      ,           pWV P                  \        V P                  V4      WÞ4      4      ,          ,           pV# )N©r   r   r   )ÚHWéÿÿÿÿ)ÚshaperJ   Útor   r   ÚreshapeÚchunkrC   r   rB   rD   rG   rE   )rK   ÚxÚyÚtÚmaskrU   ÚkwargsÚBÚNÚCÚ	shift_msaÚ	scale_msaÚgate_msaÚ	shift_mlpÚ	scale_mlpÚgate_mlps   &&&&&&,         r"   ÚforwardÚPixArtMSBlock.forward6   s  € Ø—'‘'‰ˆˆaàJN×J`ÑJ`ÐaeÕJf×JiÑJiÐpq×pwÑpwð  AB÷  AIñ  AIÐJió  KJð  MN÷  MVñ  MVð  WXð  Z[ð  ]_ó  M`õ  K`÷  Jgñ  Jgð  hið  opð  Jgó  JqÑFˆ	˜h°9Ø˜DŸI™I¤l°4·:±:¸a³=À)Ó&WÐ\^˜IÓ_Õ_Õ`ˆØ—‘  dÓ+Õ+ˆØ˜DŸH™H¤\°$·*±*¸Q³-ÀÓ%VÓWÕWÕXˆàˆr$   )rC   rD   r@   rG   rB   rE   rJ   )	ç      @ç        NNé   FNNN©NN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r?   ri   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)rQ   Ú__classdict__s   @@r"   r&   r&      s   ù‡ € ñ÷`÷*ô r$   r&   c                   óT   a € ] tR t^Bt o RtR	V 3R lR lltR
R ltRR ltR tRt	V t
R# )ÚPixArtMSz.
Diffusion model with a Transformer backbone.
Nc                ó    <€ V ^8„  d   QhRS[ /# )é   rM   )Úfloat)Úformatrw   s   "€r"   Ú__annotate__ÚPixArtMS.__annotate__F   s   ø€ ÷ \
ñ \
ñ ñ\
r$   c                óV  € \         P                  P                  V 4       VV n        W n        W0n        V
'       d
   V^,          MTV n        W n        W`n        WÐn	        Wàn
        W@n        WPn        R p\         P                  ! \         P                  ! 4       VP                  V^V,          RVVR7      4      V n        \#        VVVRVVVR7      V n        \'        VVVVR7      V n        \+        WÄVVVVVVR7      V n        VV n        V P.                  '       d7   \1        V^,          VVVR7      V n        \1        V^,          VVVR7      V n        \6        P8                  ! ^ Wµ4       Uu. uF  pVP;                  4       NK  	  ppVf	   RRR	^R
. /p\         P<                  ! \?        V4       Uu. uFD  p\A        YFY{V,          VR,          VVR
,          9   d   \C        VR	,          4      M^VVVVR7
      NKF  	  up4      V n"        \G        WBV P                  VVVR7      V n$        R# u upi u upi )r{   c                  ó0   € \         P                  ! R R7      # r1   r4   r7   r$   r"   r8   Ú#PixArtMS.__init__.<locals>.<lambda>l   r:   r$   T)Úbiasr   r   )Ú
patch_sizeÚin_chansr   rƒ   r   r   r/   )r   r   r/   )Úin_channelsr@   Úuncond_probr=   Ú	token_numr   r   r/   Nr,   Úscale_factorÚkv_compress_layer)rL   rM   r,   r-   r.   r   r   r/   )%r5   ÚModuler?   r   Ú
pred_sigmar†   Úout_channelsr„   r*   r   Úpe_precisionr@   ÚdepthÚ
SequentialÚSiLUÚLinearÚt_blockr
   Ú
x_embedderr	   Ú
t_embedderr   Ú
y_embedderÚmicro_conditioningr   Úcsize_embedderÚar_embedderr   ÚlinspaceÚitemÚ
ModuleListÚranger&   rF   Úblocksr   Úfinal_layer)rK   rN   r„   r†   r@   r�   r*   rL   Úclass_dropout_probÚlearn_sigmarŒ   rM   Úcaption_channelsr   rŽ   ÚconfigÚmodel_max_lengthÚmicro_conditionr.   Úkv_compress_configr   r   r/   r_   rP   r[   Úis   &&&&&&&&&&&&&&&&&&&&&&&,   r"   r?   ÚPixArtMS.__init__F   s#  € ô4 	�	‰	×Ñ˜4Ô ØˆŒ
Ø$ŒØ&Ôß/9˜K¨!žO¸{ˆÔØ$ŒØ"ŒØ 0ÔØ(ÔØ&ÔØŒ
á9ˆÜ—}’}Ü�GŠG‹IØ×Ñ˜k¨1¨{­?ÀÈUÐ[aÐÓbó
ˆŒô %Ø!Ø Ø!ØØØØ!ô
ˆŒô +Ø˜u¨VÀ
ô
ˆŒô *Ø(ÐOaØ!Ð-=Ø °:ô
ˆŒð #2ˆÔØ×"×"Ð"Ü".¨{¸A­~ÀUÐSYÐfpÔ"qˆDÔÜ+¨K¸­NÀ%ÐPVÐcmÔnˆDÔô (-§~¢~°a¸Ô'JÓKÑ'J !�Q—V‘V–XÑ'Jˆ	ÐKØÒ%à˜DØ Ø# Rð"Ðô
 —m’mô ˜5”\ó%
ñ "�ô Ø°)ÐQRÅ|Ø+¨JÕ7ØDEÐI[Ð\oÕIpÔDpœÐ/°Õ?Ô@ÐvwØØØØ%÷ñ "ñ%
ó ˆŒô )Ø T×%6Ñ%6¸eÈFÐ_iô
ˆÖùò' Lùò%
s   Å&H!Æ*A
H&c           
     ód  € VP                   w  r‰r«W«,           ^,          pV P                  pVe   V P                  e(   \        VR,          V P                  ;'       g    ^ 4      p\	        V P
                  W P                  ,          W°P                  ,          V\        V^@,          4      ^@,          V P                  ,          VP                  VP                  R7      P                  ^ 4      pV P                  V4      V,           pV P                  W!P                  4      pV P                  '       d`   Ve\   VeX   VP                   ^ ,          pV P                  VV4      pV P                  VV4      pV\        P                   ! WV.^R7      ,           pV P#                  V4      pV P%                  W0P&                  4      pVeú   VP                   ^ ,          VP                   ^ ,          8w  d<   VP)                  VP                   ^ ,          VP                   ^ ,          ,          ^4      pVP+                  ^4      P+                  ^4      pVP+                  ^4      P-                  VP                  R4      ^ 8g  4      P/                  ^RVP                   R,          4      pVP1                  ^R7      P3                  4       pM5RpVP+                  ^4      P/                  ^RVP                   R,          4      pV P4                   F  pV! WVVW«33/ VB pK  	  V P7                  W4      pV P9                  WV4      pV# )a  
Original forward pass of PixArt.
x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
t: (N,) tensor of diffusion timesteps
y: (N, 1, 120, C) conditioning
ar: (N, 1): aspect ratio
cs: (N ,2) size conditioning for height/width
N)r   r   r   r   r   r   r   g      P@rV   )rW   r   rŽ   Úroundr#   r@   r„   r   r   Ú	unsqueezer”   r•   r—   r˜   r™   r   r   r“   r–   ÚtrainingÚrepeatÚsqueezeÚmasked_selectÚviewÚsumÚtolistrž   rŸ   Ú
unpatchify)rK   r[   Útimestepr\   r^   Úc_sizeÚc_arr_   r`   rb   ÚHÚWÚc_resr   Ú	pos_embedr]   ÚbsÚt0Úy_lensÚblocks   &&&&&&&,            r"   Úforward_origÚPixArtMS.forward_orig¤   s‡  € ð —W‘W‰
ˆˆaØ•˜1•ˆØ×0Ñ0ÐØÒ# t×'8Ñ'8Ò'Dä$ U¨gÕ%6¸×8IÑ8I×8NÐ8NÈQÓOÐä1Ø×ÑØ—O‘OÕ#Ø—O‘OÕ#Ø-Ü˜e b�jÓ)¨BÕ.°4·?±?ÕBØ—8‘8Ø—'‘'ô
÷ ‰)�A‹,ð 	ð �O‰O˜AÓ Õ*ˆØ�O‰O˜H§g¡gÓ.ˆà×"×"Ð"¨Ò(:¸tÒ?OØ—‘˜•ˆBØ×(Ñ(¨°Ó4ˆFØ×#Ñ# D¨"Ó-ˆDØ”E—I’I˜v˜n°!Ô4Õ4ˆAà�\‰\˜!‹_ˆØ�O‰O˜AŸ}™}Ó-ˆàÒØ�z‰z˜!�} §¡¨¥
Ô*Ø—{‘{ 1§7¡7¨1¥:°·±¸AµÕ#>ÀÓB�Ø—<‘< “?×*Ñ*¨1Ó-ˆDØ—	‘	˜!“×*Ñ*¨4¯>©>¸"Ó+=ÀÑ+BÓC×HÑHÈÈBÐPQ×PWÑPWÐXZÕP[Ó\ˆAØ—X‘X !�X“_×+Ñ+Ó-‰FàˆFØ—	‘	˜!“×!Ñ! ! R¨¯©°­Ó5ˆAØ—[”[ˆEÙ�a˜B ¨¨Ñ9°&Ñ9ŠAñ !ð ×Ñ˜QÓ"ˆØ�O‰O˜A !Ó$ˆàˆr$   c                óJ  € VP                   w  rxršV P                  '       dš   VfN   \        P                  ! V	^,          V
^,          .VP                  VP
                  R7      P                  V^4      pVfE   \        P                  ! Wš,          .VP                  VP
                  R7      P                  V^4      p\        VP                   4      ^8X  d   VP                  ^4      pV P                  WW4VR7      pV P                  '       d   VRR V P                  13,          # V# )NrT   )rµ   r¶   :NNN)rW   r—   r   Útensorr   r   r­   Úlenr«   r¿   rŒ   r†   )rK   r[   Ú	timestepsÚcontextrµ   r¶   r_   r`   rb   r·   r¸   Úouts   &&&&&&,     r"   ri   ÚPixArtMS.forwardÛ   sñ   € Ø—W‘W‰
ˆˆað ×"×"Ð"ØŠ~ÜŸš q¨¥s¨A¨a­C j¸¿¹ÈÏÉÔQ×XÑXÐYZÐ\]Ó^�àŠ|Ü—|’| Q¥S E°·±ÀÇÁÔJ×QÑQÐRSÐUVÓW�ô ˆw�}‰}Ó Ô"Ø×'Ñ'¨Ó*ˆGð ×Ñ ¨gÈ4ÐÓPˆð �?�?ˆ?Ø�qÐ+˜4×+Ñ+Ð+Ð+Õ,Ð,Øˆ
r$   c                óÂ  € V P                   pV P                  P                  ^ ,          pW P                  ,          pW0P                  ,          pW#,          VP                  ^,          8X  g   Q hVP	                  VP                  ^ ,          W#WUV3R7      p\
        P                  ! RV4      pVP	                  VP                  ^ ,          WBV,          W5,          3R7      pV# )z1
x: (N, T, patch_size**2 * C)
imgs: (N, H, W, C)
)rW   znhwpqc->nchpwq)r�   r”   r„   rW   rY   r   Úeinsum)rK   r[   r   r   ÚcÚpÚimgss   &&&&   r"   r³   ÚPixArtMS.unpatchifyò   s¯   € ð
 ×ÑˆØ�O‰O×&Ñ& qÕ)ˆØ—‘Õ ˆØ—‘Õ ˆØ�u˜Ÿ™ �
Ô"Ð"Ð"à�I‰I˜QŸW™W Q�Z¨¨q°QÐ7ˆIÓ8ˆÜ�LŠLÐ)¨1Ó-ˆØ�y‰y §¡¨¥
¨A°1­u°aµeÐ<ˆyÓ=ˆØˆr$   )r™   rž   r˜   r�   r   rŸ   r@   r†   r—   r*   r�   r„   r   rŽ   rŒ   r“   r•   r”   r–   )é    r{   é   i€  é   é   rk   gš™™™™™¹?TTrl   i   NNNéx   TFNNNN)NNNrn   )ro   rp   rq   rr   rs   r?   r¿   ri   r³   rt   ru   )rw   s   @r"   ry   ry   B   s,   ø‡ € ñ÷\
ò \
ô|5ôn÷.ð r$   ry   )r   Útorch.nnr5   rž   r   r   r   r   r   r   Ú(comfy.ldm.modules.diffusionmodules.mmditr	   r
   r   r   Úfloat32r#   r‹   r&   ry   r7   r$   r"   Ú<module>rÖ      s^   ðó Ý ÷÷ ÷ @Ó ð EHÐSUÐ^bÐjo×jwÑjwô 	ô!�B—I‘Iô !ôJ~ˆr�y‰yö ~r$   