
    %	&h%^                        d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
c mZ ddlZddlm
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'  ejP                  e)      Z*dZ+dZ,	 	 	 d2deejZ                  eejZ                     df   dee.   deejZ                     deejZ                  e.f   fdZ/ G d de
j`                        Z1 G d de
j`                        Z2 G d de$      Z3 G d de      Z4 G d  d!e
j`                        Z5 G d" d#e%      Z6 G d$ d%e#      Z7 G d& d'e"      Z8 G d( d)ee      Z9 G d* d+e      Z: G d, d-e       Z; G d. d/e!      Z< G d0 d1e      Z=y)3zPyTorch Mixtral model.    )partial)ListOptionalTupleUnionN)nn   )ACT2FN)DynamicCache)FlashAttentionKwargs)MoeCausalLMOutputWithPastMoeModelOutputWithPast)Unpack)
LossKwargslogging   )	MistralAttentionMistralForCausalLMMistralForQuestionAnswering MistralForSequenceClassificationMistralForTokenClassificationMistralModelMistralPreTrainedModelMistralRMSNormMistralRotaryEmbedding   )MixtralConfigzmistralai/Mixtral-8x7B-v0.1r   gate_logitsnum_expertsattention_maskreturnc                    | t        | t              syt        | t              rC| d   j                  }t        j                  | D cg c]  }|j                  |       c}d      }t        j                  j                  j                  d      }t        j                  ||d      \  }}	t        j                  j                  j                  |	|      }
|>t        j                  |
j                         d      }t        j                  |d      }n|j                  \  }}|j                  d   ||z  z  }|dddddddf   j                  |||||f      j                  d||      j                        }t        j                   |
j                         |z  d      t        j                   |d      z  }|ddddddf   j                  ||||f      j                  d|      j                  |      }t        j                   ||z  d      t        j                   |d      z  }t        j                   ||j#                  d      z        }||z  S c c}w )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   dim)
isinstancetupledevicetorchcattor   
functionalsoftmaxtopkone_hotmeanfloatshapeexpandreshapesum	unsqueeze)r   r   top_kr    compute_device
layer_gateconcatenated_gate_logitsrouting_weights_selected_expertsexpert_masktokens_per_expertrouter_prob_per_expert
batch_sizesequence_lengthnum_hidden_layersexpert_attention_mask router_per_expert_attention_maskoverall_losss                      /var/www/pru.catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/mixtral/modular_mixtral.pyload_balancing_loss_funcrH   >   s9   : *[%"@+u%$Q..#(99^i-jPZjmmN.K-jpq#r hh))112JPR1SO**_eDA((%%--.>LK!JJ{'8'8':B "'O!C&4&:&:#
O4::1=*B^_ 4AtT12V&
OUKXYWR,R	 	 "IIk&7&7&9<Q&QWXY\a\e\e!q]
 
 4At+,V&
O[QRWR%R	 	) "'?=]+]cd!ehmhqhq,!i
 "
 99.1G1Q1QRS1TTUL+%%[ .ks   Ic                   *     e Zd Zdef fdZd Z xZS )MixtralBlockSparseTop2MLPconfigc                    t         |           |j                  | _        |j                  | _        t        j                  | j
                  | j                  d      | _        t        j                  | j                  | j
                  d      | _	        t        j                  | j
                  | j                  d      | _
        t        |j                     | _        y NF)bias)super__init__intermediate_sizeffn_dimhidden_size
hidden_dimr   Linearw1w2w3r
   
hidden_actact_fnselfrK   	__class__s     rG   rP   z"MixtralBlockSparseTop2MLP.__init__   s    // ,,))DOOT\\F))DLL$//F))DOOT\\FV../    c                     | j                  | j                  |            | j                  |      z  }| j                  |      }|S N)rZ   rV   rX   rW   )r\   hidden_statescurrent_hidden_statess      rG   forwardz!MixtralBlockSparseTop2MLP.forward   s>     $DGGM,B CdggmF\ \ $(= >$$r^   )__name__
__module____qualname__r   rP   rc   __classcell__r]   s   @rG   rJ   rJ      s    	0} 	0%r^   rJ   c                   Z     e Zd ZdZ fdZdej                  dej                  fdZ xZS )MixtralSparseMoeBlocka  
    This implementation is
    strictly equivalent to standard MoE with full capacity (no
    dropped tokens). It's faster since it formulates MoE operations
    in terms of block-sparse operations to accommodate imbalanced
    assignments of tokens to experts, whereas standard MoE either
    (1) drop tokens at the cost of reduced performance or (2) set
    capacity factor to number of experts and thus waste computation
    and memory on padding.
    c                    t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  | j                  d      | _        t        j                  t        | j                        D cg c]  }t        |       c}      | _        |j"                  | _        y c c}w rM   )rO   rP   rS   rT   rQ   rR   num_local_expertsr   num_experts_per_tokr7   r   rU   gate
ModuleListrangerJ   expertsrouter_jitter_noisejitter_noise)r\   rK   r<   r]   s      rG   rP   zMixtralSparseMoeBlock.__init__   s     ,,//!33//
 IIdoot/?/?eL	}}QVW[WgWgQh%iA&?&G%ij #66 &js   +Cra   r!   c                 f   |j                   \  }}}| j                  rQ| j                  dkD  rB|t        j                  |      j                  d| j                  z
  d| j                  z         z  }|j                  d|      }| j                  |      }t        j                  |dt        j                        }t        j                  || j                  d      \  }}||j                  dd      z  }|j                  |j                        }t        j                   ||z  |f|j                  |j"                  	      }t        j$                  j&                  j)                  || j*                  
      j-                  ddd      }	t/        | j*                        D ]  }
| j0                  |
   }t        j2                  |	|
         \  }}|d|f   j5                  d|      } ||      |||df   z  }|j7                  d||j                  |j                                |j5                  |||      }||fS ) r   g      ?r%   r   )r$   dtyper#   T)r$   keepdim)rv   r(   )num_classesr   N)r2   trainingrs   r)   
empty_likeuniform_viewrn   Fr-   r1   r.   r7   r5   r+   rv   zerosr(   r   r,   r/   r   permuterp   rq   wherer4   
index_add_)r\   ra   rA   rB   rT   router_logitsr;   r=   final_hidden_statesr>   
expert_idxexpert_layeridxtop_xcurrent_staterb   s                   rG   rc   zMixtralSparseMoeBlock.forward   s   2?2E2E/
OZ==T..2U--m<EEcDL]L]F]_beievev_vwwM%**2z:		-0))MqL,1JJ

XZ,[))?..2t.DD),,]-@-@A#kk/):6m>Q>QZgZnZn
 hh))112BPTP`P`1aiijkmnpqr   0 01 	dJ<<
3L[%<=JC
 *$+6>>r:NM$0$?/RWY\^bRbBc$c!  **1e5J5M5MmNaNa5bc	d 299*oWab"M11r^   )	rd   re   rf   __doc__rP   r)   Tensorrc   rg   rh   s   @rG   rj   rj      s(    	7&2U\\ &2ell &2r^   rj   c                       e Zd Zy)MixtralRMSNormNrd   re   rf    r^   rG   r   r          r^   r   c                       e Zd Zy)MixtralAttentionNr   r   r^   rG   r   r      r   r^   r   c                       e Zd Zdedef fdZ	 	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
ej                        dee   d	ee   d
ee   deej                     dee
ej                  ej                  f      dee   de
ej                  ee
ej                  ej                  f      f   fdZ xZS )MixtralDecoderLayerrK   	layer_idxc                    t         |           |j                  | _        t        ||      | _        t        |      | _        t        |j                  |j                        | _	        t        |j                  |j                        | _
        y )N)eps)rO   rP   rS   r   	self_attnrj   block_sparse_moer   rms_norm_epsinput_layernormpost_attention_layernormr\   rK   r   r]   s      rG   rP   zMixtralDecoderLayer.__init__   sm    !--)&)< 5f =-f.@.@fFYFYZ(6v7I7IvObOb(c%r^   ra   r    position_idspast_key_valueoutput_attentionsoutput_router_logits	use_cachecache_positionposition_embeddingskwargsr!   c
                     |}| j                  |      } | j                  d||	||||||d|
\  }}||z   }|}| j                  |      }| j                  |      \  }}||z   }|f}|r||fz  }|r||fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_router_logits (`bool`, *optional*):
                Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
                should not be returned during inference.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        )ra   r   r    r   r   r   r   r   r   )r   r   r   r   )r\   ra   r    r   r   r   r   r   r   r   r   residualself_attn_weightsr   outputss                  rG   rc   zMixtralDecoderLayer.forward   s    F !,,]; ,:4>> 
,
' 3)%)/)
,
 
,
(( !=0 !55mD'+'<'<]'K$} =0 ")++G''Gr^   )NNNFFFNN)rd   re   rf   r   intrP   r)   r   r   
LongTensorr   boolr   r   FloatTensorrc   rg   rh   s   @rG   r   r      s5   d} d d 26378<,1/4$)59KOC||C !.C u//0	C
 !u||!45C $D>C 'tnC D>C !!1!12C &eELL%,,,F&GHC -.C 
u  (51B1BEDUDU1U+V"WW	XCr^   r   c                       e Zd Zy)MixtralRotaryEmbeddingNr   r   r^   rG   r   r   ?  r   r^   r   c                       e Zd ZdZy)MixtralPreTrainedModelFN)rd   re   rf   _supports_static_cacher   r^   rG   r   r   C  s    "r^   r   c                   2    e Zd Zdef fdZ	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee	ej                        deej                     dee   d	ee   d
ee   dee   deej                     dee   defdZ xZS )MixtralModelrK   c           	          t         |   |       t        j                  t	        |j
                        D cg c]  }t        ||       c}      | _        y c c}w r`   )rO   rP   r   ro   rp   rC   r   layersr   s      rG   rP   zMixtralModel.__init__H  sD     mmEJ6KcKcEde	 3e
es   A	input_idsr    r   past_key_valuesinputs_embedsr   r   output_hidden_statesr   r   flash_attn_kwargsr!   c                 8   ||n| j                   j                  }|	|	n| j                   j                  }	||n| j                   j                  }||n| j                   j                  }|d u |d uz  rt        d      | j                  r%| j                  r|rt        j                  d       d}|r|
t               }|| j                  |      }|
F||j                         nd}t        j                  |||j                  d   z   |j                         }
||
j#                  d      }| j%                  |||
||      }|}| j'                  ||      }|rdnd }|rdnd }|	rdnd }| j(                  D ]  }|r||fz  }| j                  r;| j                  r/| j+                  t-        |j.                  fi |||||||	||
|
      }n ||f|||||	||
|d|}|d   }|r	||d   fz  }|	sw||d	   fz  } | j1                  |      }|r||fz  }t3        |||||
      S )Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr   r   )r(   r   )r    r   r   r   r   r   r   r   r%   )last_hidden_stater   ra   
attentionsr   )rK   r   r   r   r   
ValueErrorgradient_checkpointingry   loggerwarning_oncer   embed_tokensget_seq_lengthr)   aranger2   r(   r6   _update_causal_mask
rotary_embr   _gradient_checkpointing_funcr   __call__normr   )r\   r   r    r   r   r   r   r   r   r   r   r   past_seen_tokenscausal_maskra   r   all_hidden_statesall_self_attnsall_router_logitsdecoder_layerlayer_outputss                        rG   rc   zMixtralModel.forwardN  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 %9$D $++JjJj 	 "+!6IDKK<Q<Q	-t";<YZZ&&4==##p "	0*nO  --i8M!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6L..M>?L]
 & #oom\J #7BD0d"6BD![[ %	:M#!m%55!**t}} $ A AM22H6GH! #%("'! !.!!#.!-#2&7)='#1(;! (! *!,M =#3"55#!mB&7%99!K%	:N 		-0  -!11%+++%+
 	
r^   )
NNNNNNNNNN)rd   re   rf   r   rP   r   r)   r   r   r   r   r   r   r   r   rc   rg   rh   s   @rG   r   r   G  s   
} 
 151537=A59$(,0/3/359p
E,,-p
 !.p
 u//0	p

 "$u'8'8"9:p
   1 12p
 D>p
 $D>p
 'tnp
 'tnp
 !!1!12p
 $$89p
 
 p
r^   r   c                       e Zd Zy)KwargsForCausalLMNr   r   r^   rG   r   r     s    r^   r   c                   v    e Zd ZdgZ fdZ	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee	ej                        deej                     deej                     d	ee   d
ee   dee   dee   deej                     deeej                  f   dee   defdZ xZS )MixtralForCausalLMzlm_head.weightc                     t         |   |       t        |      | _        |j                  | _        |j
                  | _        |j                  | _        y r`   )rO   rP   r   modelrouter_aux_loss_coefrl   r   rm   r[   s     rG   rP   zMixtralForCausalLM.__init__  sF     !&)
$*$?$?!!33#)#=#= r^   r   r    r   r   r   labelsr   r   r   r   r   logits_to_keepr   r!   c                    ||n| j                   j                  }|
|
n| j                   j                  }
|	|	n| j                   j                  }	 | j                  d||||||||	|
|d
|}|j
                  }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |}d}|
rYt        |j                  | j                  | j                  |      }|+|| j                   |j#                  |j$                        z  z  }t'        ||||j(                  |j*                  |j,                  |j                        S )a4  
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

            logits_to_keep (`int` or `torch.Tensor`, *optional*):
                If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
                `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
                token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
                If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
                This is useful when using packed tensor format (single dimension for batch and sequence length).

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MixtralForCausalLM

        >>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)
r   r    r   r   r   r   r   r   r   r   )lossaux_losslogitsr   ra   r   r   r   )rK   r   r   r   r   r   r&   r   slicelm_headloss_function
vocab_sizerH   r   r   rm   r   r+   r(   r   r   ra   r   )r\   r   r    r   r   r   r   r   r   r   r   r   r   r   r   ra   slice_indicesr   r   r   s                       rG   rc   zMixtralForCausalLM.forward  s   ` 2C1N-TXT_T_TqTq$8$D $++JjJj 	
 %9$D $++JjJj 	
 +5$** +
)%+'/!5!5)+
 +
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD/%%  ((	H !11HKK4LLL(#33!//))!//
 	
r^   )NNNNNNNNNNNr   )rd   re   rf   _tied_weights_keysrP   r   r)   r   r   r   r   r   r   r   r   r   r   rc   rg   rh   s   @rG   r   r     sI   *+> 151537=A59-1$(,0/3/35934d
E,,-d
 !.d
 u//0	d

 "$u'8'8"9:d
   1 12d
 ))*d
 D>d
 $D>d
 'tnd
 'tnd
 !!1!12d
 c5<</0d
 *+d
 
#d
r^   r   c                       e Zd Zy) MixtralForSequenceClassificationNr   r   r^   rG   r   r   5  r   r^   r   c                       e Zd Zy)MixtralForTokenClassificationNr   r   r^   rG   r   r   9  r   r^   r   c                       e Zd Zy)MixtralForQuestionAnsweringNr   r   r^   rG   r   r   =  r   r^   r   )Nr   N)>r   	functoolsr   typingr   r   r   r   r)   torch.nn.functionalr   r,   r}   torch.utils.checkpointactivationsr
   cache_utilsr   modeling_flash_attention_utilsr   modeling_outputsr   r   processing_utilsr   utilsr   r   mistral.modeling_mistralr   r   r   r   r   r   r   r   r   configuration_mixtralr   
get_loggerrd   r   _CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCr   r   rH   ModulerJ   rj   r   r   r   r   r   r   r   r   r   r   r   r   r^   rG   <module>r      s  (   / /      ! ' B '
 
 
 1 
		H	%3 !
 "&
-1	O&u||U5<<%8$>?O&#O& U\\*	O&
 5<<O&d%		 %$A2BII A2H	^ 		' 	N")) Nb	3 	#3 #w
< w
t ?,j >n
+ n
b	'G 		$A 		"= 	r^   