
    %	&h:3                        d Z ddlmZmZmZ ddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZ  ej<                  e      Z dZ!dZ" G d dejF                        Z$d"dZ% G d dejF                        Z& G d de      Z' G d de      Z( G d dee(      Z) G d de      Z* G d d e      Z+g d!Z,y)#zPyTorch Phi-3 model.    )CallableOptionalTupleN)nn   )ACT2FN)Cache)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONS)Unpack)logging   )MistralDecoderLayerMistralForCausalLM MistralForSequenceClassificationMistralForTokenClassificationMistralPreTrainedModeleager_attention_forwardrotate_half   )
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Phi3MLPc                 *   t         |           || _        t        j                  |j
                  d|j                  z  d      | _        t        j                  |j                  |j
                  d      | _        t        |j                     | _        y )Nr   Fbias)super__init__configr   Linearhidden_sizeintermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fn)selfr   	__class__s     {/var/www/pru.catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/phi3/modular_phi3.pyr   zPhi3MLP.__init__1   sp    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56    hidden_statesreturnc                     | j                  |      }|j                  dd      \  }}|| j                  |      z  }| j                  |      S )Nr   dim)r#   chunkr&   r$   )r'   r+   	up_statesgates       r)   forwardzPhi3MLP.forward9   sL    %%m4	#//!/4i 2 24 88	~~i((r*   )__name__
__module____qualname__r   torchFloatTensorr4   __classcell__r(   s   @r)   r   r   0   s'    7)U%6%6 )5;L;L )r*   r   c                 `   |j                  |      }|j                  |      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }
}	t        j                  ||z  t	        |      |z  z   |gd      }t        j                  |	|z  t	        |	      |z  z   |
gd      }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r.   .Nr/   )	unsqueezeshaper8   catr   )qkcossinposition_idsunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds                r)   apply_rotary_pos_embrM   B   s    ( --
&C
--
&C2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6Eii%#++e*<s*BCVLRTUGii%#++e*<s*BCVLRTUGGr*   c                   >    e Zd ZdZddedee   f fdZ	 	 ddej                  de
ej                  ej                  f   deej                     dee   d	eej                     d
ee   de
ej                  eej                     ee
ej                        f   fdZ xZS )Phi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperr   	layer_idxc                 |   t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        |j                  | _        | j                  dz  | _
        |j                  | _        d| _        |j                  | j                  z  d|j                  | j                  z  z  z   }t        j                  |j                  | j                  z  |j
                  d      | _        t        j                  |j
                  |d      | _        y )Nhead_dimg      Tr   Fr   )r   r   r   rP   getattrr!   num_attention_headsrR   num_key_value_headsnum_key_value_groupsscalingattention_dropout	is_causalr   r    o_projqkv_proj)r'   r   rP   op_sizer(   s       r)   r   zPhi3Attention.__init__e   s    "
F4F4F&JdJd4de$*$>$>&B\B\$\!#)#=#= }}d*!'!9!9,,t}}<qFD^D^aeananDn?ooii : :T]] JFL^L^ejk		&"4"4gEJr*   r+   position_embeddingsattention_maskpast_key_valuecache_positionkwargsr,   c           
         |j                   d d }g |d| j                  }| j                  |      }	| j                  j                  | j                  z  }
|	dd |
f   }|	d|
|
| j
                  | j                  z  z   f   }|	d|
| j
                  | j                  z  z   d f   }|j                  |      j                  dd      }|j                  |      j                  dd      }|j                  |      j                  dd      }|\  }}t        ||||      \  }}|'|||d}|j                  ||| j                  |      \  }}t        }| j                  j                  dk7  r^| j                  j                  dk(  r(|j                  dd	      rt        j                  d
       nt         | j                  j                     } || ||||f| j"                  sdn| j$                  | j&                  t)        | j                  dd       d|\  }} |j*                  g |d j-                         }| j/                  |      }||fS )Nr.   .r   r   )rC   rB   r`   eagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.g        sliding_window)dropoutrW   rf   )r>   rR   r[   r   rT   rU   view	transposerM   updaterP   r   _attn_implementationgetloggerwarning_oncer   trainingrX   rW   rS   reshape
contiguousrZ   )r'   r+   r]   r^   r_   r`   ra   input_shapehidden_shapeqkv	query_posquery_states
key_statesvalue_statesrB   rC   cache_kwargsattention_interfaceattn_outputattn_weightss                       r)   r4   zPhi3Attention.forwardt   s[    $))#2.88b8$--8mmM*KK33dmmC	3

?+i)d6N6NQUQ^Q^6^*^^^_
3	D,D,Dt}},T T VVW#((6@@AF__\2<<QB
#((6@@AF&S#7jRUWZ#[ j%#&snUL'5'<'<ZW[WeWegs't$J(?;;++w6{{//69fjjI\^c>d##L
 '>dkk>^>^&_#$7
%
  $}}C$2H2HLL"4;;0@$G
%
 
%
!\ *k));;;;FFHkk+.L((r*   )N)NN)r5   r6   r7   __doc__r   r   intr   r8   Tensorr   r	   
LongTensorr   r
   r4   r:   r;   s   @r)   rO   rO   b   s    GKz Khsm K( +/596)||6) #5<<#=>6) !.	6)
 !6) !!1!126) -.6) 
u||Xell3XeELL>Q5RR	S6)r*   rO   c                   p    e Zd Zdedef fdZ	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
   dee   d	ee   d
eej                     deeej                  ej                  f      dee   deej                  eeej                  ej                  f      f   fdZ xZS )Phi3DecoderLayerr   rP   c                    t         |   ||       || _        t        ||      | _        t        |      | _        t        j                  |j                        | _
        t        j                  |j                        | _        y )N)r   rP   )r   r   r   rO   	self_attnr   mlpr   Dropoutresid_pdropresid_attn_dropoutresid_mlp_dropout)r'   r   rP   r(   s      r)   r   zPhi3DecoderLayer.__init__   s`    +&f	J6?"$**V-?-?"@!#F,>,>!?r*   r+   r^   rD   r_   re   	use_cacher`   r]   ra   r,   c	                    |}
| j                  |      } | j                  d||||||||d|	\  }}|
| j                  |      z   }|}
| j                  |      }| j	                  |      }|
| j                  |      z   }|f}|r||fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
                Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
                `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
            past_key_value (`Cache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        )r+   r^   rD   r_   re   r   r`   r]    )input_layernormr   r   post_attention_layernormr   r   )r'   r+   r^   rD   r_   re   r   r`   r]   ra   residualself_attn_weightsoutputss                r)   r4   zPhi3DecoderLayer.forward   s    D !,,]; ,:4>> 
,
')%)/) 3
,
 
,
(( !4#:#:=#II 55mD/ 4#9#9-#HH ")++Gr*   )NNNFFNN)r5   r6   r7   r   r~   r   r8   r   r   r   r	   boolr   r   r
   r9   r4   r:   r;   s   @r)   r   r      s   @z @c @ 2637*.,1$)59KO=||= !.= u//0	=
 != $D>= D>= !!1!12= &eELL%,,,F&GH= -.= 
u  (51B1BEDUDU1U+V"WW	X=r*   r   c                       e Zd ZdZy)Phi3PreTrainedModelz0.0.5N)r5   r6   r7   _versionr   r*   r)   r   r      s    Hr*   r   c                   "    e Zd Z	 	 	 	 	 	 	 ddZy)Phi3ForCausalLMNc	                    |r_| j                   j                  rI|j                  d   | j                   j                  dz   k\  r |d   }
|
| j                   j                  k  rd } t	               j
                  d||||||||d|	}|S )Nr   r   )	input_idspast_key_valuesr^   inputs_embedsr`   rD   r   logits_to_keepr   )r   rope_scalingr>    original_max_position_embeddingsr   prepare_inputs_for_generation)r'   r   r   r^   r   r`   rD   r   r   ra   past_lengthmodel_inputss               r)   r   z-Phi3ForCausalLM.prepare_inputs_for_generation   s    $ (("dkk&R&RUV&VV(+KdkkJJJ"&J*,JJ 

+)')%)

 

 r*   )NNNNNTN)r5   r6   r7   r   r   r*   r)   r   r      s     %r*   r   c                       e Zd Zy)Phi3ForSequenceClassificationNr5   r6   r7   r   r*   r)   r   r   #      r*   r   c                       e Zd Zy)Phi3ForTokenClassificationNr   r   r*   r)   r   r   '  r   r*   r   )r   	Phi3Modelr   r   r   )Nr   )-r}   typingr   r   r   r8   torch.utils.checkpointr   activationsr   cache_utilsr	   modeling_flash_attention_utilsr
   modeling_utilsr   processing_utilsr   utilsr   mistral.modeling_mistralr   r   r   r   r   r   r   configuration_phi3r   
get_loggerr5   rm   _CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCModuler   rM   rO   r   r   r   r   r   __all__r   r*   r)   <module>r      s      , ,    !   B 5 &    + 
		H	%8 )bii )$@H)BII H)VF* FR0 &(*= &R	$D 		!> 	r*   