
    %	&ho                       d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZmZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4  e2       rd dl5m6Z6 d dl7m8Z8m9Z9 nd\  Z6Z8Z9 e1       r	d dl:m;Z;m<Z< nd\  Z<Z; e,jz                  e>      Z?dZ@ G d dej                  j                        ZB G d dej                        ZC G d d e      ZD G d! d"ej                        ZEd#ej                  d$eGd%ej                  fd&ZH	 dOd'ej                  d(ej                  d)ej                  d*ej                  d+e	ej                     d,eId-eIfd.ZJd/ ZKdPd0ZL G d1 d2ej                        ZMd3ej                  d4eGfd5ZNd6 ZOd7 ZP eQe6e;e<f      ZR G d8 d9ej                        ZS G d: d;ej                        ZT G d< d=ej                        ZU G d> d?ej                        ZV G d@ dAej                        ZW G dB dCe&      ZXdDZYdEZZ e*dFeY       G dG dHeX             Z[ G dI dJeXe      Z\ e*dKeY       G dL dMeX             Z]g dNZ^y)Q    N)cycle)AnyCallableDictListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)CacheDynamicCache)GenerationMixin)AttentionMaskConverter)FlashAttentionKwargs)BaseModelOutputWithPastCausalLMOutputWithPast SequenceClassifierOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)deprecate_kwarg)is_causal_conv1d_availableis_mamba_ssm_available   )Zamba2Config)selective_state_update)mamba_chunk_scan_combined mamba_split_conv1d_scan_combinedNNN)causal_conv1d_fncausal_conv1d_updateNNzZyphra/Zamba2-2.7Bc                   (     e Zd Zd fd	ZddZ xZS )Zamba2RMSNormGatedc                     t         |           t        j                  t	        j
                  |            | _        || _        || _        y N)	super__init__r   	Parametertorchonesweightvariance_epsilon
group_size)selfhidden_sizer9   eps	__class__s       /var/www/pru.catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/zamba2/modeling_zamba2.pyr3   zZamba2RMSNormGated.__init__F   s6    ll5::k#:; #$    c                 b   |j                   }|j                  t        j                        }|?|t        j
                  j                  |j                  t        j                              z  }|j                  ^ }}|| j                  z  } |j                  g ||| j                   }|j                  d      j                  dd      }|t        j                  || j                  z         z  } |j                  g ||| j                  z   }| j                  |j                  |      z  S N   T)keepdim)dtypetor5   float32r   
functionalsilushaper9   viewpowmeanrsqrtr8   r7   )	r:   hidden_statesgateinput_dtypeprefix_dimslast_dimgroup_counthidden_states_groupvariances	            r>   forwardzZamba2RMSNormGated.forwardL   s   #))%((7)BMM,>,>twwu}}?U,VVM!.!4!4h$//10m00\+\{\DOO\&**1-222t2D1EKK4K`K`@`4aa0+00]+]{T__?\]{{]--k:::r?   gư>r1   )__name__
__module____qualname__r3   rW   __classcell__r=   s   @r>   r/   r/   E   s    %;r?   r/   c                   ,     e Zd Zd fd	Zd Zd Z xZS )Zamba2RMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z<
        Zamba2RMSNorm is equivalent to T5LayerNorm
        N)r2   r3   r   r4   r5   r6   r7   r8   )r:   r;   r<   r=   s      r>   r3   zZamba2RMSNorm.__init__[   s1     	ll5::k#:; #r?   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S rA   )	rE   rF   r5   rG   rL   rM   rN   r8   r7   )r:   rO   rQ   rV   s       r>   rW   zZamba2RMSNorm.forwardc   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r?   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler7   rJ   r8   r:   s    r>   
extra_reprzZamba2RMSNorm.extra_reprj   s*    ))*+6$2G2G1HIIr?   rX   )rY   rZ   r[   r3   rW   re   r\   r]   s   @r>   r_   r_   Z   s    $;Jr?   r_   c                      e Zd ZdZej
                  dfdededej                  de	e
   fdZ	 ddej                  d	ej                  d
ede	ee
ef      deej                  ej                  f   f
dZdej"                  fdZdd
e	e   defdZdeeej                     eej                     f   fdZedde	eeej,                           ddfd       Zd
edej                  dej"                  dej                  fdZd Zy)Zamba2HybridDynamicCachea  
    A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache
    (which has a constant shape regardless of seq_len).

    This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`
    and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor
    For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,
    while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).
    For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),
    while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,
    and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`.
    Nconfig
batch_sizerE   devicec           	      .   || _         |j                  | _        d| _        t        |j                  |j
                  z        | _        |j                  | _        |j                  | _
        |j                  | _        g | _        i | _        i | _        i | _        i | _        i | _        t%        |j&                        D ]  }t)        j*                  || j                  d|j,                  z  |j                  z  z   | j                  ||      | j                   |<   t)        j*                  || j                  |j.                  | j                  ||      | j"                  |<   | j                  |   dk(  s| j                  j1                  |        t%        |j&                        D cg c]  }t)        j2                  g g|z  |       c}| _        t%        |j&                        D cg c]  }t)        j2                  g g|z  |       c}| _        y c c}w c c}w )NFrB   rj   rE   hybridrj   )rE   layers_block_typehas_previous_stateintmamba_expandr;   intermediate_sizemamba_d_statessm_state_sizemamba_d_convconv_kernel_sizen_mamba_headstransformer_layers_modules_parameters_buffersconv_states
ssm_statesrangenum_hidden_layersr5   zerosmamba_ngroupsmamba_headdimappendtensor	key_cachevalue_cache)r:   rh   ri   rE   rj   i_s          r>   r3   z!Zamba2HybridDynamicCache.__init__|   s    
!'!9!9"'!$V%8%86;M;M%M!N$22 & 3 3#11"$v//0 	2A"'++&&V-A-A)AFDXDX)XX%%#DQ "'D..0D0DdFYFYbhpu"DOOA %%a(H4''..q1	2 SXX^XpXpRqrQ%,,tj'8HrTYZ`ZrZrTstqELL"
):6Jt sts   !"H""H
key_statesvalue_states	layer_idxcache_kwargsreturnc                    | j                   |   j                  d   dk(  r|| j                   |<   || j                  |<   nft        j                  | j                   |   |gd      | j                   |<   t        j                  | j                  |   |gd      | j                  |<   | j                   |   | j                  |   fS )NrC   r   rB   dim)r   rJ   r   r5   cat)r:   r   r   r   r   s        r>   updatezZamba2HybridDynamicCache.update   s     >>)$**2.!3(2DNN9%*6DY'(-		4>>)3Lj2Y_`(aDNN9%*/))T5E5Ei5PR^4_ef*gDY'~~i($*:*:9*EEEr?   beam_idxc                    t        t        | j                              D ]S  }| j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |            | j                  |<   V y)zDReorders the cache for beam search, given the selected beam indices.r   N)	r   lenr   rj   index_selectrF   r   r}   r~   )r:   r   r   rj   s       r>   reorder_cachez&Zamba2HybridDynamicCache.reorder_cache   sD   s4>>23 		iI^^I.55F(,y(A(N(NqRZR]R]^dRe(fDNN9%%%i077F*.*:*:9*E*R*RSTV^VaVabhVi*jDY'%%i077F*.*:*:9*E*R*RSTV^VaVabhVi*jDY'__Y/66F)-)C)P)PQRT\T_T_`fTg)hDOOI&		ir?   c                     || j                   vr| j                   d   n|}t        | j                        |k  s | j                  |   j                         dk(  ry| j                  |   j                  d   S )zYReturns the sequence length of the cached states. A layer index can be optionally passed.r   )ry   r   r   numelrJ   )r:   r   s     r>   get_seq_lengthz'Zamba2HybridDynamicCache.get_seq_length   sl     3<4CZCZ2ZD++A.`i	t~~)+t~~i/H/N/N/PTU/U~~i(..r22r?   c                     t        d      NzAZamba2HybridDynamicCache does not have a legacy cache equivalent.NotImplementedErrorrd   s    r>   to_legacy_cachez(Zamba2HybridDynamicCache.to_legacy_cache   s    !"effr?   past_key_valuesr   c                     t        d      r   r   )clsr   s     r>   from_legacy_cachez*Zamba2HybridDynamicCache.from_legacy_cache   s    !"effr?   new_conv_statecache_positionc                 T   | j                   |   }|j                  d| j                  dz
        }|j                  dd      }|j	                  |j
                        |d d d d |f<   | j                   |   j                          | j                   |xx   |z  cc<   | j                   |   S )Nr   r%   rC   shiftsdims)r}   clamprw   rollrF   rj   zero_)r:   r   r   r   
conv_states        r>   update_conv_statez*Zamba2HybridDynamicCache.update_conv_state   s     %%i0
'--a1F1F1JK__BR_8
+9+<+<Z=N=N+O
1a'(#))+#z1#	**r?   c                 l    | j                   j                          | j                  j                          y r1   )r}   r   r~   rd   s    r>   resetzZamba2HybridDynamicCache.reset   s$     r?   r1   )r   )rY   rZ   r[   __doc__r5   float16r&   rq   rE   r   strr3   Tensorr   r   r	   r   
LongTensorr   r   r   classmethodFloatTensorr   r   r    r?   r>   rg   rg   n   sz    KP--quu"u03u<AKKuaijmanuJ 26FLLF llF 	F
 tCH~.F 
u||U\\)	*F"ie&6&6 i3 3c 3guU\\':E%,,<O'O!P g guUEVEV?W9X0Y ges g g
+
+.3ll
+LQL\L\
+	
+ r?   rg   c                   `     e Zd Z	 ddef fdZ ej                         ed               Z xZ	S )Zamba2RotaryEmbeddingrh   c                    t         |           t        |d      rG|j                  ;|j                  j	                  d|j                  j	                  d            | _        nd| _        |j                  | _        |j                  | _        || _	        t        | j
                     | _        | j                  ||j                  |j                        \  }| _        | j                  d|d       | j                   | _        y )	Nrope_scaling	rope_typetypedefault)rj   baser   inv_freqF)
persistent)r2   r3   hasattrr   getr   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrh   r   rope_init_fn
rope_thetaattention_head_dimattention_scalingregister_bufferr   original_inv_freq)r:   rh   rj   r   r=   s       r>   r3   zZamba2RotaryEmbedding.__init__   s    
 	6>*v/B/B/N#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+< 1 1v7P7P ,= ,
($( 	ZeD!%r?   c                 b   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        j                  |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   rC   r%   mpscpuF)device_typeenabledrB   r   rE   )r   floatexpandrJ   rF   rj   
isinstancer   r   r5   autocast	transposer   cosr   sinrE   )
r:   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             r>   rW   zZamba2RotaryEmbedding.forward   sV    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk^^UC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s    BF%%F.r1   )
rY   rZ   r[   r&   r3   r5   no_gradr   rW   r\   r]   s   @r>   r   r      s9     //. U]]_<  <r?   r   rO   n_repr   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r%   N)rJ   r   reshape)rO   r   batchnum_key_value_headsslenhead_dims         r>   	repeat_kvr     so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr?   modulequerykeyvalueattention_maskscalingdropoutc                 T   t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
|#|d d d d d d d |j
                  d   f   }|
|z   }
t        j                  j                  |
dt        j                        j                  |j                        }
t        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )NrB   r   r   rC   )r   rE   )ptrainingr%   )r   num_key_value_groupsr5   matmulr   rJ   r   rH   softmaxrG   rF   rE   r   r   
contiguous)r   r   r   r   r   r   r   kwargsr   r   attn_weightscausal_maskattn_outputs                r>   eager_attention_forwardr     s    3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!$Q1.D
0@0@0D.D%DE#k1==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r?   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..NrC   rB   r   )rJ   r5   r   )r   x1x2s      r>   rotate_halfr  (  sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r?   c                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer  )qkr   r   r   unsqueeze_dimq_embedk_embeds           r>   apply_rotary_pos_embr
  /  sY    ( --
&C
--
&C3w;q>C/0G3w;q>C/0GGr?   c                   F    e Zd ZdZ	 	 	 ddedee   dee   dee   f fdZ	 	 	 ddej                  dedeej                     d	ee
   d
eeej                  ej                  f      dee   deej                  eej                     eeej                        f   fdZ xZS )Zamba2Attentiona  
    Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
    and "Generating Long Sequences with Sparse Transformers".

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://arxiv.org/pdf/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)

    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://arxiv.org/pdf/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://arxiv.org/pdf/2411.15242).
    rh   r   num_fwd_mem_blocksblock_idc           	         t         |           || _        || _        |j                  | _        |j
                  | _        |j                  |j                  z  | _	        |j                  | _
        | j                  dz  dz  | _        d| _        |j                  | _        t        j                  |j                  |j                  | j                  z  d      | _        t        j                  |j                  |j                  | j                  z  d      | _        t        j                  |j                  |j                  | j                  z  d      | _        t        j                  |j                  | j                  z  |j&                  d      | _        || _        |j,                  | _        || _        |j2                  rt        j4                  g       | _        t        j4                  g       | _        t        j4                  g       | _        t=        | j*                        D ]  }||j>                  z  |k(  r{t        j@                  t        j                  | j                  | j                  jB                  d      t        j                  | j                  jB                  | j                  d            }t        j@                  t        j                  | j                  | j                  jB                  d      t        j                  | j                  jB                  | j                  d            }t        j@                  t        j                  | j                  | j                  jB                  d      t        j                  | j                  jB                  | j                  d            }n<t        jD                         }t        jD                         }t        jD                         }| j6                  jG                  |       | j8                  jG                  |       | j:                  jG                  |       ! tI        | j.                        D 	
ci c]  \  }	}
|
|	
 c}
}	| _%        y c c}
}	w )NrB   g      TFbias)&r2   r3   rh   r   attention_hidden_sizer   r   num_attention_headsr   r   r   r   	is_causalattention_dropoutr   Linearq_projk_projv_projr;   o_projr  hybrid_layer_idslayer_block_mapr  use_shared_attention_adapter
ModuleListlinear_q_adapter_listlinear_k_adapter_listlinear_v_adapter_listr   num_mem_blocks
Sequentialadapter_rankIdentityr   	enumerate	layer_dic)r:   rh   r   r  r  r   linear_q_adapterlinear_k_adapterlinear_v_adapterindexr   r=   s              r>   r3   zZamba2Attention.__init__e  sJ    	"%+%A%A"11$*$>$>&B\B\$\!'-'E'E$)d2!'!9!9ii < <f>X>X[_[h[h>hotuii < <f>X>X[_[h[h>hotuii < <f>X>X[_[h[h>hotuii : :T]] JFL^L^ejk"4%66 ..)+r):D&)+r):D&)+r):D&4223 Dv,,,8')}}		$"<"<dkk>V>V]bc		$++":":D<V<V]bc($ (*}}		$"<"<dkk>V>V]bc		$++":":D<V<V]bc($ (*}}		$"<"<dkk>V>V]bc		$++":":D<V<V]bc($
 (*{{}$'){{}$'){{}$**112BC**112BC**112BC)D, <ETEYEY;Z[<5%%,[[s   Q6rO   r   past_key_valueposition_embeddingsr   r   c                    |j                   d d }g |d| j                  }| j                  |      }	| j                  |      }
| j	                  |      }| j
                  j                  rW| j                  |   }|	 | j                  |   |      z   }	|
 | j                  |   |      z   }
| | j                  |   |      z   }|	j                  |      j                  dd      }	|
j                  |      j                  dd      }
|j                  |      j                  dd      }| j
                  j                  r|\  }}t        |	|
||      \  }	}
||j                  |
||      \  }
}t         }| j
                  j"                  dk7  r^| j
                  j"                  dk(  r(|j%                  dd      rt&        j)                  d       nt*        | j
                  j"                     } || |	|
||f| j,                  sd	n| j.                  | j0                  d
|\  }} |j2                  g |d j5                         }| j7                  |      }||fS )NrC   r%   rB   eagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.        )r   r   )rJ   r   r  r  r  rh   r  r'  r  r   r!  rK   r   use_mem_roper
  r   r   _attn_implementationr   loggerwarning_oncer   r   r  r   r   r   r  )r:   rO   r   r   r,  r-  r   input_shapehidden_shapequery_statesr   r   adapter_layer_idxr   r   attention_interfacer   r   s                     r>   rW   zZamba2Attention.forward  s\    $))#2.88b8$--8{{=1[[/
{{=1;;33 $y 9'*W$*D*DEV*WXe*ffL#&Sd&@&@AR&STa&bbJ'*W$*D*DEV*WXe*ffL#((6@@AF__\2<<QB
#((6@@AF;;##*HC';L*VY[^'_$L*%'5'<'<ZW`'a$J(?;;++w6{{//69fjjI\^c>d##L
 '>dkk>^>^&_#$7	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r?   r*   )rY   rZ   r[   r   r&   r   rq   r3   r5   r   rg   r	   r   r   rW   r\   r]   s   @r>   r  r  J  s    : $(,0"&6\6\ C=6\ %SM	6\
 3-6\x 26=AKO7)||7) 7) !.	7)
 !!9:7) &eELL%,,,F&GH7) -.7) 
u||Xell3XeELL>Q5RR	S7)r?   r  input_tensorpad_sizec                     t        | j                        dk(  r
ddddd|ddfnddd|ddf}t        j                  j                  j                  | |dd      S )z
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
       r   constant)moder   )r   rJ   r5   r   rH   pad)r<  r=  	pad_shapes      r>   pad_tensor_by_sizerD    sf     47|7I7I3Ja3OAq!Q!Q/VWYZ\]_gijlmUnI88""<ST"UUr?   c                    t        | |      } t        | j                        dk(  r.| j                  | j                  d   d|| j                  d         S | j                  | j                  d   d|| j                  d   | j                  d         S )z
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   rC   rB   )rD  r   rJ   r   )r<  r=  
chunk_sizes      r>   reshape_into_chunksrG    s     &lH=L
<!###L$6$6q$92z<K]K]^_K`aa ##q!2z<3E3Ea3H,J\J\]^J_
 	
r?   c                 "   | j                  d      } | d   j                  g | j                         | } t        j                  t        j                  ||| j
                  t        j                        d      }| j                  | d      } t        j                  | d      }t        j                  t        j                  ||| j
                  t        j                        d      }|j                  | t        j                         }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    rC   .Nrl   diagonalr   r   r   )
sizer   r5   trilr6   rj   boolmasked_fillcumsuminf)r<  rF  masktensor_segsums       r>   segment_sumrT    s     ""2&J 2<	*11S<3D3D3FS
SL::ejjZ@S@S[`[e[efqstD++TE15LLL26M ::ejjZ@S@S[`[e[efqrsD!--teeiiZ@Mr?   c                        e Zd ZdZddedee   f fdZ	 	 ddej                  dee
   deej                     fdZddee
   deej                     fd	Z	 	 ddee
   deej                     fd
Z xZS )Zamba2MambaMixeru  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    rh   r   c           	         t         |           || _        |j                  | _        |j                  | _        |j                  | _        t        |j                  | j                  z        | _
        || _        |j                  | _        d| _        t        j                         | _        |j"                  | _        |j$                  | _        |j(                  | _        | j                  j,                  | _        |j0                  | _        |j2                  | _        |j4                  | _        |j6                  | _        | j                  d| j&                  z  | j
                  z  z   | _        t        j:                  | j8                  | j8                  d|j                  | j8                  |j                  dz
        | _        | j                  | j8                  z   | j.                  z   }t        j>                  | j                  ||j@                        | _!        t        jD                  tG        jH                  | j.                              | _%        tG        jL                  d| j.                  dz         }t        jD                  tG        jN                  |            | _(        d| jP                  _)        tU        | j                  | j                  | j&                  z  d      | _+        t        jD                  tG        jH                  | j.                              | _,        d| jX                  _)        t        j>                  | j                  | j                  |j@                        | _-        t\        st^        ja                  d	       y y )
NrI   rB   Tr%   )in_channelsout_channelsr  kernel_sizegroupspaddingr  gh㈵>)r9   r<   a  The fast path is not available because on of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)1r2   r3   rh   r;   rt   ru   rv   rw   rq   rr   rs   r   use_conv_bias
activationr   SiLUactuse_mem_eff_pathr   n_groupsr   r   rx   	num_headsrF  time_step_limittime_step_mintime_step_maxconv_dimConv1dconv1dr  add_bias_linearin_projr4   r5   r6   dt_biasarangelogA_log_no_weight_decayr/   normDout_projis_fast_path_availabler5  r6  )r:   rh   r   projection_sizeAr=   s        r>   r3   zZamba2MambaMixer.__init__  s   !--$22 & 3 3!$V%8%84;K;K%K!L"#11 779 & 7 7,,,,22 ++%55#11#11..T]]1BTEXEX1XXii++==''!+
 004==@4>>Qyy''
 ||EJJt~~$>? LLDNNQ./\\%))A,/
&*

#&""t/E/E/V\`
	 ejj89"&		$"8"8$:J:JQWQgQgh%> &r?   rO   cache_paramsr   c                    |j                   \  }}}| j                  | j                  z  }d| j                  z  d| j                  z  | j                  z  z   | j                  z   }|4|j
                  r'| j                  |j                  d            }	|	j                   d   |z
  dz  }
|
|
| j                  | j                  | j                  g}t        j                  |	|d      \  }}}}}t        ||j                  | j                     | j                  j                  j                  d      | j                  j                   | j"                        }t        j                  || j                  ||gd      \  }}}t        j$                  | j&                  j)                                }|d d d df   d d d d d f   j+                  d| j,                  | j                        j/                  t        j0                        }|d d d d d f   j+                  dd| j,                        }| j2                  d d d df   j+                  d| j,                        }| j4                  d d d df   j+                  d| j,                        }|j7                  || j                  |j                   d   | j                  z        }|j7                  || j                  |j                   d   | j                  z        }|j7                  || j                  | j,                        }t9        |j:                  | j                     ||||||d |d
      }|j7                  || j                  | j,                  z        }| j=                  ||      }| j?                  |      d d d df   }|S |Bt        j@                  |dk(        s*|jB                  }||d d d d d f   z  j/                  |      }| j                  |      }t        j$                  | j&                  j)                                }| jD                  i nd	| jD                  i}|t        j@                  |dk(        }nd}| jF                  r| jH                  r||rtK        || j                  j                  j                  d      | j                  j                   | j2                  |f| j4                  | jL                  d | j"                  | j<                  j                  | j<                  jN                  | j>                  j                  | j>                  j                   | j,                  | j                  d
dd|\  }}|S t        j                  || j                  | j                  | j                  gd      \  }}}|v|jQ                  dd      }tR        jT                  jW                  || jX                  |j                   d   z
  df      }|j                  | j                     j[                  |       t\        | j"                  dvrJ| j_                  | j                  |jQ                  dd            jQ                  dd      d d d |f         }nyt]        |jQ                  dd      | j                  j                  j                  d      | j                  j                   | j"                        jQ                  dd      d d d |f   }t        j                  || j                  ||gd      \  }}}|Bt        j@                  |dk(        s*|jB                  }||d d d d d f   z  j/                  |      }ta        |j7                  ||d| j,                        |||j7                  ||| j                  d      |j7                  ||| j                  d      f| jL                  | j4                  d d d| j2                  dd|\  }}|*|(|j:                  | j                     j[                  |       |j7                  ||d      }| j=                  ||      }| j?                  |      }|S )NrB   r%   rC   r   .r   T)zrl  dt_softplusdt_limitF)rr  rF  seq_idxr^  rmsnorm_weightrmsnorm_epsoutproj_weightoutproj_biasheaddimngroupsnorm_before_gatereturn_final_statesr   )rI   swish)r   r7   r  r^  )rF  rr  ry  r|  r  rl  rz  )1rJ   rb  ru   rs   rc  rp   rk  squeezerg  r5   splitr,   r}   r   ri  r7   r  r^  expro  r   r   r   rF   rG   rl  rr  rK   r'   r~   rq  rs  allrE   rd  ra  r   r)   rF  r8   r   r   rH   rB  rw   copy_r+   r`  r(   )r:   rO   rw  r   ri   seq_lenr   groups_time_state_sized_to_removein_projected_statesd_mlpsplit_projection_dimrP   hidden_states_B_CdtBCrv  rl  rr  hidden_states_reshapedoutrE   projected_statesdt_limit_kwargsinput_not_masked	ssm_state	time_stephidden_states_B_C_tr   scan_outputs                                  r>   cuda_kernels_forwardz%Zamba2MambaMixer.cuda_kernels_forwardY  sv    "/!4!4
GQ!%1D1D!D$0001t}}3DtGZGZ3ZZ]a]k]kk #(G(G"&,,}/D/DQ/G"H(..r2[@QFE$)5$2H2H$--Y]YgYg#h 05<OQekm0n-Aq$)2 4!((8""**1-  ! #(++!'')?AWX#M1a
 4::++-..A!T3,1d
+222t}}dFYFYZ]]didqdq]rAAq$J&&r2t}}=Bll1dC<077DMMJGq$|$++B>Az4==!''!*2MNAz4==!''!*2MNA%2%7%7
DNNTXTaTa%b"2''7& M *..z4>>DMM;YZM IImT:M--.q$|<Cz 
u )%))Na<O2P%++!.1d
1K!K O OPU V#||M:4::++-..A$($8$8$@bzSWSgSgFhO)#(99^q-@#A #' $$<;OTd!A$KK&&..q1KK$$LL" ff# ##'99#3#3 $		 : :#'==#7#7!%!3!3 MM MM%*(,#"$ &%"YX 
m 6;[[$++T]]DNNK62'  +*;*E*Ea*K'!#!2!2+d.C.CFYF_F_`bFc.cef-g"J !,,T^^<BB:N#+tFW/W(,$5$?$?1$EFPPQRTUVWXZb[bZbWbc)% )9+55a;#{{1199!<![[--#'??	)
  i1oa'k)3% ',kk%++-CE[\'#q!
 "-eiiRS@S6T)//E%2^Aq$J5O%O$S$STY$ZM)B!&&z7BNFF:wrBFF:wrB*  $ff (, LL $* &*&Y (\-E ++DNN;AA)L)..z7BG"iiT:mmK0
r?   c                    |j                   \  }}}|j                  }|-|j                  r!| j                  |j	                  d            }nI|6t        j                  |dk(        s||d d d d d f   z  j                  |      }| j                  |      }|j                   d   d| j                  z  z
  d| j                  z  | j                  z  z
  | j                  z
  dz  }	|j                  |	|	| j                  | j                  | j                  gd      \  }}}
}}|w|j                  | j                     j!                         }|j                  |j"                        }|j                  r1|
j%                  d      }
|j&                  | j                     }t        j(                  |dd      }|j*                  dk(  r|d d dd d f   n||d d d d df<   |j&                  | j                     j-                  |       t        j.                  |j                  |j"                        | j0                  j2                  d d dd d f   z  d      }| j4                  r|| j0                  j6                  z  }| j9                  |      j                  |      d d d df   }n|j;                  dd      }t<        j>                  jA                  || jB                  |j                   d   z
  df      }|j&                  | j                     j-                  |       | j9                  | j1                  |      j;                  dd            d d d |d d f   }|t        j                  |dk(        s|j                  }||d d d d d f   z  j                  |      }nt        jD                  || j                  | jF                  | j                  f|j"                  |	      }| j9                  | j1                  |j;                  dd            dd |f   j;                  dd            }t        j                  || j                  | j                  | j                  z  | j                  | j                  z  gd      \  }}}t        jH                  | jJ                  jM                                }|t|j                  rg|j*                  dk(  r
|d d d df   n|d d dd d f   d d d df   }|j;                  dd      jO                  ||j                   d   | jF                        }| jP                  d
   jO                  | jP                  j                   d   | jF                        }t
        j<                  j>                  jS                  ||j                  |j                        z         }t        jT                  || jV                        }|d   jO                  | j                  | jF                  | j                        j                  t
        jX                        }t        jH                  |d
   |z        }|j[                  || j                  d      dd d d f   }|jO                  || j                  | j                  | j                  z  |j                   d         j]                         }|j[                  |d|j                   d         }|d
   |dd d d f   z  }|j[                  |d| jF                        }||d
   z  }|j                  | j                     j-                  |j                  | j                     |z  |z          |j[                  || j                  d      dd d d f   }|jO                  || j                  | j                  | j                  z  |j                   d         j]                         }|j[                  |d|j                   d         }|j                  | j                     j                  |j                        }|j_                  || j                  z  | jF                  | j                        }|j_                  || j                  z  | j                  d      }t        j`                  ||      }|j_                  || j                  | jF                        }| jb                  d
   jO                  | jb                  j                   d   | jF                        }|||z  z   j                  |j                        }|j[                  |d      d d d df   }n t<        j>                  jS                  || jP                  z         }t        jT                  || jV                        }|j[                  ||d| jF                        jM                         }|j[                  ||d| j                        jM                         }|j[                  ||d| j                        jM                         }|je                  dd| j                  | j                  z  d      }|je                  dd| j                  | j                  z  d      }| jf                  || jf                  z  z
  | jf                  z  }| jb                  d
   ti        ||      z  }||d
   z  }|j                  |j                        |z  }||||fD cg c]  }tk        ||| jf                         c}\  }}}}|jm                  dddd      }t        jn                  |d      }t        jH                  tq        |            }|d d d d d d d d d d d f   |d d d d d d d d d d d f   z  } | j/                  d      }!|!d
   |jm                  ddddd      d
   z  }"|"j/                  d      }#|#d
   |d d d d d f   z  j/                  d      }$t        jH                  |d d d d d d dd f   |z
        }%||%jm                  dddd      d
   z  }&|&jm                  ddddd      d
   |jm                  ddddd      dd d d f   z  j/                  d      jm                  ddddd      }'|.|j                  r"|j                  | j                     d d d df   }(nt        jr                  |'d d d df         }(t        jt                  |(|'gd      }'t        jH                  tq        t<        j>                  jA                  |d d d d d d df   d                  })|'jm                  ddddd      }*|)d   |*d d d d d df   z  j/                  d      }+|+jm                  ddddd      },|,d d d df   |,d d df   }}'t        jH                  |      }-|dd d d f   |'d d d d d df   z  }.|-jm                  dddd      }/|.j/                  d      |/d
   z  }0|$|0z   }|j[                  |d| j                  | jF                        }||z   }|dkD  r|d d d |d d d d f   }|j[                  ||d      }|*|(|j                  | j                     j-                  |       | jw                  ||
      }1| jy                  |1j                  |            }2|2S c c}w )Nr%   rC   rB   r   r   r   r   .rl   rI  ).NNr   r?  )r%   r   )=rJ   rE   rp   rk  r  r5   r  rF   rs   rb  ru   rc  r  rg  r~   r   clonerj   r  r}   r   ndimr  sumri  r7   r]  r  r`  r   r   rH   rB  rw   r   r   r  ro  r   r   rl  softplusr   re  rG   r   r   rK   bmmrr  repeatrF  rD  rG  permuterP  rT  
zeros_liker   rq  rs  )3r:   input_statesrw  r   ri   r  r   rE   r  r  rP   rO   r  r  r   r  r  rv  rl  dAdBdBxr~   ssm_states_reshaped
C_reshapedyrr  r=  
D_residualtA_cumsumLG_intermediateGM_intermediateMY_diagdecay_statesB_decay_contractionstatesprevious_statesdecay_chunkstates_permutedresult
new_statesstate_decay_outC_times_statesstate_decay_out_permutedY_offr  contextualized_statess3                                                      r>   torch_forwardzZamba2MambaMixer.torch_forward  sj   !-!3!3
GQ""#(G(G $\-A-A!-D E)%))NA<M2N$0>!Q*3M$M#Q#QRW#XL $\ :!''+a$2H2H.HHAPTP]P]L]`d`s`sLssuy  vD  vD  D  IJ  J(8(>(>t55t~~V\^ )? )
%1dM2
 #$//?EEGI!]%9%9:I..~~a()55dnnE
"ZZ
2BG
ANASASWXAX}Q1W'=^k
1a8$((8>>zJ %		*--8H8O8O*PSWS^S^SeSefgijlmfmSn*ntv w%%!T[[%5%55M $ 7 : :5 A!T3, O - 7 7! <]]..!**]-@-@-DDaH
 ((8>>zJ $])C)M)MaPQ)R STUW_X_W_abTb c!-eiiPQ@Q6R)//E%2^Aq$J5O%O$S$STY$ZMT^^T]]D<O<OP$++5I !HHT[[1H1HA1N%OPSU]V]U]P]%^%h%hijlm%noM#kk-$:P:PRVR_R_bfbubuRuw{  xE  xE  HL  H[  H[  x[  :\  bd  eq!YYtzz'')**#(G(G &(WW\AtSL!r!Q'{1dC<7PBa#**:rxx|T]]SBll9-44T\\5G5G5JDMMZG$$--b7::bhh3G.GHBR!3!34B/"))$..$--I\I\]``glgtgt`uA2i=1,-B
 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6AI3a<0B *11*b$--PM}Y//C ##DNN399''7"<sB 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6A &00@CCAGGLJ",//*t~~2Mt}}^b^q^q"r
T^^ ;T=P=PRSTJ		-z:Az4>>4==AA y!((a$--HA]Q&&**1773A 		*b)!T3,7A ''T\\(9:BR!3!34B)11*gr4==Y__aM		*gD4G4GHNNPA		*gr43F3FGMMOAAt~~>BAAt~~>BA'DOO*CCtVH	*-?x-XXJ *ByM9M](()B.A cpqrtuwxay%z\]&9!Xt&W%z"M1a 		!Q1%A||A2.H 		+a.)A q!Qa23a1dAq!8K6LLN""r"*A y\AIIaAq!,DY,OON""r"*A 	l]1a:%>>CCAFF !99hq!Q|&<x&GIL"#l&:&:1aA&Fy&Q"Q)11!Q1a@K}OdOdefhiklnoqrOstwy}  @A  uA  PB  B  G  G  LM  G  N  V  V  WX  Z[  ]^  `a  cd  eF'L,K,K"."9"9$.."I!TSV,"W"'"2"26!RaR%="AYY8a@F))K0A0A(1aQRTV;BWY_0`$abK$nnQ1a;O!/2_Q4QT_5UUZZ_`ZaF1aA6J *1crc6 2Jq"u4EIF $ii1OT1oq!T30GGN'6'>'>q!Q'J$#''+.Fy.QQE A		*b$..$--HAJA!|a'1a'(		*gr2A$)A''7==iHii4(
 !%knnU.C D$$I &{s   7y2c                     t         r?d| j                  j                  j                  j                  v r| j                  |||      S | j                  |||      S )Ncuda)rt  rk  r7   rj   r   r  r  )r:   rO   rw  r   s       r>   rW   zZamba2MambaMixer.forward  sN     "f0C0C0J0J0O0O&O,,]L.YY!!-~NNr?   r1   r-   )rY   rZ   r[   r   r&   r   rq   r3   r5   r   rg   r  r  rW   r\   r]   s   @r>   rV  rV    s    ?| ? ?H <@15	T||T 78T !.	Tn%AY8Z %qyz  {G  {G  rH %J <@15		O 78	O !.		Or?   rV  c                   8     e Zd Zddedee   f fdZddZ xZS )	Zamba2MLPrh   r  c           	          t         	|           || _        |j                  | _        |j                  | _        || _        || _        t        j                  | j                  d| j                  z  |j                        | _
        t        j                  | j                  | j                  |j                        | _        t        |j                     | _        t        j                  g       | _        t#        | j
                        D ]  }||j$                  z  |k(  rt        j&                  t        j                  | j                  j                  | j                  j(                  d      t        j                  | j                  j(                  d| j                  z  d            }nt        j*                         }| j                   j-                  |        |j.                  }t1        |      D ci c]  \  }}||
 c}}| _        yc c}}w )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        rB   r  FN)r2   r3   rh   r;   rs   r  r  r   r  rj  gate_up_proj	down_projr   
hidden_actact_fnr  gate_up_proj_adapter_listr   r"  r#  r$  r%  r   r  r&  r'  )
r:   rh   r  r  r   gate_up_proj_adapterr  r+  r   r=   s
            r>   r3   zZamba2MLP.__init__  s   
 	!--!'!9!9"4 IId&6&6D<R<R8RY_YoYop4#9#94;K;KRXRhRhiV../)+r):&t../ 	HA6(((H4')}}IIdkk55t{{7O7OV[\IIdkk66D<R<R8RY^_($
 (*{{}$**112FG	H !11;D_;UV<5%%,VVs   3H
c                     | j                  |      }| j                  |   }| | j                  |   |      z   }t        j                  |dd      }| j                  |d         |d   z  }| j                  |      }|S )NrB   rC   r   r   r%   )r  r'  r  r5   chunkr  r  )r:   hidden_stater   gate_up_stateoutputs        r>   rW   zZamba2MLP.forward  s    )),7NN9-	%(Q(F(Fy(QR^(__M1"={{=#34}Q7GG-r?   r-   r1   )	rY   rZ   r[   r&   r   rq   r3   rW   r\   r]   s   @r>   r  r    s%    W| WPXY\P] W<r?   r  c                   4    e Zd Zddedee   dee   f fdZ	 	 	 	 ddej                  dej                  dedeej                     dee	   d	ee
   d
eej                     dee   deej                  eeej                  ej                  f      f   fdZ xZS )Zamba2AttentionDecoderLayerrh   r  r   c                 @   t         |           || _        t        |j                        }t        |d||      | _        t        |||      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        y )NrC   )r   r  r  )r  r  r<   )r2   r3   r  r   r  r  	self_attnr  feed_forwardr_   r  rms_norm_epsinput_layernormr;   pre_ff_layernorm)r:   rh   r  r   num_gsr=   s        r>   r3   z$Zamba2AttentionDecoderLayer.__init__  s     V,,-(2RXckl%fRZ[,V-I-IvObObc -f.@.@fFYFY Zr?   rO   original_hidden_statesr   r,  r1  r-  r   r   c           
          t        j                  ||gd      }| j                  |      } | j                  d||||||d|\  }}	| j	                  |      }| j                  ||      }|f}
|r|
|	fz  }
|
S )a  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://arxiv.org/pdf/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        rC   r   )rO   r   r   r,  r1  r-  r   )r5   concatenater  r  r  r  )r:   rO   r  r   r   r,  r1  r-  r   self_attn_weightsoutputss              r>   rW   z#Zamba2AttentionDecoderLayer.forward  s    > ))=:P*QWYZ,,];+94>> ,
'))/ 3,
 ,
(( --m<))-C ")++Gr?   r-   )NNFN)rY   rZ   r[   r&   r   rq   r3   r5   r   rg   rN  r   r   r   r	   r   rW   r\   r]   s   @r>   r  r    s    [| [x} [X`adXe [ 26=A,1:>3||3 !&3 	3
 !.3 !!9:3 $D>3 &e&6&673 -.3 
u  (51B1BEDUDU1U+V"WW	X3r?   r  c                   t    e Zd Zdedef fdZ	 	 	 	 	 	 	 	 	 ddej                  deej                     dee   deej                     deej                     dee	   d	ee
   d
ee
   deej                     deej                     deej                  eeej                  ej                  f      f   fdZ xZS )Zamba2MambaDecoderLayerrh   r   c                     t         |           t        ||      | _        t	        |j
                  |j                        | _        || _        y )N)rh   r   r  )	r2   r3   rV  mambar_   r;   r  r  r   )r:   rh   r   r=   s      r>   r3   z Zamba2MambaDecoderLayer.__init__)  s>    %VyI
,V-?-?VEXEXY"r?   rO   r  r   r   r,  r1  	use_cacher   transformer_hidden_statesr   c                     |}|
||
z   n|}| j                  |      }| j                  |||      }d}||z   }|f}|r||fz  }|r||fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence.
        N)rO   rw  r   )r  r  )r:   rO   r  r   r   r   r,  r1  r  r   r  r   residualr  r  s                  r>   rW   zZamba2MambaDecoderLayer.forward/  s    < !
 :S9^M55dq 	 ,,];

'') # 
 ! !=0 ")++G((Gr?   )	NNNNNFFNN)rY   rZ   r[   r&   rq   r3   r5   r   r   rg   rN  r   r	   r   rW   r\   r]   s   @r>   r  r  (  s   #| # # :>#'15.2=A,1$)59<@:||: !) 6: C=	:
 !.: ell+: !!9:: $D>: D>: !!1!12: $,ELL#9: 
u  (51B1BEDUDU1U+V"WW	X:r?   r  c                   l    e Zd Zdedej
                  def fdZ	 	 	 	 	 	 	 	 ddej                  de
ej                     de
e   de
ej                     d	e
ej                     d
e
e   de
e   de
e   de
ej                     deej                   e
eej                   ej                   f      f   fdZ xZS )Zamba2HybridLayershared_transformerlinearr  c                 L    t         |           || _        || _        || _        y r1   )r2   r3   r  mamba_decoderr  )r:   r  r  r  r=   s       r>   r3   zZamba2HybridLayer.__init__m  s'     	""4r?   rO   r  r   r   r   r,  r1  r  r-  r   c
           	          | j                  |||||||	      }
|
d   }|r|
d   }| j                  |      }| j                  |||||||	      }
|r|
d   f|
dd z   }
|
S )aX  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )r  r   r   r,  r1  r-  r   r%   )r  r   r,  r1  r  r-  rB   N)r  r  r  )r:   rO   r  r   r   r   r,  r1  r  r-  layer_outputsr  r  s                r>   rW   zZamba2HybridLayer.forwardu  s    @ //#9&)/ 3 0 
 %2!$4! -a 0$(KK0I$J!**&?))/ 3 + 
 *1-/@AMRSRTDUUMr?   )NNNNNFFN)rY   rZ   r[   r  r   r  r  r3   r5   r   r   rq   rg   rN  r   r	   r   rW   r\   r]   s   @r>   r  r  l  s   5"=5GIyy5Yp5 :>#'15.2=A,1$):>>||> !) 6> C=	>
 !.> ell+> !!9:> $D>> D>> &e&6&67> 
u  (51B1BEDUDU1U+V"WW	X>r?   r  c                   >    e Zd ZeZdZdZddgZdZdZ	dZ
dZdZdZd Zy)Zamba2PreTrainedModelmodelTr  r  r   c                 ^   | j                   j                  }t        |t        j                  t        j
                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j                  j                  j                  d|       |j                  2|j                  j                  |j                     j                          y y t        |t              rwd|j                  _        d|j                   _        t#        j$                  t#        j&                  | j                   j(                        t+        j,                  | j                   j.                        t+        j,                  | j                   j0                        z
  z  t+        j,                  | j                   j0                        z         j3                  | j                   j4                        }|t#        j,                  t#        j6                  |              z   }t#        j8                         5  |j:                  j=                  |       d d d        d|j:                  _        y y # 1 sw Y   xY w)Nr2  )rM   stdT)min) rh   initializer_ranger   r   r  rh  r7   datanormal_r  r   	Embeddingpadding_idxrV  ro  rp  rr  r5   r  randrx   mathrn  rf  re  r   time_step_floorexpm1r   rl  r  
_no_reinit)r:   r   r  r  inv_dts        r>   _init_weightsz#Zamba2PreTrainedModel._init_weights  s   kk++fryy"))45MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> . 01,0FLL)(,FHH%

4;;44588DKK556$++B[B[9\\^((4;;4456 e33e4	  %))U[["%5$566F -$$V,-(,FNN% 2- -s   ,J##J,N)rY   rZ   r[   r&   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_flex_attn_supports_sdpa_supports_cache_class_is_statefulr  r   r?   r>   r  r    sF    L&*#68QR"3!N L-r?   r  aK  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`Zamba2Config`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
            `past_key_values`).

            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
            information on the default strategy.

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        past_key_values (`Zamba2HybridDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            A Zamba2HybridDynamicCache object containing pre-computed hidden-states (keys and values in the
            self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
            Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`.
            Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and
            `(batch_size, d_inner, d_state)` respectively.
            See the `Zamba2HybridDynamicCache` class for more details.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
            Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
            this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
            the complete sequence length.
zTThe bare Zamba2 Model outputting raw hidden-states without any specific head on top.c                   J    e Zd ZdZdef fdZd Zd Z ee	      	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     d	e
e   d
e
ej                     de
e   de
e   de
e   de
e   de
ej                     deeef   fd       Zd Zd Z xZS )Zamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    rh   c                    t         |   |       || _        |j                  | _        |j
                  | _        t        j                  |j
                  |j                  | j                        | _	        t        |j                        D cg c]  }t        ||       }}g }g }|j                  | _        t        |j                        D ]  }|j                  |   dk(  r|j                  t!        ||             2|j                  |   dk(  sE|j                  t        j"                  | j                  j                  | j                  j                  d             |j                  t!        ||              t%        |      }t%        |      }t'        |      }| j)                  |||      }t        j*                  |      | _        |j.                  | _        t1        |j                  |j2                        | _        |j6                  r1|j8                  rt:        j=                  d       t?        |      | _         d| _!        | jE                          y c c}w )	N)r  r  r   rm   Fr  r  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.)#r2   r3   rh   pad_token_idr  
vocab_sizer   r  r;   embed_tokensr   r"  r  ro   r   r   r  r  iterr   
get_layersr  layersr4  r_   r  final_layernormr3  use_long_contextr5  r6  r   
rotary_embgradient_checkpointing	post_init)	r:   rh   r  blocksmamba_layerslinear_layersr   r  r=   s	           r>   r3   zZamba2Model.__init__?  s    !.. ++LL):):F<N<NPTP`P`aKPQWQfQfKgha-fqAhh!'!9!9v//0 	RA''*g5##$;Fa$PQ))!,8$$RYYt{{/F/FH_H_fk%lm##$;Fa$PQ	R L)]+vEmmF+$*$?$?!,V-?-?VEXEXY&&##{ 4F;DO&+# 	7 is   Ic                     | j                   S r1   r  rd   s    r>   get_input_embeddingsz Zamba2Model.get_input_embeddingsc  s       r?   c                     || _         y r1   r$  r:   r   s     r>   set_input_embeddingsz Zamba2Model.set_input_embeddingsf  s
    !r?   	input_idsr   r   r   inputs_embedsr  r1  output_hidden_statesreturn_dictr   r   c                 r   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|	|	n| j                   j                  }	|d u |d uz  rt        d      | j                  r%| j                  r|rt        j                  d       d}|| j                  |      }|}t        j                  |      }|rO|M||j                  d   n|j                  d   }t        | j                   || j                  | j                         }|
R||j#                  | j$                        nd}t        j&                  |||j                  d   z   |j                         }
||
j)                  d      }| j+                  |||
      }| j                   j,                  r| j/                  ||      }nd }|rd	nd }|rd	nd }t1        | j2                        D ]r  \  }}|r||fz  }| j                  r1| j                  r%| j5                  |j6                  |||||||||
      }n ||||||||||
	      }|d   }|sd|d   j||d   fz  }t | j9                  |      }|r||fz  }|r|j:                  sd|_        t=        ||r|nd ||      }|	r|S |j?                         S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either onezX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   rE   rj   r  r%   rn   r   )r  r   r   r   r,  r1  r  r-  T)last_hidden_stater   rO   
attentions) rh   r1  r+  r  use_return_dict
ValueErrorr  r   r5  r6  r  r5   r  rJ   rg   rE   rj   r   first_transformer_layer_idrm  r  _update_causal_maskr3  r  r&  r  _gradient_checkpointing_func__call__r  rp   r   to_tuple)r:   r)  r   r   r   r*  r  r1  r+  r,  r   rO   r  ri   past_seen_tokensr   r-  all_hidden_statesall_self_attnsr   layerr  r  s                          r>   rW   zZamba2Model.forwardi  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	%0%<k$++B]B]-t";<s  &&4==Yj I  --i8M%!&]!; 0/8/D+-J]J]^_J`J6t{{JVZV`V`imitituO! #.  ..9X9X.Y 
 #\\ "2]5H5H5K"KTaThThN )33A6L..~}n] ;;##"&//-"N"&"6BD0d )$++ 6 "	:Iu#!m%55!**t}} $ A ANN!*"#%'! !&!+A'#1 +#2&7'(;
! *!,M  #/"}Q'7&99NE"	:H ,,];  -!11?#E#E15O.(+/8Od+%	
 %v;&//*;;r?   c                    | j                   j                  dk(  r	|d|v r|S y |j                  |j                  }}t	        j
                  |      j                  }|j                  d   }|d   dz   }t	        j                  ||f|||      }	|dk7  rt	        j                  |	d      }	|	t	        j                  ||      |j                  dd      kD  z  }	|	d d d d d d f   j                  |j                  d   ddd      }	||	j                         }	|j                         d	k(  rd|j                  d   }
|	d
d |
f   j                  d      |d d d d d d f   j                  d      z  }|	d
d |
f   j!                  ||      |	d
d |
f<   | j                   j                  dk(  r0|.|j                  j"                  dv rt%        j&                  |	|      }	|	S )Nflash_attention_2r2  r%   rC   )
fill_valuerE   rj   rJ  rn   r   rB   .r0  )r  xpu)rh   r4  rE   rj   r5   finfor  rJ   fulltriurm  r   r   r  r   eqrO  r   r   _unmask_unattended)r:   r   r<  r   rE   rj   	min_dtypesequence_lengthtarget_lengthr   mask_lengthpadding_masks               r>   r4  zZamba2Model._update_causal_mask  s   ;;++/BB)c^.C%%$**L,?,?vKK&**	&,,Q/&r*Q.jj/=!Ai_dmsta**[1=Ku||M&ANDZDZ[]_`Daaa!$a"23::<;M;Ma;PRSUWY[\%%++-K!!#q(,2226*3+<=@@EWXZ^`dfgWgHhHkHkloHpp1<S,;,=N1O1[1[\hjs1tC+-. KK,,6*%%**o=
 1CCKQZ[Kr?   c           
      x   g }g | _         d| _        t        | j                        D ]  \  }}|dk(  r| j                  dk(  r|| _        t	        |      }| j
                  j                  t        | j
                  j                        z  dkD  r_d| d}t        j                  |dz   dz   dz   d	z   d
z         }	| j                   j                  |	       d}
| j                  D ]q  }|dk(  re|
| j
                  j                  z  |j                  k(  r?t        j                  dt        |
      z   dz         }| j                   j                  |       |
dz  }
s | j
                  j                  rd}
| j                  D ]q  }|dk(  re|
| j
                  j                  z  |j                  k(  r?t        j                  dt        |
      z   dz         }| j                   j                  |       |
dz  }
s |j                  t        |t	        |      t	        |                   |j                  t	        |              |S )Nr   rm   r%   z	^layers\.z\.shared_transformer\.z(?:z3self_attn\.(?:q_proj|k_proj|v_proj|o_proj)\.weight|z1feed_forward\.(?:gate_up_proj|down_proj)\.weight|z,(?:input_layernorm|pre_ff_layernorm)\.weightz)$z>^shared_transformer\.feed_forward\.gate_up_proj_adapter_list\.z\.(?:0|1)\.weight$zg^shared_transformer\.self_attn\.(?:linear_q_adapter_list|linear_k_adapter_list|linear_v_adapter_list)\.)_tied_weights_keysr3  r&  ro   nextrh   r"  r   r  recompiler   r  r   r  r  )r:   r   r"  r!  r  layer_id
layer_typeblockprefix_patternmain_keys_pattern
adapter_id_layer_typeadapter_patternattn_adapter_patterns                 r>   r  zZamba2Model.get_layers  sH   "$*+'$-d.D.D$E )	2 HjX%22a76>D3V;;--DKK4P4P0QQTUU(1(;Q%RN(*

& !PQ OO J	J
   )% ++223DE!"J'+'='= (&(2zDKKD^D^7^bgbpbp7p.0jj a"%j/!2"7!8/O
 !33::?K"a
( {{??%&
+/+A+A 	,K*h6:HbHb;bfkftft;t79zz%q&)*o%6 '<%<8" 4 !% 7 7 > >?S T&!OJ	, /tM7JDQ]L^_`d<01S)	2T r?   
NNNNNNNNNN)rY   rZ   r[   r   r&   r3   r%  r(  r   ZAMBA2_INPUTS_DOCSTRINGr   r5   r   r   rg   r   rN  r
   r	   r   rW   r4  r  r\   r]   s   @r>   r  r  3  s5   
"| "H!" ++BC 151537>B59$(,0/3&*59v<E,,-v< !.v< u//0	v<
 "":;v<   1 12v< D>v< $D>v< 'tnv< d^v< !!1!12v< 
u--	.v< Dv<p!F.r?   r  c                        e Zd Zdef fdZd Zd Zd Zd Zd Z	d Z
 ed	d
d       ee       eee      	 	 	 	 	 	 	 	 	 	 	 	 ddeej&                     deej(                     deej&                     dee   deej,                     deej&                     dee   dee   dee   dee   deej&                     deeej(                  f   deeef   fd                     Z	 	 	 	 	 	 ddZ xZS )Zamba2ForCausalLMrh   c                 $   t         |   |       t        |      | _        dg| j                  j                  | _        |j
                  | _        t        j                  |j                  |j
                  d      | _	        | j                          y )Nzlm_head.weightFr  )r2   r3   r  r  rK  r  r   r  r;   lm_headr  r:   rh   r=   s     r>   r3   zZamba2ForCausalLM.__init__8  so      (
#3"Tdjj6S6S"T ++yy!3!3V5F5FUS 	r?   c                 .    | j                   j                  S r1   r  r  rd   s    r>   r%  z&Zamba2ForCausalLM.get_input_embeddingsB      zz&&&r?   c                 &    || j                   _        y r1   r`  r'  s     r>   r(  z&Zamba2ForCausalLM.set_input_embeddingsE      "'

r?   c                     | j                   S r1   r]  rd   s    r>   get_output_embeddingsz'Zamba2ForCausalLM.get_output_embeddingsH  s    ||r?   c                     || _         y r1   re  )r:   new_embeddingss     r>   set_output_embeddingsz'Zamba2ForCausalLM.set_output_embeddingsK  s	    %r?   c                     || _         y r1   r  )r:   decoders     r>   set_decoderzZamba2ForCausalLM.set_decoderN  s	    
r?   c                     | j                   S r1   rk  rd   s    r>   get_decoderzZamba2ForCausalLM.get_decoderQ  s    zzr?   num_logits_to_keepz4.50logits_to_keep)versionnew_name)output_typer  r)  r   r   r   r*  labelsr  r1  r+  r,  r   r   c                    ||n| j                   j                  }|	|	n| j                   j                  }	|
|
n| j                   j                  }
| j	                  ||||||||	||

      }|d   }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |}|
s|f|dd z   }||f|z   S |S t        |||j                  |j                  |j                        S )a"  
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

            logits_to_keep (`int` or `torch.Tensor`, *optional*):
                If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
                `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
                token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
                If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
                This is useful when using packed tensor format (single dimension for batch and sequence length).

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Zamba2ForCausalLM

        >>> model = Zamba2ForCausalLM.from_pretrained("Zyphra/Zamba2-7B-v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba2-7B-v1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)
r)  r   r   r   r*  r  r1  r+  r   r,  r   r%   losslogitsr   rO   r0  )rh   r1  r+  r1  r  r   rq   slicer]  loss_functionr  r   r   rO   r0  )r:   r)  r   r   r   r*  ru  r  r1  r+  r,  r   rq  loss_kwargsr  rO   slice_indicesry  rx  r  s                       r>   rW   zZamba2ForCausalLM.forwardT  sL   f 2C1N-TXT_T_TqTq %9$D $++JjJj 	 &1%<k$++B]B] **)%+'/!5)#  
  
8B>SV8W~ot4]kmA}a,?@A%4%%ffdooUUDY,F'+'7D7V#CVC%#33!//))
 	
r?   c           	      t   |d u }	|	sZ||d   |j                   d   k\  r|d d |j                   d    d f   }nc|j                   d   |j                   d   k7  rD|d d |f   }n:t        | j                  |j                   d   | j                  | j                        }|T|R|j                         j                  d      dz
  }|j                  |dk(  d       |	s|d d |j                   d    d f   }||	rd|i}
nd|j                         i}
|
j                  ||||| j                  j                  |d       |
S )NrC   r%   r   r.  r*  r)  )r   r   r  r   rq  r   )rJ   rg   rh   rE   rj   longrP  masked_fill_r   r   rp  )r:   r)  r   r   r*  r   r   r  r   empty_past_kvmodel_inputss              r>   prepare_inputs_for_generationz/Zamba2ForCausalLM.prepare_inputs_for_generation  sc    (4/  )!"%);;%a.*>*>q*A)A)C&CD	#~';';A'>>%a&78	6Y__Q/tzz$++O %,*>)..077;a?L%%n&91= +A	0B/B/D,DE $+];L')=)=)?@L ,#2&"0"&++"@"@"0		
 r?   )NNNNNNNNNNNr   )NNNNNT)rY   rZ   r[   r&   r3   r%  r(  rf  ri  rm  ro  r"   r   rY  r!   r   _CONFIG_FOR_DOCr   r5   r   r   rg   r   rN  r
   rq   r	   rW   r  r\   r]   s   @r>   r[  r[  7  s   | '(& )6DTU*+BC+AP_` 151537>B59-1$(,0/3&*5934X
E,,-X
 !.X
 u//0	X

 "":;X
   1 12X
 ))*X
 D>X
 $D>X
 'tnX
 d^X
 !!1!12X
 c5<</0X
 
u,,	-X
 a D VX
z 9r?   r[  a  
    The Zamba2 Model with a sequence classification head on top (linear layer).

    [`Zamba2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   X    e Zd Z fdZd Zd Z ee      	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     deeeee	j                     f      dee	j                     d	ee	j                     d
ee   dee   dee   dee   deeef   fd       Z xZS )Zamba2ForSequenceClassificationc                    t         |   |       |j                  | _        t        |      | _        | j                  j
                  | _        t        j                  |j                  | j                  d      | _	        | j                          y )NFr  )r2   r3   
num_labelsr  r  rK  r   r  r;   scorer  r^  s     r>   r3   z(Zamba2ForSequenceClassification.__init__  se      ++ (
"&**"?"?YYv114??O
 	r?   c                 .    | j                   j                  S r1   r`  rd   s    r>   r%  z4Zamba2ForSequenceClassification.get_input_embeddings  ra  r?   c                 &    || j                   _        y r1   r`  r'  s     r>   r(  z4Zamba2ForSequenceClassification.set_input_embeddings
  rc  r?   r)  r   r   r   r*  ru  r  r1  r+  r,  r   c                    |
|
n| j                   j                  }
| j                  ||||||||	|
	      }|d   }| j                  |      }||j                  d   }n|j                  d   }| j                   j
                  |dk7  rt        d      | j                   j
                  d}n||| j                   j
                  k7  j                  |j                  t        j                        }t        j                  |j                  d   |j                  t        j                        }||z  j                  d      }n.d}t        j                  | j                  j                    d       |t        j                  ||j                  	      |f   }d}||j                  |j                        }| j                   j"                  | j$                  dk(  rd
| j                   _        nl| j$                  dkD  rL|j&                  t        j(                  k(  s|j&                  t        j*                  k(  rd| j                   _        nd| j                   _        | j                   j"                  d
k(  rIt-               }| j$                  dk(  r& ||j/                         |j/                               }n |||      }n| j                   j"                  dk(  r=t1               } ||j3                  d| j$                        |j3                  d            }n,| j                   j"                  dk(  rt5               } |||      }|
s|f|dd z   }||f|z   S |S t7        |||j8                  |j:                  |j<                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   r   r   r*  r  r1  r+  r,  r   r%   z=Cannot handle batch sizes > 1 if no padding token is defined.rC   rl   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rn   
regressionsingle_label_classificationmulti_label_classificationrw  )rh   r1  r  r  rJ   r  r2  rF   rj   r5   int32rm  argmaxr5  r6  r=   rY   problem_typer  rE   r  rq   r   r  r   rK   r   r   r   rO   r0  )r:   r)  r   r   r   r*  ru  r  r1  r+  r,  transformer_outputsrO   ry  ri   last_non_pad_tokennon_pad_masktoken_indicespooled_logitsrx  loss_fctr  s                         r>   rW   z'Zamba2ForSequenceClassification.forward  s   ( &1%<k$++B]B]"jj)%+'/!5# ) 

 ,A.M* "+J&,,Q/J;;##+
a\]];;##+!#"%)A)AAEEfmmUZU`U`aL!LL)<V]]Z_ZeZefM"/,">!F!Fr!J!#>>**+ ,Z Z
 u||Jv}}MOaabYYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- 2 22t GUWY))-II,.v6#%(;AB(??F)-)9TGf$EvE/ /??-;;*55
 	
r?   rX  )rY   rZ   r[   r3   r%  r(  r   rY  r   r5   r   r   r
   r   r   r   rN  r	   r   rW   r\   r]   s   @r>   r  r    s0    '( ++BC 151537KO59-1$(,0/3&*[
E,,-[
 !.[
 u//0	[

 "%tE4E4E/F(F"GH[
   1 12[
 ))*[
 D>[
 $D>[
 'tn[
 d^[
 
u66	7[
 D[
r?   r  )r[  r  r  r  )r2  )Nr%   )_r  rM  	itertoolsr   typingr   r   r   r   r   r	   r
   r5   r   torch.nnr   r   r   activationsr   cache_utilsr   r   
generationr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   modeling_outputsr   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r    r!   utils.deprecationr"   utils.import_utilsr#   r$   configuration_zamba2r&   +mamba_ssm.ops.triton.selective_state_updater'   !mamba_ssm.ops.triton.ssd_combinedr(   r)   causal_conv1dr+   r,   
get_loggerrY   r5  r  Moduler/   r_   rg   r   r   rq   r   r   r   r  r
  r  rD  rG  rT  r  rt  rV  r  r  r  r  r  ZAMBA2_START_DOCSTRINGrY  r  r[  r  __all__r   r?   r>   <module>r     s  ,  	  D D D   A A ! . ) > B q q K F &  1 T . RmmZjW57WDD-7** 
		H	% '; ;*JBII J(i | i X%<BII %<P	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 U\\*% % %4(6J)bii J)`VU\\ VS V
(( 46FH\]^ kOryy kO\'		 'T=")) =@Abii AHG		 GT$-O $-N "B J Z|' |	|@s- sl  m
&; m
m
` kr?   