
    !&h$V                       d dl Z d dlZd dlmZ d dlmZmZmZ d dlZd dl	m
Z
 d dlmZmZmZmZmZmZ d dlmZmZmZ d dlmZmZmZ d dlmZmZ d d	lmZmZ d d
l m!Z! d dl"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8 ddl9m:Z:m;Z; ddl<m=Z=m>Z> ddl?m@Z@mAZA ddlBmCZC ddlDmEZEmFZFmGZGmHZH ddlImJZJmKZKmLZL ddlMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZ g dZ[e\e]ej                  dgZ_ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                  g
Zj ej                  ej                        Zlej                  j                  j                  j                  j                  ej                  j                  j                  j                  j                  dZrej                  j                  j                  j                  j                  ej                  j                  j                  j                  j                  dZtdee)e(f   fdZude6deve6e*f   d evewej                  j                  f   d!ee*   fd"Zzd#ee*   d$eve'ef   d%e^fd&Z{	 dnd'ed(e^d)ed*e^d+e^d,e^d!e^fd-Z|d.e6d evewej                  j                  f   d!e^fd/Z}d0ee   d!e~eej                     e^f   fd1Zde8d.e6d2ed3ed4ed!e^fd5Zd.e6d2ed3ed!e^fd6Zd.e6d evewej                  j                  f   d$eve'ef   fd7Zd8ee1   d9eee6      d2ed4ed!e^f
d:Zd.e6d evewej                  j                  f   d;e=d<ed=ee   d!e~e!e~ed>f   e=ee   f   fd?Zd@ej                  j                  dAeve1eej                  j                     f   d!dfdBZd9e0dCeew   fdDZd.e6d0edEej                  j                  d evewej                  j                  f   dFe5d!e6fdGZd9e0dHe6d2edIeeL   d4ed evewej                  j                  f   dJeve6e^f   dKee6   d!dfdLZd.e6d2edIeeL   d evewej                  j                  f   d4edJeve6e^f   d!evewef   fdMZde6d evewej                  j                  f   d$eve'ef   d%e^d!ee   f
dNZde6d evewej                  j                  f   d$eve'ef   d%e^d!eej                     f
dOZde6d.e6d evewej                  j                  f   d$eve'ef   d%e^d!ee   fdPZ	 dod.ee6ef   de8d2edEej                  j                  d evewej                  j                  f   dFe5dIeeL   d;e=d$eve'ef   d%e^d4ee   d!e8fdQZ	 dod.e6d2edEej                  j                  d evewej                  j                  f   dFe5dIeeL   d;e=d$eve'ef   d%e^d4ee   d!dfdRZd.e6dSedEej                  j                  d evewej                  j                  f   dFe5dTe^d!dfdUZd.e6dEej                  j                  d evewej                  j                  f   dFe5d$eve'ef   d%e^d!ee6   fdVZdWe6dEej                  j                  d evewej                  j                  f   dFe5d$eve'ef   d%e^d!dfdXZd.e6dYeej                  ef   dZeveweAf   d!dfd[ZdFe5dZeveweAf   d!dfd\Zd.e6dEej                  j                  d evewej                  j                  f   d!e^fd]Zd.e6dEej                  j                  d evewej                  j                  f   fd^Zd.e6d2ed evewej                  j                  f   d;e=fd_ZdEe3dZeveweAf   d`evewef   d;e=daevewef   d4edbeew   d%e^d!ee6   fdcZdEej                  j                  d%e^d evewej                  j                  f   dZed;e=d4ed!dfddZdee3d`evewef   dfevewe~ewef   f   d;e=dgevewef   dhe!d%e^dbeew   d!dfdiZ	 	 	 	 dpdEe3dhee!evewef   f   d%e^dfevewe~ewef   f   dje~ed>f   d;ee=evewef   df   dkee!evewef   df   d4eeevewef   df   dle^d!e3fdmZy)q    N)asdict)AnyOptionalUnion)
FakeTensor)_DerivedObserverOrFakeQuantizeFixedQParamsFakeQuantizeFixedQParamsObserverObserverBaseObserverOrFakeQuantizePlaceholderObserver)BackendConfigDTypeConfigget_native_backend_config)&get_fusion_pattern_to_root_node_getterget_module_to_qat_moduleget_pattern_to_dtype_configs)_is_activation_post_process_PartialWrapper)_is_reuse_input_qconfig
QConfigAny)QConfigMapping)convertpropagate_qconfig_)DerivedQuantizationSpec
EdgeOrNodeFixedQParamsQuantizationSpecQuantizationSpecQuantizationSpecBaseSharedQuantizationSpec)_parent_nameget_qconfig_dtypesget_swapped_custom_module_classNodePatternPattern)GraphModule)GraphNode)Argument   )is_equalization_observernode_supports_equalization)PrepareCustomConfigStandaloneModuleConfigEntry)_find_matches_MatchResultWithQConfig)_sorted_patterns_dict)_generate_node_name_to_qconfig_get_flattened_qconfig_dict_update_qconfig_for_fusion_update_qconfig_for_qat)_default_root_node_getter!_get_pattern_to_quantize_handlersQuantizeHandler)3_insert_dequant_stubs_for_custom_module_lstm_output_is_custom_module_lstm+_maybe_get_custom_module_lstm_from_node_arg+_qconfig_satisfies_dtype_config_constraintsall_node_args_have_no_tensorsassert_and_get_unique_deviceget_custom_module_class_keysget_new_attr_name_with_prefix(get_non_observable_arg_indexes_and_typesnode_arg_is_biasnode_arg_is_weightNON_QUANTIZABLE_WEIGHT_OPSObservedGraphModuleAttrs)insert_observers_for_modelprepare propagate_dtypes_for_known_nodesdtypeinput_act_obs_or_fq_ctroutput_act_obs_or_fq_ctr
quant_specc                 B    t        |       }t        j                  |      S N)r   copydeepcopy)rN   kwargs_dicts     v/var/www/pru.catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/torch/ao/quantization/fx/prepare.py_get_observer_kwargsrU   w   s     $K==%%    arginput_qspec_mapnamed_modulesreturnc                 v    t        | |      r| j                  d   } t        | |      r|j                  | d       S )Nr   ) _is_activation_post_process_nodeargsget)rW   rX   rY   s      rT   _get_qspec_for_argr_   ~   s:    
 +3
>hhqk +3
>sD))rV   quantization_specobs_or_fq_mapis_qatc                    | yt        | t              r0| j                  }||v sJ d| d|j                                 ||   S t        | t              r| j
                  | j                  | j                  | j                  | j                  | j                  d}| j                  }|D cg c]  }||   	 }}||d<    t        j                  d
i |       S t        | t              rDt        |       }t!        j                  d
i |}|r t#        j                  |             S  |       S t        | t$              sJ | j&                  }	t        |       }|j)                  d       |	}
t        |	t*              r|	j,                  j.                  }
d|
j0                  vr|j)                  d	         |	j                  d
i |       S c c}w )ad  Create observer or fake quantize objects based on quantization spec

    Args:
       quantization_spec: used to store parameters to create the observer or fake quantizer
       obs_or_fq_map: this is a map from edge/output to the corresponding observer/fake_quant
       instance, it may be reused for different edge/output depending on configuration
    NzTplease make sure only refer to edge or node that has observer/fake_quant inserted: 'z	' not in
)rJ   derive_qparams_fn	quant_min	quant_maxqschemech_axis
obs_or_fqs)observerobserver_or_fake_quant_ctr
PerChannelrh    )
isinstancer    edge_or_nodekeysr   rJ   rd   re   rf   rg   rh   derived_fromr   	with_argsr   rU   r
   r	   r   rk   popr   pfunc__name__)r`   ra   rb   ro   kwargsedge_or_nodeskri   observer_ctrrk   obs_or_fq_classs              rT   _create_obs_or_fq_from_qspecr|      s     #%;<(55}, 	
..:^:mFXFXFZE[]	
, \**	%'>	? ',,!2!D!D*44*44(00(00
 *660=>1mA&>
>)|A-77A&ACC	%'C	D%&78+55??L+55|LNN>!')9:::!2!M!M!"34F
JJ+, 1O,o>466;;?333

99/%//9&9;;- ?s   )Gprev_output_dtypeprev_output_is_dynamiccur_target_dtypecur_target_is_dynamicreuse_input_obs_or_fqis_zeroth_argc                     |r|t         v s
J d|        | t        vsJ |S |ry|t         v r!| t         t        j                  gz   v xr || k7  S y)a  
    note: we will treat "not specified" as torch.float for now
    utility function that checks if we should insert an observer or fake quant node
    base on the requested dtype for the nodes from user

    is_zeroth_arg: we only dynamically quantize the first arg of the node right now
      this should be removed when we enable configuring dynamic quantization
      for a specific argument, this can be removed if we deprecate fx graph mode
      quantization

    z6Expected cur_target_dtype to be torch.float, but got: F)_OBS_DTYPE_LIST_DO_NOT_OBS_DTYPE_LISTtorchfloat)r}   r~   r   r   r   r   s         rT   _needs_obs_or_fqr      sx    , /	WCDTCUV	W/ (>>>>?*EKK=!@@ 6 $55	
 rV   nodec                     t        | t        j                  j                        xr2 | j                  dk(  xr! t        |t        | j                                 S )Ncall_module)rn   r   fxr(   opr   strtarget)r   rY   s     rT   r\   r\      sI     	4' 	IGG}$	I'c$++6F(GHrV   	obs_or_fqc                 :    | y| j                   t        | dd      fS )zgGiven a constructor for observer or fake quant module, returns
    a Tuple of dtype and is_dynamic
    )NF
is_dynamicF)rJ   getattr)r   s    rT   _get_dtype_and_is_dynamicr      s%     	< GGGrV   qconfigdtype_configbackend_configc                 h   t        | t        t        f      rt        fd| D              S t        | t              syt        |       }t        |       }| xr | }|rj                  d   j                  d      }|r |       nd}	t        |	      \  }
}j                  du xsJ j                  |
k(  xr9 t        j                        t        |      k(  xr t        j                        S |rmj                  d   j                  dd      }|r |       nd}t        |      \  }}j                  }||k(  }t        j                   d      }|du xs |xr |S j                  d   j                  d	d      }|r |       nd}t        |      \  }}j"                  }|du xs ||k(  S )
z\Check if the configured qconfig for the argument
    is supported by the backend or not
    c              3   <   K   | ]  }t        |        y wrP   )(_is_input_arg_dtype_supported_by_backend).0ar   r   r   r   s     rT   	<genexpr>z;_is_input_arg_dtype_supported_by_backend.<locals>.<genexpr>  s+      
  54,
s   Ttarget_dtype_inforL   Nweight_obs_or_fq_ctrF)is_activationbias_obs_or_fq_ctr)rn   listtupleallr(   rC   rB   metar^   r   input_dtypeboolr   r<   input_dtype_with_constraintsweight_dtypeweight_dtype_with_constraints
bias_dtype)rW   r   r   r   r   	is_weightis_biasr   rL   input_act_obs_or_fqqconfig_dtypeqconfig_is_dynamicr   weight_obs_or_fqqconfig_weight_dtype_backend_config_weight_dtypedtype_matchesqconfig_satisfies_constraintsr   bias_obs_or_fqqconfig_bias_dtypebackend_config_bias_dtypes    ````                  rT   r   r     s    #e}% 
 	
 
 	
 c4 "4-ItS)G!M1'kM"&)),?"@"D"D%#
 *A#%d 	 -F-
))
 ((D0 
$$5 \,,-6H1II;BB	
 
#yy)<=AA"D 
 6J/1t";<L"Ma&2&?&?#,0KK(S\??u)
% +d2 
;;	

 "YY':;?? $
 2D+- 9. IA$0$;$;!%- ?!%>>	
rV   c                    |j                   }d}| j                  d   j                  dt              }|r |       nd}t	        |      \  }}|rt
        j                  }||k(  }t        ||j                        }	|du xs |xr |	S )zZCheck if the configured qconfig for the output
    is supported by the backend or not
    Nr   rM   )	output_dtyper   r^   _DEFAULT_FP32_OBS_OR_FQ_CTRr   r   float32r<   output_dtype_with_constraints)
r   r   r   backend_config_output_dtypeqconfig_output_dtyperM   output_act_obs_or_fqqconfig_output_is_dynamicr   r   s
             rT   %_is_output_dtype_supported_by_backendr   H  s     #/";";  #yy)<=AA"$?  '? "D  7P733 !$}}(,GGM$O;;%! '$. 77rV   c                    t        | |||      }t        | j                        dkD  r\t        | j                  d   t              r?|t
        j                  t
        j                  fv r| j                  d   j                  dk(  ryy)zCheck if observer in same graph
    when the node output is not fp32 and input is 'placeholder'
    the input is assumed to be quantized, so it is observed
    in a different place rather than not observed.
    r   placeholderFT)	_get_arg_target_dtype_as_outputlenr]   rn   r(   r   quint8uint8r   )r   rY   ra   rb   node_output_dtypes        rT   _is_observer_in_same_graphr   m  sj     8m]F 499~j1t<%,,!<<		!=0rV   patternmatched_node_patternc           	         || y|t        |      dk\  sJ t        |      }|j                  | g       }t        |      }|j                  | t              } ||      }|}	|d   }
|D ]j  }d}t        |	j                        t        |	j                  j                               z   D ]  }|xr t        ||	|||      } |xr t        |
||      }|sj y y)zCheck if the dtype configuration of a pattern is supported by
    the backend or not, and whether the qconfig satisfies constraints
    specified in the corresponding dtype config.
    Tr*   r   F)r   r   r^   r   r6   r   r]   rw   valuesr   r   )r   r   r   r   pattern_to_dtype_configsdtype_configspattern_to_root_node_getterroot_node_getter	root_node
input_nodeoutput_noder   	supportedrW   s                 rT   9_is_pattern_dtype_config_and_qconfig_supported_by_backendr     s    +4H0IQ0NNN;NK'?'C'CGR'PM"H"X266* !!56IJ&q)K% 	
(4
0A0A0H0H0J+KK 	C! &NZ,'I	
  
"G,#
	  rV   prepare_custom_configparent_qconfigparent_backend_config.c                    t        | j                        }t        ||         }t        dddd      }|j                  j                  ||      }|j                  j                  ||      }|j                  xs t               j                  |      }|j                  }	|j                  xs
 t               }|j                  xs |}
||	||
fS )z
    Returns the standalone module QConfigMapping and PrepareCustomConfig
    for `node`, assuming that the module pointed to by `node` is
    a standalone modules.
    Nrm   )r   r   typer.   standalone_module_classesr^   standalone_module_namesqconfig_mappingr   
set_globalexample_inputsr   r-   r   )r   rY   r   r   r   module_namemodule_typeconfig_entryr   r   r   s              rT   _get_standalone_module_configsr     s     dkk"K}[12K.tRtDL(BBFF\L )@@DD\L #22 n6F6Q6Q7O "00N(>>WBUBW!00I4IN^-BNSSrV   rootmodule_to_qat_modulec                 "    t        | |dd       y )NTF)mappinginplaceremove_qconfig)r   )r   r   s     rT   _qat_swap_modulesr     s     D.USrV   sc                     t        | t              r|j                  | j                         y t        | t        t
        f      r| D ]  }t        ||        y y rP   )rn   r(   addnamer   r   _add_matched_node_name_to_set)r   r   
maybe_nodes      rT   r   r     sM    &-	"''(	(4-	8. 	9J)*a8	9 
9rV   modelgraphc                 >   t        |      }|r|j                  |       t        |      r| j                  dz   }nd}t	        |      } ||      }t        |||       |||<   |j                  |       5  |j                  d|| fi       }	ddd       |	S # 1 sw Y   	S xY w)z
    Attaches `obs_or_fq` to `model`, and creates a node which calls
    `obs_or_fq` on the output of `node`.

    obs_or_fq: an instance of Observer or FakeQuantize module
    _equalization_process_activation_post_process_r   N)r>   tor+   r   r@   setattrinserting_aftercreate_node)
r   r   r   rY   r   model_deviceprefixget_new_obs_or_fq_nameobs_or_fq_namenew_obss
             rT   _insert_obs_or_fqr    s     06L\"	*55+:6B+E2NE>9-$-M.!			t	$ P##M>D7BOPNPNs   2BB	last_nodeqhandlercache_for_no_tensor_checkprocessed_nodesc                    t        | t        t        f      r| D ]  }t        ||||||||        yt        | t              rMt        | t              sJ | }	|	|v ry|j                  |	       |yt        |	|||||      }
|
|	j                  d<   yy)zSets the target_dtype_info for each node in matched_node_pattern
    Note: processed_nodes is used to ensure we only process each node once
    Nr   )rn   r   r   /_set_target_dtype_info_for_matched_node_patternr(   r   %_get_target_activation_dtype_for_noder   )r   r  r   r  r   rY   r  r  node_patternr   r   s              rT   r  r    s     &u60 
	L;)	
	 
($	/.555#?"D!? -R%-
 *;		%&1 
0rV   c           
         t        | ||      }|rdddS |t        |      \  }}}	|t        j                  k(  r%|t        j                  k(  r|	st        j                  nt        j                  }
|duxr |j                         }|duxr |j                         }d}t        | t              rY| j                  dk(  rJ| j                  |j                  v r2|j                  | j                     j                  j                  d      }d}t        | t              rY| j                  dk(  rJ| j                  |j                  v r2|j                  | j                     j                  j                  d      }|j                  |j                  t!        j"                  |
      |||j                  t%        |      ||d	S t'        j&                  t(              S )a<  
    For each op attribute in the op's input activation, output activation,
    weight, bias - returns the settings of dtype and is_dynamic we expect
    for the `quantize` call in the reference model representation, or None
    if there is no `quantize` call needed.

    For example, if we have a node corresponding to `op0` in

      x0 -> op0 -> x1

    And we want a reference quantized representation to be

      x0 -> quant_static -> dequant -> op0 -> quant_dynamic -> dequant -> x1

    Then this function will return

      {
        "input_act_obs_or_fq_ctr": MinMaxObserver.with_args(dtype=torch.quint8, is_dynamic=False),
        "output_act_obs_or_fq_ctr": MinMaxObserver.with_args(dtype=torch.quint8, is_dynamic=False),
      }

    TODO(future PR, if needed): explicitly spell out the non-Tensor
    dtypes.
    NrK   call_functionweightbiasrI   )	rL   r   r   weight_index
bias_indexrM   r   input_output_share_observers_is_standalone_module)r=   r"   r   float16r   is_general_tensor_value_opis_standalone_modulern   r(   r   r   !_pattern_complex_format_to_config_input_type_to_indexr^   
activationr  r   rr   r   rQ   +_DEFAULT_FP32_QCONFIG_FOR_TARGET_DTYPE_INFO)r   r   r  rY   r   r  args_have_no_tensors	act_dtyper   input_act_is_dynamicr   r  r  r  r  s                  rT   r	  r	  0  s   @ 9m6 '+(,
 	

 8J78S5	<!5 U]]* EMM1-	 MM  	 D JX%H%H%J 	# !) 4 X9V9V9XtT"?*~OOO)KK""33x=  
tT"?*~OOO'II""33v; 
 (/'9'9$+NN"5"?"?j"Q($(/(:(:%<W%E,F%:

 
	
 99@AArV   c                    t        | t              sJ d| j                  v r$t        | j                  d   j                  ||      S t        | |      }d}|!|j                  d   d   }|r	 |       }|S d}|S t        | |      r| j                  d   }t        |t              sJ d       d|j                  v r&t        |j                  d   j                  ||      }|S d|j                  v sJ |j                  d   d   }|r |       nd}|S d| j                  v r$| j                  d   j                  dt              }nt        }|r |       nd}|S )aC  Get the constructor for observer or fake quant object for
    the argument in the original graph as the output of previous node,
    skipping inserted observers

    We are assuming that the observers are inserted correctly, and the dtype for
    argument in quantized graph will match what is specified by the qconfig
    quantization_annotationNr   rM   r   z(Currently we only support observing Node)
rn   r(   r   r|   output_qspecr;   r\   r]   r^   r   )rW   rY   ra   rb   custom_module_lstm_noderM   r   observed_args           rT   _get_output_act_obs_or_fqr"    s    c4    CHH,+HH./<<mV
 	
 J]  $*#:#?#?@S#T&$
  +C$& 	B  A IM 	B  = 
*#}	=xx{$
 	65	6 
 %(9(99#?!!";<II$ 0  % ',*;*;;;;'3'8'89L'M*($ /G(*D !   #((*'*xx0C'D'H'H*,G($ (C$*B$& 	  rV   c                 >    t        | |||      }t        |      \  }}|S rP   )r"  r   )rW   rY   ra   rb   arg_as_output_act_obs_or_fqarg_as_output_target_dtyper   s          rT   r   r     s5     #<]M6# %>#%! &%rV   c                 $   t        | t              sJ d|j                  v rC|j                  d   j                  }t	        | ||      }|t               }|S t        |||      }|S t        ||       }t        ||       }	| xr |	 }
d}|
r$|j                  d   j                  dt
              }n[|r6|j                  t        vrG|j                  d   j                  dt
              }n#|j                  d   j                  dt
              }|r |       S dS )zcGet the observer or fake quant constructor for the Argument `arg`, as input
    to Node `node`
    r  Nr   rL   r   r   )rn   r(   r   rX   r_   r   r|   rC   rB   r^   r   rD   )rW   r   rY   ra   rb   rX   input_arg_qspecinput_arg_obs_or_fqr   r   r   obs_or_fq_ctrs               rT   _get_arg_as_input_act_obs_or_fqr*    s-    c4    !DII-))$=>NN,S/=Q""="?
 #" #?# #" #4-ItS)G!M1'kMM		"56::%'B
 
;;88 II&9:>>&(CM 		"56:: "=
 ,=?55rV   c                 P   t        |t        t        f      r@g }|D ](  }t        | |||||||||	|
      }|j	                  |       *  t        |      |      S t        |t              s|S t        |t              sJ |}|duxr |j                         }|sd| j                  v r| j                  d   j                  }n1d| j                  v sJ | j                  d   }|j                  dd      }t        || |||	      }t        |      \  }}t        ||||	      }t        |      \  }}t        |||||t        | j                         dkD  xr || j                   d   u       }n|J t#        | ||||
      \  }}}}|j$                  }d}t'        | j                         D ]  \  }}||u s|} n |d}nLt)        ||||	      }||v rt*        j,                  nt*        j.                  }||k7  xr |t*        j.                  k7  }|j0                  }|r |       nd}|rd} |j2                  j5                         D ]N  }!|!j6                  dk(  s||!j8                     }"t        |"      t        |      k(  s:|"j:                  k(  sJ|"}|!}  n |J |||| f<   | t=        |||||      }#|#}|S | }|S )	zk
    Given a `node` and an `arg`, inserts an input observer between
    `node` and `arg` if necessary.
    Nr  r   r   Fr   )r   r   )rn   r   r   -_maybe_insert_input_observer_for_arg_or_kwargappendr   r(   r  r   _reuse_input_obs_or_fqr^   r*  r   r"  r   r   r]   r   input_quantized_indexes	enumerater   r   r   r   r  usersrp   r   r   rJ   r  )$r   rW   r   r   rY   r   r  r   ra   rb   r   new_arg_to_return	inner_argnew_inner_argnew_argr  r   r   arg_as_input_act_obs_or_fqarg_as_input_target_dtypearg_as_input_target_is_dynamicr$  r%  arg_as_output_target_is_dynamicneeds_obs_or_fqr   sm_prepare_custom_configsm_input_quantized_idxscur_input_idxarg_idxarg_to_checkact_post_process_ctrexisting_obs_nodemaybe_obs_nodemaybe_obs_modnew_obs_nodes$                                       rT   r,  r,    sq   & #e}% 	4II%M $$]3	4 tCy*++c4 
c4   G#4/SH4Q4Q4S
 %		1$(II)%$$ " '$))333 !%		*= > %6$9$9'%! &E}mV&
" &&@A	
%* '@v'
# &&AB	
&+ +&+%*!dii.1,D		!1D
 """,J-!6-
)1& #;"R"R %.tyy%9 	!G\s" '	
  #O)H]M6*&
 !$;; [[ & +.GG=,;   '11&: " 	#   "iinn. 
	N  M1 -n.C.C D'40J+KK%++01 2?.(6%
	 *555%?sDk"$,/uL #G N (GNrV   c
                     g }
| j                   D ](  }t        | ||||||||||	      }|
j                  |       * i }| j                  j	                         D ]  \  }}t        | ||||||||||	      }|||<   ! t        |
      | _         || _        y)aK  
    If needed, inserts observers to the input args and kwargs of `node`.
    Note: modifies `node` inplace.

    For example, if cur_node needs an observer after prev_node, we change from

      prev_node -> cur_node

    To

      prev_node -> obs -> cur_node

    Note: backend_config only needed for standalone_module node
    N)r]   r,  r-  rw   itemsr   )r   r   r   rY   r   r  r   ra   rb   r   new_argsrW   r5  
new_kwargsry   kwarg	new_kwargs                    rT   &_maybe_insert_input_observers_for_noderK    s    8 Hyy !?!
 	 !  JKK%%' "5A!
	 "
1"" hDIDKrV   equalization_qconfig	is_branchc                    |t        | |      sy|rt        j                  d|  d       yg }| j                  D ]}  }t	        |t
              rt        | |      r|j                  |       1t        | |      }|r|j                  n|j                  }	 |	       }
t        ||
|||      }|j                  |        t        |      | _        y)z
    If `node` needs to be equalized, find the input/weight observers it needs in
    `equalization_qconfig`, creates them, and inserts it into `graph`.

    If `node` does not need an equalization observer, returns None.
    NzCannot equalize z  because it is part of a branch.)r,   warningswarnr]   rn   r(   rB   r-  rC   r  input_activationr  r   )r   rL  r   rY   r   rM  rG  rW   r   act_eq_process_ctrnew_eq_obs_modnew_eq_obs_nodes               rT   3_maybe_insert_input_equalization_observers_for_noderU    s     #+Em, 	(.NOPHyy )#t$(8s(COOC &tS1	  !''%66 	 ,-+u
 	(%)* hDIrV   c                    | j                   dk7  sJ d       d}d| j                  v r%t        | j                  d   j                  ||      }nXd| j                  v sJ | j                  d   j	                  dd      }| j                  d   j	                  d      }|r |       nd}t        |      \  }	}
d}t        t        j                  d|	|
|      }|
rd}|xr | }|r||| <   t        | ||||      S y)	a$  
    If `node` needs an output observer, creates it, inserts it into `graph`
    and returns it.

    If `node` does not need an output observer, returns None.

    Note: inserting dynamic quantization ops for output is not supported in fx graph mode
    quantization code path right now
    outputz3observer insertion for outputs is handled elsewhereFr  r   r  rM   N)
r   r   r|   r  r^   r   r   r   r   r  )r   r   rY   r   ra   rb   r  r   rM   target_dtypetarget_is_dynamicr   r:  s                rT   &_maybe_insert_output_observer_for_noderZ  2  s5   " 77hU UU  DII-;II/0==}f 
 #dii///#yy)<=AA#U 
 $(99-@#A#E#E&$
  +C$& 	 '@@T&U#L# " 'UL*;=RO  &D/C+CO2d &}e
 	
 rV   graph_output_nodec           
      .   dt         dt        j                  j                  dt        t
        t        j                  j                  f   dt        dt         f
fd| j                  D cg c]  } ||||       }}t        |      | _        yc c}w )z
    If the output needs to be quantized and there are any nodes
    in the output which are not already observed, inserts observers
    for those nodes.
    r   r   rY   r   rZ   c           	         t        | t              rt        | |      }d}t        j                  }d| j
                  v r4| j
                  d   j                  dd      }| |       }|j                  }||k7  xr |t        j                  k7  }|r|J t        | ||||      }	|	S | S t        | t        t        f      r5| D 
cg c]  }
 |
|||       }}
t        | t              r|S t        |      S t        | t              r*i }| j                         D ]  \  }} ||||      ||<    |S | yt        d|       c c}
w )a`  
        Navigate an arbitrary data structure of lists, tuples, dicts.
        For each container type, recurse on all inputs. Once any Node
        is found, insert an observer if needed and do not recurse further.

        For example, given a structure of

          {'foo1': [[bar1]], 'foo2': {'foo3': [[[bar3]]]}}

        we recurse down to bar1 and bar3, observe them if necessary,
        and if we inserted an observer then replace the original node
        with its observer.

        Returns the data structure with all nodes needing observation being
        replaced by their observers.
        Nr   rL   z!Unhandled type for returned node:)rn   r(   r   r   r   r   r^   rJ   r  r   r   dictrF  	Exception)r   r   rY   r   r%  observer_modr7  observer_clsneed_obsobserver_node
inner_noderesultsresults_dictry   inner_v&_recursive_maybe_replace_node_with_obsrb   ra   s                  rT   rh  z[_maybe_insert_observers_before_graph_output.<locals>._recursive_maybe_replace_node_with_obs  s   , j$')HM=&*&  L(-%"joo5)/BCGG-t   +#/>L0<0B0B- +.GG =-<  #/// 1e]E! %$!!
T5M2
 #-	  7}eG  *d+W~%
D)L(..0 
7"HUM5#Q  3Z )s   D;N)	r)   r   nnModuler^  r   r'   r]   r   )	r[  r   rY   r   ra   rb   old_argrG  rh  s	       ``  @rT   +_maybe_insert_observers_before_graph_outputrl    s    IIxxI C01I 	I
 
IZ )-- 	/w}eTH 
 #8_s   -BrX  &node_name_to_match_result_with_qconfigc                    d| j                   d   d<   d| j                   d   d<   |j                  | j                  d      \  }}}}}|?|j                         r.| j                  d   }t        |t              rt        |||       yyyy)z
    Assigns `target_dtype` to `node`, setting `is_dynamic` to False. If `node`
    is a general tensor shape op, also call this function recursively on
    the first argument, to propagate the dtype to the caller.
    Nr   rL   rM   NNNNNr   )r   r^   r   r  r]   rn   r(   _maybe_propagate_dtype_for_node)	r   rX  rm  
_root_noder   _patternr  _qconfig	prev_nodes	            rT   rp  rp    s     AEDII!"#<=AEDII!"#=> 	/22		1		
  C C EIIaL	i&+<)O ' !FrV   c           	      \   | j                   D ]  }t        |      }|D ]  } ||   |      }|D ]y  }|j                  |   }t        |t        t
        f      rt        |      }n|g}|D ]>  }	t        |	t        j                  j                  j                        s2t        |	||       @ {   y)a  
    Currently we assume that inputs to the graph are either `torch.float` or
    `torch.quint8`, which is not always correct. For ops such as
    `x.masked_fill(mask, value)`, we know that the dtype of  `mask` is a
    `BoolTensor`. Propagate this information throughout the graph.

    Note: not all dtypes in the graph will be correct after this pass, but a
    higher percentage of them will be correct. Hopefully in the future we can
    replace this with a better way to reason about dtypes of tensors.
    N)nodesrA   r]   rn   r   r   r   r   r   r(   rp  )
r   rm  r   non_observable_arg_dictarg_typenon_observable_indicesindexrW   arg_listcur_args
             rT   rH   rH     s      "J4"P/ 	H%F%<X%Ft%L"/ ii& cE4=1#CyH #uH' G!'588==+=+=>7#X/U	rV   c                    d}t        t        | j                              D ];  }t        | j                  |   t        t
        t        f      s,| j                  |   } n |yt        |t
        t        f      r|d   }nt        |t              r|}nyd}t        ||      st        |t              sy|j                  dk(  ryd}t        t        |j                              D ]#  }|j                  |   }t        |t              s# n |y|}|dz  }|dkD  rt        d      t        ||      st        |t              sJ |j                  }t        |t              sJ ||   }	t        |t
        t        f      rt        |      D ]  \  }
}|
dk(  rd}t        ||      sKt        |j                        dk  r y|j                  d   }|dz  }|dkD  rt        d      t        ||      sKt        |j                        \  }}t        ||   ||	        | j                  j!                         D ]8  }t        ||      sJ t        |j                        \  }}t        ||   ||	       : y)	a  
    Ensures that we share an observer
    for all input arguments as well as the output argument. In detail, given
    a graph of

      x0 -> obs0 -> op -> x2
                  /
      x1 -> obs1 /

    where node obs0 points to observer instance observer0,
    obs1 points to observer1 and obs2 points to observer2, we make nodes obs1
    and ob2 point to observer0.
    Returns: whether the operation succeeded or not
    NFr   r   r*   i'  z(Unable to find observer of previous nodeT)ranger   r]   rn   r(   r   r   r\   r   AssertionErrorr   r   r0  r!   r   r1  rp   )r   r   rY   	first_argifirst_arg_argiteration_guardtrace_back_nodetarget_to_useobs_mod_to_use	input_idx	input_argparent_namer   output_obs_nodes                  rT   (_maybe_make_input_output_share_observersr  )  sr   & I3tyy>" diilT4$78		!I )dE]+!!	It	$!
 O.}mL-.},s=--./ 	A+003O/40	 "'1U" !KLL% /}mL( mT***!((MmS)))"=1N)dE]+$-i$8 	F IyA~O6y-Py~~&* %NN1-	1$"U*()STT 7y-P !-Y-=-= >KM+.nE	F"  ::??, B/OOO()?)?@Tk*D.AB rV   c                     t        | j                  j                               }|D ]?  \  }}t        ||      sJ |j	                  |        |j
                  j                  |       A y rP   )r   r1  rF  r\   replace_all_uses_withr   
erase_node)r   r   rY   rF  r  r   s         rT   _remove_output_observerr    s_     !!#$E# 0/OOO--d3/0rV   c                     || j                      }|j                  }t        |||      }|j                  |      }t	        | j                         \  }}	t        ||   |	|       y rP   )r   float_to_observed_mappingr#   
from_floatr!   r   )
r   r   rY   r   custom_modulecustom_module_class_mappingobserved_custom_module_classobserved_custom_moduler  r   s
             rT   _swap_custom_module_to_observedr    si     "$++.M"7"Q"Q#B2G$  :DD]S$T[[1KM+&.DErV   node_name_to_qconfigequalization_config_mapobserved_node_namesc                    i }t        | j                  d            }	|j                  }
|j                  }t	               }| j
                  j                  D ](  }t        j                  t              |j                  d<   * d}d}i }i }| j
                  j                  D ]5  }|j                  dk(  r
|||<   |dz  }|j                  dk(  s,|||<   |dz  }7 |j                         D ]   }|\  }}}}}|J t        ||||||	||       " | j
                  j                  D ]  }|j                  dk(  r.||   |
v r't        j                  t              |j                  d<   @|j                  d	v r#t        ||	|      }|s^ddd
|j                  d<   q|j                  dk(  s||   |v st        j                  t              |j                  d<    t        | j
                  |       t	               }|j                         D ]  }|\  }}}}}t!        ||||      }|J j                  d   d   }|r |       nd}t#        |      \  }}|rK|dt$        t&        t(        j*                  fvsjt        ||t(        j,                  j.                  j0                  j2                  d||	||        t5        | j
                  j                        }t	               }d}d}d} i }!|D ]  }|j                  dk(  rn|j                  dv r|j7                  |j8                  d      \  }}}}}|j7                  |j8                  d      }"|j                  d   }#d|j                  v r$|#duxr t;        |j                  d   t<              }$n|#du}$|du xs |$ xr |j                  dk(   }%t!        ||||      }|%s|rt        | j                  d            }	|j                  dk7  r|J t?        ||       d}&tA        |jB                        dkD  rt;        |jB                  d   tD              rtA        |jB                  d   jF                        dkD  r|jB                  d   jF                  D ]d  }'|j7                  |'j8                  d      duxs7 |'j                  dk(  xr& t;        |	tI        |'jJ                           tL              }(|'|k7  s`|(scd}&f tO        |      })|)j7                  |tP              }* |*|      }+||+u },|,r8tS        ||| |	| j
                  |||!||
       tU        ||"| |	| j
                  |&       ||u }-|j                  d   j7                  dd      }.|j                  d   j7                  dd      }/|-rNtW        ||	||      rQtY        || |	| j
                         |jJ                  |vr|j[                  |jJ                         t]        |||	|       nt_        || |	| j
                  |!|      }0|0t5        |jF                  ja                               }1|1D ]  }2|2|0u r|2jc                  ||0        te        ||	|!|      }3|.r|3s|/rtg        || |	      sti        || |	       |b|jk                         rR|jJ                  |vrD|j[                  |jJ                         t]        |||	|       ntm        || |	| j
                  |!|       |j                  dk(  r|dz  }|j                  dk(  s|dz  }|}  | S )a$  
    Inserts observers, using the following high level algorithm:

    For each node in the graph:
      1. determine the target dtype of this node in the quantized graph, and save
           it for future steps
      2. determine the target dtype or all args and kwargs of this node
      3. if any arg or kwarg's target dtype does not match the current node's
           dtype, insert an observer
      4. if the current node needs an output observer, insert it

    For example:

    - starting graph:
        x0 -> linear -> x1

    - observed graph after processing x0:
        x0(fp32)

    - observed graph after processing linear:
        x0(fp32) -> x0_obs0(int8) -> linear(int8) -> linear_obs0(int8)

    - observed graph after processing x1:
        x0(fp32) -> x0_obs0(int8) -> linear(int8) -> linear_obs0(int8) -> x1

    After a node is processed, the naive observer placement is guaranteed to be
    complete for that node and all of its predecessors. There can be future
    passes which optimize the graph by deduplicating observers, etc.
    Fremove_duplicater   r   r   r*   rW  N)r   call_methodr  rK   rM   )r   r  r  rW  ro  valr   Tr  r   )7r^  rY   r/  output_quantized_indexessetr   rv  rQ   r  r   r   r   r  -_DEFAULT_QUINT8_QCONFIG_FOR_TARGET_DTYPE_INFOr=   rH   r   r   intr   r   r   aoquantizationr   !_default_fp32_placeholder_qconfigr   r^   r   rn   r   r   r   r]   r(   r1  r   r   r   r   r6   rK  rU  r:   r9   r   r  rZ  rp   replace_input_withr   r  r  is_custom_modulerl  )4r   rm  r  r   r  r   r  rb   r  rY   input_quantized_idxsoutput_quantized_idxsr  r   inputs_seen_counteroutputs_seen_counterplaceholder_node_to_input_indexoutput_node_to_output_indexmatch_res_with_qconfigr  r   r   r  r   r  is_supported_by_backendoutput_act_or_fq_ctroutput_act_or_fqoutput_act_dtyper   nodes_before_observation#custom_module_names_already_swappedresults_nodera   rL  this_node_dtype_infooutput_is_a_tensorskip_inserting_observersis_quantized_branchuseris_user_quantizedr   r   r   is_input_node_of_the_patternis_last_node_of_patternr  r   maybe_output_obs_node
orig_users	user_node_is_observer_in_same_graph_s4                                                       rT   rF   rF     s%   x 35 ,,e,DEM&;&S&S'<'U'U!$O!! 
)-7*
		%&

 79# 46!! &77m#4G+D11$77h0D'- A% & #I"O"O"Q 
 #	
 ###7 %		

2 !! !GG}$/59MM .2YY=.DII)* WWGG#@m%>$  $/3042		-.
 GGx+D15JJ .2YY=.DII)*?!N %; "%O"H"O"O"Q ' #	
  F-w 	 
 ###
  $yy)<=& 
 6J/1t78HI!&+;JJ	D
 ,
 <$%%--OO)	='`  $EKK$5$56 58E' L>@M ) h 77m# WWQQ 7::		9$ $;#>#>tyy$#O #'99-@#A 		!%9%E &*IIe$jK" &:%E" D;);%;(,ww(** % J17N $ ,0G $U%8%8%%8%P Q77h&/;;;1,.A +0'DII*&tyy|T:		! 2 23a7$(IIaL$6$6 ;D0D0H0H $		41%)1* 1 !%= 8 !"$.$1#dkk2B$C\%"	 .  $t|0A6: 3; ?~N 0 (C'F'F!:($ !11E FI3793D03> #!)!KK$1)"* L 0!)!KK/ /3i.?+3799=P3Q3U3U640 -1II6I,J,N,N/-) /1 -( P $e]EKK  ${{2UU C G G T ?$('=BW!" !G$($)$1$)KK$1$*!" 2  5@  .2$**//2C-D
1; !&I'04I'I(0$-$@$@(,.C%&!& %?(,m]F%& !< %A(C%:+S(,e],& )@,0%)* $,#7H<U<U<W(,/R)S )L(O(O,0KK)* )H,0,3,9,A	)* @e]EKKPV 77m#1$WW  A% LQh T rV   c           	         |j                         D ]  \  }}}}	}
|	|	j                         st        ||||
|      \  }}}}||j                     }t        j
                  j                  j                  j                  } |||||||      }t        |j                        \  }}t        ||   ||       |||j                  <    y)z
    Runs prepare_fx on each standalone module. Note: this does
    not modify the graph, it just replaces the unobserved modules with
    their observed versions.
    N)r   r   r   )r   r  r   r   r   r  r  quantize_fx_prepare_standalone_module_fxr!   r   )r   rb   rY   rm  r   r   r   r   rr  r  r   sm_qconfig_mappingsm_example_inputsr;  sm_backend_configstandalone_modulerG   observed_standalone_moduler  r   s                       rT   %_run_prepare_fx_on_standalone_modulesr  r  s    & 
0	6	6	8#E 		..0 +}&;Wn
	
$
 *)*:*:;HH!!--KK 	 &-,":,&
" ))9)9:Tk*D2LM*Di&&'G#ErV   observednode_name_to_scope!equalization_node_name_to_qconfigr   c           	      B    t        |||||||      | j                  d<   y )N)r  r  r   r  r   rb   r  _observed_graph_module_attrs)rE   r   )r  r  r  r   r  r   rb   r  s           rT   _save_stater    s.     5M1-3*K'/5HMM01rV   r   _equalization_configr  c	           
      J   |
t               }|
t               }t        |t              r1t	        j
                  dt        d       t        j                  |      }t        |t              r1t	        j
                  dt        d       t        j                  |      }t        |t              r1t	        j
                  dt        d       t        j                  |      }t        |t              r1t	        j
                  dt        d       t        j                  |      }t        |t              sJ t        |t              sJ t        j                  |      }t        j                  |      }i }	|
t               }t        |      }	t        |	      }	t        |      }
t        | |       t        | |       t!        |      }t#        | ||j%                                |r#t'        |      }t)        | |       t+        ||       t        | j-                  d            }t/        | || j0                  ||      }t/        | || j0                  ||      }t3        |j4                  j7                               }t3        |j8                  j7                               }t;        |j<                        }t?        | j0                  ||	|
|||      }i }|jA                         D ]  \  }}g |||   }|||<    tC        | |||||       tE               }tG        | |||||||      }tI        | | j0                        } tK        | |||||||       |rd|J t        |jL                  d	   tN              sJ d
       |jP                  }|jR                  }| jT                  d   }d|_+        ||_,        ||_-        | S )a9  standalone_module means it a submodule that is not inlined in
    parent module, and will be quantized separately as one unit.

    How the standalone module is observed is specified by `input_quantized_idxs` and
    `output_quantized_idxs` in the prepare_custom_config for the standalone module
    Args:
        node_name_to_scope: mapping from node name to the scope of the module which contains the node.
        The scope is a tuple of fully qualified path of the module and the type of the module
    Returns:
        model(GraphModule): prepared standalone module
        attributes related to standalone module
        in model.meta["_observed_graph_module_attrs"]:
            is_observed_standalone_module (bool): boolean value that shows whether the
            current model is a observed standalone module or not
            standalone_module_input_quantized_idxs(List[Int]): a list of
                indexes for the graph input that is expected to be quantized,
                same as input_quantized_idxs configuration provided
                for the standalone module
            standalone_module_output_quantized_idxs(List[Int]): a list of
                indexs for the graph output that is quantized
                same as input_quantized_idxs configuration provided
                for the standalone module
    zPassing a QConfig dictionary to prepare is deprecated and will not be supported in a future version. Please pass in a QConfigMapping instead.   )
stacklevelzPassing a QConfig dictionary to prepare for equalization is deprecated and will not be supported in a future version. Please pass in a QConfigMapping instead.zPassing a prepare_custom_config_dict to prepare is deprecated and will not be supported in a future version. Please pass in a PrepareCustomConfig instead.zPassing a backend_config_dict to prepare is deprecated and will not be supported in a future version. Please pass in a BackendConfig instead.Fr  r   zVstandalone module only supports returning simple value currently(not tuple, dict etc.)r  T).r-   r   rn   r^  rO  rP  FutureWarning	from_dictr   rQ   rR   r   r7   r1   r   r4   r3   r   to_dictr   r   r5   rY   r2   r   r   r   rp   r   r?   r  r/   rF  r  r  rF   r&   r  r]   r(   r/  r  r   is_observed_standalone_module&standalone_module_input_quantized_idxs'standalone_module_output_quantized_idxs)r   r   rb   r  r   r   r  r   r  pattern_to_quantize_handlerroot_node_getter_mappingflattened_qconfig_dictr   rY   r  r  r   r   custom_module_classesmatches_without_qconfigrm  	node_namematch_without_qconfigmatch_with_qconfigr  result_noder  r  observed_graph_module_attrss                                rT   rG   rG     s   D $ 3 5#-//4(L		
 )22?C&-Y		
  .778LM'.Q		
 !4 = =>S T.$'K		
 '00@o~666*N;;;mmO4O==)=> CE24"CN"S"78S"TEnUuo6u&:;8Iu46K6S6S6UV7G%!56@ ,,e,DEM )G}ekk+?AS)% :}ekk?<N
 ##8#P#P#U#U#WX $77<<>! 977 ,# ! .0*,C,I,I,K O(	(V4V6J96UV<N.y9O *. %(E,.)	K u{{+E)	 &&&+**1-t4 	
%	
4 +@*W*W ":: 	 ',jj1O&P#DH#A  	$J " 	$K LrV   )FrP   )NNNF)rQ   rO  dataclassesr   typingr   r   r   r   torch._subclassesr   torch.ao.quantizationr   r	   r
   r   r   r   $torch.ao.quantization.backend_configr   r   r   *torch.ao.quantization.backend_config.utilsr   r   r   torch.ao.quantization.observerr   r   torch.ao.quantization.qconfigr   r   %torch.ao.quantization.qconfig_mappingr   torch.ao.quantization.quantizer   r   torch.ao.quantization.quantizerr   r   r   r   r   r    torch.ao.quantization.utilsr!   r"   r#   r$   r%   torch.fxr&   torch.fx.graphr'   r(   torch.fx.noder)   	_equalizer+   r,   custom_configr-   r.   match_utilsr/   r0   pattern_utilsr1   qconfig_mapping_utilsr2   r3   r4   r5   quantize_handlerr6   r7   r8   utilsr9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   __all__r  r   r   r   r   qint8qint32r  r   int8int16int32float8_e5m2float8_e4m3fnr   rr   r   r  r  r   r  r  r  #_default_quint8_placeholder_qconfigr  rU   r^  r   ri  rj  r_   r|   r   r\   r   rJ   r   r   r   r   r   r   r   r   r   r  r   r  r  r	  r"  r   r*  r,  rK  rU  rZ  rl  rp  rH   r  r  r  rF   r  r  rG   rm   rV   rT   <module>r     ss      ' '  (  
 
 X M @ F   ! & " K K ? 0  
   " uejj$7 	LL	KK	LL	MM	KK	JJ	KK	KK		 <1;;%++N 
  %xx44<<^^ii % 5 5 = = _ _ j j/ +  %xx44<<``kk % 5 5 = = a a l l1 -&&(DDE&*	*$ 445* UXX__,-* "#	*6< 456<
$::;6< 6<~  && & &  	&
  & & 
&R
#C$89	
H./
H
8EKK $&'
HD
	D

D
 D
 	D

 "D
 
D
N"
"" " 
	"J
UXX__,- 
$::;.$g$"4:.$ $ "	$
 
$NT
TUXX__,-T /T 	T
 $M2T E#s(O%8(=:QQTDT
((//T15gtEHHOO?T6T1UT	T9 9C 9
% 88?? UXX__,-	
  
<4;%4;4; 4; '	4;
 "4; UXX__,-4;  $D$J/4; Y4; 
4;nbB
bBbB 'bB UXX__,-	bB
 "bB  $D$J/bB 
#s(^bBJA 	A UXX__,-A  
$::;A  	A 
 $%A H&	&UXX__,-& 
$::;& 	&
 ekk&16	16
16 UXX__,-16 
$::;	16
 16 $%16~ /3`
c	
`	` ` 88??	`
 UXX__,-` ` '` /` 
$::;` ` ]+` `Z /3@
@@ 88??@ UXX__,-	@
 @ '@ /@ 
$::;@ @ ]+@ 
@F- 
- -  88??-  UXX__,-	- 
 -  -  
- `N
N88??N UXX__,-N 	N
 
$::;N N d^Nb^-^-88??^- UXX__,-^- 	^-
 
$::;^- ^- 
^-B
T)* -16M1M,N 
	>##,06M1M,N# 
#L[
[88??[ UXX__,-[ 
	[|0
0xx07;C<P7Q0F
FF UXX__,-F /	F NN,06M1M,NN sJ/N /	N
 "#s(^N "N SN N d^Nb0E88??0E0E UXX__,-0E -0	0E
 /0E "0E 
0EfsJ/ S%T	"223 /	
 (,CH~ $  S 
6 OSHLAE!&SS>4S>9:S S S%T	"223	S
 #s(OS !!4d38nd!JKS  S#X DES -c3h=>S S SrV   