o
    MeM                     @   s@   d dl Z d dlZd dlZd dlZddlmZ G dd deZdS )    N   )
get_loggerc                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 ZdddZdd Z	e
dd Ze
dd Ze
dd Ze
dd Ze
dd Ze
dd Ze
dd ZdS )	Converterz
    Converter is a class object for auto parallel to convert tensors from 
    one parallel strategy to another one. Tensors will merge and slice value 
    with their strategy when strategies are different.
    c                 C   s4   |  || _| || _| || _ttj| _	dS )a^  
        Args:
            tensors_dict(dict): tensors' value of all ranks that to be converted. 
                key is tensor's name(str), value is all ranks' data(list(numpy.ndarray))
            pre_strategy(dict): tensors' distributed attribute of last training process.
                key is tensor's name(str), value is tensor's distributed attribute in last 
                training process.
            cur_strategy(dict): tensors' distributed attribute of current rank.
                key is tensor's name(str), value is tensor's distributed attribute in current
                rank.
        N)
_check_tensor_dict_tensors_dict_check_pre_strategy_pre_strategy_check_cur_strategy_cur_strategyr   loggingINFO_logger)selftensors_dictpre_strategycur_strategy r   ZD:\Projects\ConvertPro\env\Lib\site-packages\paddle/distributed/auto_parallel/converter.py__init__   s   zConverter.__init__c                 C   0   |st dt|tstdtt||S )NzC'tensors_dict' is None, the tensors to be converted cannot be None.z:The type of 'tensors_dict' should be 'dict', but got '{}'.
ValueError
isinstancedict	TypeErrorformatstrtype)r   r   r   r   r   r   .   s   
zConverter._check_tensor_dictc                 C   r   )Nz='pre_strategy' is None, there are not tensors in pre process.z:The type of 'pre_strategy' should be 'dict', but got '{}'.r   )r   r   r   r   r   r   8   s   
zConverter._check_pre_strategyc                 C   s2   |st d t|tstdtt||S )Nz<'cur_strategy' is None, there are not tensors in cur processz:The type of 'cur_strategy' should be 'dict', but got '{}'.)warningswarnr   r   r   r   r   r   )r   r   r   r   r   r	   A   s   

zConverter._check_cur_strategyTc                 C   s|  i }g }g }g }| j d | jD ]Q}|| jvr|| q|| jvr)|| q|| _|| _| j| }| j| }| j| }	zt	|||	||< W q t
yb }
 zt
dt|t|
 d}
~
ww | jD ]}|| jvrr|| qf|s| |||\}}}n|g g }}}t|t| }t|t| }|rtdt| |rtdt| |rtdt| |S )a   
        Convert tensors

        Args:
            strict(bool): whether to strict convert tensor with tensor's name. If False, it will
            convert tensors by prefix matching. Otherwise, tensors will be converted with
            their name strictly.

        Returns:
            converted tensors(dict)

        Examples:
            .. code-block:: python

                import numpy as np
                complete_tensors = np.arange(4).reshape([2, 2])
                partitial_tensors = np.split(complete_tensors, 2, axis=0)
                name = "tmp_0"
                tensors_dict = {name: partitial_tensors}
                strategy_1 = {
                    name: {
                        "process_shape": [2],
                        "process_group": [0, 1],
                        "dims_mapping": [0, -1]
                    }
                }
                strategy_2 = {
                    name: {
                        "process_shape": [2],
                        "process_group": [0, 1],
                        "dims_mapping": [-1, -1]
                    }
                }
                converter = Converter(tensors_dict, strategy_1, strategy_2)
                result = converter.convert()
                # the result's value is equal to `complete_tensors`
        zStart to convert tensors.zFail to convert tensor '{}'. Nz5tensors [{}] are not found in last training strategy.z8tensors [{}] are not found in current training strategy.zqtensors [{}] are found in pre_strategy, but are not foundin checkpoint files, please check your checkpoint files.)r   infor
   r   appendr   	_pre_name	_cur_namer   merge_and_slicer   r   r   convert_with_prefix_matchsetr   r   )r   strictr   tensor_not_in_pretensor_not_in_curZtensor_not_in_ckptZtensor_nametensor_listpre_dist_attrcur_dist_attrerrtensor_match_with_pretensor_match_with_curr   r   r   convertJ   sr   &












zConverter.convertc                 C   s   g }g }|D ]q}|}| ddkrw|d |d }|D ]Y}||v ru|| _|| _| j| }	| j| }
| j| }zt|	|
|||< W n t	y^ } zt	d
t|t|t| d }~ww | jd
|| || ||  nq	 q|||fS )N_z%Fail to convert tensor '{}' by '{}'. z'tensor [{}] is matched with tensor [{}])findrfindr"   r#   r   r   r
   r   r$   r   r   r   r   r    r!   )r   r   r(   r)   r.   r/   Zcur_nameZprefix_nameZpre_nameZpre_tensor_listr+   r,   r-   r   r   r   r%      sN   





z#Converter.convert_with_prefix_matchc                 C   s   t | tsJ tdd | D sJ ||kr(tj }|d |}| | }|S |d }|d }tt|dks<d|vrCt	
| |}n| d }tt|dksSd|vrYt	||}|S )z
        Merge tensors with previous dist_attr and slice tensors with current dist_attr

        Returns:
            tensor(numpy.narray): a tensor's value of current rank.
        c                 s   s    | ]	}t |tjV  qd S )N)r   npZndarray).0pr   r   r   	<genexpr>   s    z,Converter.merge_and_slice.<locals>.<genexpr>process_groupdims_mapping   r2   r   )r   listallpaddledistributedget_rankindexlenr&   r   merge_with_dist_attrslice_with_dist_attr)r*   r+   r,   rank_idrA   tensorZpre_dims_mappingZcur_dims_mappingr   r   r   r$      s"   
zConverter.merge_and_slicec                 C   s   ddl m} |d }|d }|d }|| d j||}g }g }|D ]#}	||	||||}
||	}|
|vrE||
 t|| | |
| q"t	|dkrUt
dt||d d }|S )z) Merge tensor with distributed attribute r;   	Resharderr:   process_shaper9   r   z)Fail to merge tensor with dist_attr '{}'.)reshardrH   Zcompute_complete_shapeshapecompute_partition_indexrA   r!   r   mergerB   r   r   r   )r*   	dist_attrrH   r:   rI   r9   complete_shapepartition_tensor_listZmerged_partitonprocesspartition_indexrA   complete_tensorr   r   r   rC      s6   

zConverter.merge_with_dist_attrc           
      C   s   |d }|d }|d }t | j|||}t | |t|}tj }t || j|||}|t	t|vr>t
dt||| }	|	S )z) Slice tensor with distributed attribute r:   rI   r9   z)Fail to slice tensor with dist_attr '{}'.)r   _get_split_indicesrK   splitrB   r>   r?   r@   _get_sliced_indexranger   r   r   )
rF   rN   r:   rI   r9   partition_index_listsliced_tensor_listrE   Zsliced_tensor_indexsliced_tensorr   r   r   rD   
  s&   

zConverter.slice_with_dist_attrc                 C   s"  ddl m} t| dkr1d}t| d d D ]\}}|d dks(|d || kr,d} nq|r1dS | s<| ||f dS d}|t| k r|| | d |\}	}
}|	dkr|
dkrgtj| | d |f|	d}ntj|| | d f|	d}| | t	
| ||| dS |d7 }|t| k sDdS dS )	a3  
        Merge partitial tensors to a complete.

        Returns:
            None

        Examples:
            .. code-block:: python

                import numpy as np
                partition_tensor_list = [(np.array([[[1.11, 1.12]]]), [[0,1],[0,1],[0,2]])]
                tensor = np.array([[[1.13, 1.14]]])
                partition_index = [[0,1],[0,1],[2,4]]

                _merge_tensor(partition_tensor_list, tensor, partition_index)
                # partition_tensor_list: [(np.array([[[1.11, 1.12, 1.13, 1.14]]]), [[0,1],[0,1],[0,4]])]
        r;   rG   Tr   FNr2   axis)rJ   rH   rB   	enumerater!   Zcompute_concat_infor5   Zconcatenatepopr   rM   )rP   rF   rR   rO   rH   Zis_complete_dataidxitemiZconcat_axisZfirst_orderZnew_partitionZ
new_tensorr   r   r   rM     sD   

zConverter.mergec              	   C   sX   g }t | j| }tj| || |d}|dkr|S |D ]}|t|||d  q|S )a  
        Slice a complete tensor.

        Returns:
            sliced_tensor_list(list): sliced tensors with 'partition_index_list'

        Examples:
            .. code-block:: python

                import numpy as np
                complete_tensor = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]])
                rank = 2
                complete_shape = [1, 1, 6]
                dims_mapping = [-1, -1, 0]
                process_shape = [3]
                process_group = [0, 1, 2]

                sliced_tensor_list = split(complete_tensor, [[], [], [2, 4]], 3)
                # [array([[[1.11, 1.12]]]), array([[[1.13, 1.14]]]), array([[[1.15, 1.16]]])]
        r[   r;   )rB   rK   r5   rU   extendr   )rS   rX   lengthrY   r\   rZ   rF   r   r   r   rU   T  s   zConverter.splitc           	      C   s   ddl m} g }|D ]"}||| |||}|r*tt|D ]}|| ||  qq
|}q
ttdd || }dd |D }|S )a  
        Get split indices of every dimension.

        Returns:
            split_indices_list(list): the split indices of every dimension of the tensor

        Examples:
            .. code-block:: python

                import numpy as np
                complete_tensor = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]])
                complete_shape = [1, 1, 6]
                dims_mapping = [-1, -1, 0]
                process_shape = [3]
                process_group = [0, 1, 2]

                index = _get_split_indices(complete_shape, dims_mapping, process_shape, process_group)
                # index: [[], [], [2, 4]]
        r;   rG   c                 S   s    t t| t|g tdg S )Nr   )r<   r&   )xyr   r   r   <lambda>  s     z.Converter._get_split_indices.<locals>.<lambda>c                 S   s   g | ]}t |qS r   )sorted)r6   rd   r   r   r   
<listcomp>  s    z0Converter._get_split_indices.<locals>.<listcomp>)rJ   rH   rL   rW   rB   rb   r<   map)	rO   r:   rI   r9   rH   Zsplit_indices_listrQ   rR   dimr   r   r   rT   v  s&   zConverter._get_split_indicesc                 C   s   ddl m} || ||||}d}t|D ]2\}}	|| dkr"|	}
n|	|||   }
|
dkr5|| d }n
|| d d |
 }||	|
  | }q|S )a  
        Get sliced_tensor's index of current rank in all sliced tensors list.

        Returns:
            sliced_tensor_index(int): the index of sliced tensor in sliced_tensor_list

        Examples:
            .. code-block:: python

                import numpy as np
                complete_tensor = np.array([[[1.11, 1.12, 1.13, 1.14, 1.15, 1.16]]])
                rank = 2
                complete_shape = [1, 1, 6]
                dims_mapping = [-1, -1, 0]
                process_shape = [3]
                process_group = [0, 1, 2]

                slice_tensor = _slice_tensor(complete_tensor, [[], [], [2, 4]], 3)
                # slice_tensor: 
                # [array([[[1.11, 1.12]]]), array([[[1.13, 1.14]]]), array([[[1.15, 1.16]]])]

                index = _get_sliced_index(rank, complete_shape, dims_mapping
                                                process_shape, process_group)
                # index: 2
        r;   rG   r   r2   )rJ   rH   rL   r]   )rE   rO   r:   rI   r9   rH   rR   Zsliced_indexra   rK   Zslice_shaperA   r   r   r   rV     s   
zConverter._get_sliced_indexN)T)__name__
__module____qualname____doc__r   r   r   r	   r0   r%   staticmethodr$   rC   rD   rM   rU   rT   rV   r   r   r   r   r      s,    
	
	^#
 


4
!
'r   )	r>   r   r   numpyr5   Zutils.log_utilsr   objectr   r   r   r   r   <module>   s   