o Nen@sdZddlZddlZddlZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZddlmZdd lmZejjZejZgd Zdd d Zdd dZddZdddZGdddeZGdddeZGdddeZ dS)z; Functions for Auto SParsity (ASP) training and inference. N) dygraph_only) global_scope program_guardlayers)ConstantInitializer)sparsity)core)#supported_layers_and_prune_func_map)_default_pruning)decorate prune_modelset_excluded_layersreset_excluded_layerscCs$|dur tj}tj||ddS)a Set parameter name of layers which would not be pruned as sparse weights. Args: param_names (list of string): A list contains names of parameters. main_program (Program, optional): Program with model definition and its parameters. If None is given, then it would be set as `paddle.static.default_main_program(). Default is None. Examples: 1. Usage of Dynamic Graph .. code-block:: python import paddle class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 100) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) prediction = self.linear1(hidden) return prediction my_layer = MyLayer() optimizer = paddle.optimizer.SGD( learning_rate=0.01, parameters=my_layer.parameters()) # Need to set excluded layers before calling decorate paddle.incubate.asp.set_excluded_layers([my_layer.linear1.full_name()]) optimizer = paddle.incubate.asp.decorate(optimizer) 2. Usage of Static Graph .. code-block:: python import paddle paddle.enable_static() class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 100) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) prediction = self.linear1(hidden) return prediction main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): input_data = paddle.static.data(name='data', shape=[None, 3, 224, 224]) label = paddle.static.data(name='label', shape=[None, 100]) my_layer = MyLayer() prob = my_layer(input_data) loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label)) # Setup exluded layers out from ASP workflow. # Please note, excluded_layers must be set before calling optimizer.minimize(). paddle.incubate.asp.set_excluded_layers([my_layer.linear1.full_name()], main_program) optimizer = paddle.optimizer.SGD(learning_rate=0.1) optimizer = paddle.static.amp.decorate(optimizer ) # Calling paddle.incubate.asp.decorate() to wrap minimize() in optimizer, which # will insert necessary masking operations for ASP workflow. optimizer = paddle.incubate.asp.decorate(optimizer) optimizer.minimize(loss, startup_program) N param_names main_program)paddlestaticdefault_main_program ASPHelperr rrQD:\Projects\ConvertPro\env\Lib\site-packages\paddle/fluid/contrib/sparsity/asp.pyr 's P  r cCstj|ddS)an Reset exculded layers setting corresponding to :attr:`main_program`. If :attr:`main_program` is None, then all configurations of excluded_layers would be cleaned. Args: main_program (Program, optional): Program with model definition and its parameters. If None is given, then this function would reset all excluded_layers. Default is None. Examples: 1. Usage of Dynamic Graph .. code-block:: python import paddle class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 100) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) prediction = self.linear1(hidden) return prediction my_layer = MyLayer() optimizer = paddle.optimizer.SGD( learning_rate=0.01, parameters=my_layer.parameters()) # Need to set excluded layers before calling decorate paddle.incubate.asp.set_excluded_layers([my_layer.linear1.full_name()]) # Reset excluded_layers, all supported layers would be included into Automatic SParsity's workflow. # Please note, reset_excluded_layers also must be called before calling sparsity.decorate(). paddle.incubate.asp.reset_excluded_layers() optimizer = paddle.incubate.asp.decorate(optimizer) 2. Usage of Static Graph .. code-block:: python import paddle paddle.enable_static() class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 100) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) prediction = self.linear1(hidden) return prediction main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): input_data = paddle.static.data(name='data', shape=[None, 3, 224, 224]) label = paddle.static.data(name='label', shape=[None, 100]) my_layer = MyLayer() prob = my_layer(input_data) loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label)) # Setup exluded layers out from ASP workflow. # Please note, excluded_layers must be set before calling optimizer.minimize(). paddle.incubate.asp.set_excluded_layers([my_layer.linear1.full_name()], main_program) # Reset excluded_layers, all supported layers would be included into Automatic SParsity's workflow. # Please note, reset_excluded_layers also must be called before calling optimizer.minimize(). paddle.incubate.asp.reset_excluded_layers(main_program) optimizer = paddle.optimizer.SGD(learning_rate=0.1) optimizer = paddle.static.amp.decorate(optimizer ) # Calling paddle.incubate.asp.decorate() to wrap minimize() in optimizer, which # will insert necessary masking operations for ASP workflow. optimizer = paddle.incubate.asp.decorate(optimizer) optimizer.minimize(loss, startup_program) rN)rrrrrrr}sVrcCs t|S)a Wrap the given optimizer as a OptimizerWithSparsityGuarantee, If runnig with dynamic graph mode. ASP would creates mask variables for supported parameters. Else if in static graph mode, ASP would creates mask variables and inserts necessary ops when calling minimize() Args: optimizer (Optimizer): A Optimizer used for training. Returns: OptimizerWithSparsityGuarantee: A wrapper for ASP to decorate `minimize` function of the given optimizer. Examples: 1. Usage of Dynamic Graph .. code-block:: python import paddle class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 32) self.linear2 = paddle.nn.Linear(32, 32) self.linear3 = paddle.nn.Linear(32, 10) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) hidden = self.linear1(hidden) hidden = self.linear2(hidden) prediction = self.linear3(hidden) return prediction my_layer = MyLayer() optimizer = paddle.optimizer.SGD( learning_rate=0.01, parameters=my_layer.parameters()) # Calling paddle.incubate.asp.decorate() to wrap step() in optimizer, which # will apply necessary masking operations for ASP workflow. # In dynamic graph mode, ASP would create related mask variables during decoration. optimizer = paddle.incubate.asp.decorate(optimizer) 2. Usage of Static Graph .. code-block:: python import paddle paddle.enable_static() class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 100) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) prediction = self.linear1(hidden) return prediction main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): input_data = paddle.static.data(name='data', shape=[None, 3, 224, 224]) label = paddle.static.data(name='label', shape=[None, 100]) my_layer = MyLayer() prob = my_layer(input_data) loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label)) optimizer = paddle.optimizer.SGD(learning_rate=0.1) # Calling paddle.incubate.asp.decorate() to wrap minimize() in optimizer, which # will insert necessary masking operations for ASP workflow. # In static graph mode, ASP creates related mask variables # during minimize(). optimizer = paddle.incubate.asp.decorate(optimizer) optimizer.minimize(loss, startup_program) )rr ) optimizerrrrr s Sr mask_1dTc Cstj}t|}tjjtjjtjjd}||vsJdd}t |tj j r+t j }n3t |tjjrUt j}t|drT|jddkrTtjrTttjdd} t| }n td t|||||||||d S) a Pruning parameters of supported layers in :attr:`model` via specified mask generation function given by :attr:`mask_algo`. This function supports both training and inference controlled by :attr:`with_mask`. If :attr:`with_mask` is True, it would also prune parameter related ASP mask Variables, else only prunes parameters. *Note*: (Static graph mode) If calling this function with :attr:`with_mask`, it should call `OptimizerWithSparsityGuarantee.minimize` and initialization (`exe.run(startup_program`)) before (For successfully obtain mask Variable). Typically set `with_mask` as true for training (have called `OptimizerWithSparsityGuarantee.minimize`) and false for inference only. To obtain OptimizerWithSparsityGuarantee, please see `paddle.incubate.asp.decoreate()`. Args: model (Program|nn.Layer): Program with model definition and its parameters, or a object of `paddle.nn.Layer`. n (int, optional): n of `n:m` sparse pattern. Default is 2. m (int, optional): m of `n:m` sparse pattern. Default is 4. mask_algo (string, optional): The function name to generate spase mask. Default is `mask_1d`. The vaild inputs should be one of 'mask_1d', 'mask_2d_greedy' and 'mask_2d_best'. with_mask (bool, optional): To prune mask Variables related to parameters or not. Ture is purning also, False is not. Default is True. Returns: dictionary: A dictionary with key: `parameter name` (string) and value: its corresponding mask Variable. Examples: 1. Usage of Dynamic Graph .. code-block:: python import paddle import numpy as np class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 32) self.linear2 = paddle.nn.Linear(32, 32) self.linear3 = paddle.nn.Linear(32, 10) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) hidden = self.linear1(hidden) hidden = self.linear2(hidden) prediction = self.linear3(hidden) return prediction my_layer = MyLayer() loss_fn = paddle.nn.MSELoss(reduction='mean') optimizer = paddle.optimizer.SGD( learning_rate=0.01, parameters=my_layer.parameters()) # Calling paddle.incubate.asp.decorate() to wrap step() in optimizer, which # will apply necessary masking operations for ASP workflow. # In dynamic graph mode, ASP would create related mask variables during decoration. optimizer = paddle.incubate.asp.decorate(optimizer) # Must call paddle.incubate.asp.decorate() first before calling paddle.incubate.asp.prune_model() paddle.incubate.asp.prune_model(my_layer, mask_algo='mask_2d_best') for i in range(10): imgs = paddle.to_tensor( np.random.randn(64, 3, 32, 32), dtype='float32', stop_gradient=False) labels = paddle.to_tensor( np.random.randint(10, size=(64, 1)), dtype='float32', stop_gradient=False) output = my_layer(imgs) loss = loss_fn(output, labels) loss.backward() optimizer.step() optimizer.clear_grad() 2. Usage of Static Graph .. code-block:: python import paddle import numpy as np paddle.enable_static() class MyLayer(paddle.nn.Layer): def __init__(self): super(MyLayer, self).__init__() self.conv1 = paddle.nn.Conv2D( in_channels=3, out_channels=4, kernel_size=3, padding=2) self.linear1 = paddle.nn.Linear(4624, 32) self.linear2 = paddle.nn.Linear(32, 32) self.linear3 = paddle.nn.Linear(32, 10) def forward(self, img): hidden = self.conv1(img) hidden = paddle.flatten(hidden, start_axis=1) hidden = self.linear1(hidden) hidden = self.linear2(hidden) prediction = self.linear3(hidden) return prediction main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): input_data = paddle.static.data(name='data', shape=[None, 3, 32, 32]) label = paddle.static.data(name='label', shape=[None, 1]) my_layer = MyLayer() prob = my_layer(input_data) loss = paddle.mean(paddle.nn.functional.square_error_cost(prob, label)) optimizer = paddle.optimizer.SGD(learning_rate=0.1) # Calling paddle.incubate.asp.decorate() to wrap minimize() in optimizer, which # will insert necessary masking operations for ASP workflow. # In static graph mode, ASP creates related mask variables # during minimize(). optimizer = paddle.incubate.asp.decorate(optimizer) optimizer.minimize(loss, startup_program) device = paddle.device.get_device() place = paddle.set_device(device) exe = paddle.static.Executor(place) exe.run(startup_program) # Must call exe.run(startup_program) first before calling paddle.asp.prune_model() paddle.incubate.asp.prune_model(my_layer, mask_algo='mask_2d_best') # it also be accepted to call # paddle.incubate.asp.prune_model(main_program, mask_algo='mask_2d_best') for i in range(10): imgs = np.random.randn(64, 3, 32, 32).astype('float32') labels = np.random.randint(10, size=(64, 1)).astype('float32') exe.run(main_program, feed={'data':imgs, 'label':labels}) )rZmask_2d_greedyZ mask_2d_bestzNThe "mask_algo" should be one of ["mask_1d", "mask_2d_greedy", "mask_2d_best"]Ndistributed_info_Zsharding_degreeZFLAGS_selected_gpusrzDmodel should be paddle.nn.Layer or paddle.static.Program, but got {}nm mask_algo with_mask)rdeviceZ get_deviceZ set_devicerMaskAlgoMASK_1DZMASK_2D_GREEDYZ MASK_2D_BEST isinstancennZLayerrprune_model_by_layerrZProgramprune_model_by_programhasattrrfluidZis_compiled_with_cudaintosenvirongetZ CUDAPlace TypeErrorformattype) modelr r!r"r#r$placeZMaskAlgo_mapping prune_funcZgpu_idrrrr ,sB      r c@s\eZdZdZddZddZddZdd Zd d Ze d d Z e ddZ e ddZ dS)ProgramASPInfoa$ ProgramASPInfo is a container to keep ASP relevant information of Pragrom. It contains three inner-variables: 1. __mask_vars (Dictionary): Key is parameter's name and vaule is its corresponding sparse mask Variable object, which is created by `ASPHelper.create_mask_variables`. 2. __masks (Dictionary): Key is parameter's name and vaule is its corressponding sparse mask Numpy array, which is created by `ASPHelper.prune_model`. 3. __excluded_layers (List): It stores name of layers which should not involve into ASP workflow. cCsi|_i|_g|_dSN)_ProgramASPInfo__mask_vars_ProgramASPInfo__masks _ProgramASPInfo__excluded_layersselfrrr__init__s zProgramASPInfo.__init__cC||j|<dSr8r9r= param_namevarrrrupdate_mask_varszProgramASPInfo.update_mask_varscCr?r8r:rArrr update_masksrEzProgramASPInfo.update_maskscCs|jt|dSr8)r;extendcopydeepcopy)r=rrrrupdate_excluded_layerssz%ProgramASPInfo.update_excluded_layerscCs g|_dSr8r;r<rrrr z$ProgramASPInfo.reset_excluded_layerscC|jSr8r@r<rrr mask_varszProgramASPInfo.mask_varscCrNr8rFr<rrrmasksrPzProgramASPInfo.maskscCrNr8rLr<rrrexcluded_layersrPzProgramASPInfo.excluded_layersN) __name__ __module__ __qualname____doc__r>rDrGrKrpropertyrOrQrRrrrrr7s  r7c@seZdZdZdZdZiZeddZed$ddZ e d d Z edd d e j jd fddZed d e j jd fddZe ddZe ddZeddZeddZeddZe    d%ddZeeddZed d!Zed"d#ZdS)&ra0 ASPHelper is a collection of Auto SParsity (ASP) functions to enable 1. training models with weights in 2:4 sparse pattern on FP16 or 1:2 sparse pattern on FP32 from scratch. 2. pruning well-trained models into 2:4 sparse pattern on FP16 or 1:2 sparse pattern on FP32 for fine-tuning. Zasp_maskZw_cCs||}||dS)z This is the implementation of `sparsity.set_excluded_layers`, for details please see explanation in `sparsity.set_excluded_layers`. N)_get_program_asp_inforK)clsrrasp_inforrrr s zASPHelper.set_excluded_layersNcCs8|dur|jD] }|j|qdS||dS)z This is the implementation of `sparsity.reset_excluded_layers`, for details please see explanation in `sparsity.reset_excluded_layers`. N)_ASPHelper__asp_inforrX)rYrprogrrrrs  zASPHelper.reset_excluded_layerscCs4trtj}tj}t|||jt|S)z This is the implementation of `sparsity.decorate`, for details please see explanation in `sparsity.decorate`. ) rin_dynamic_moderrdefault_startup_programr_create_mask_variables_parameter_listOptimizerWithSparsityGuarantee)r main_progZ startup_progrrrr s  zASPHelper.decoraterrTcCs|dur tj}||}|D]i}t||jr}t |j } t | } t|j} | | ||||j\} } | | j} | | ||rvt t|j}|duscJdt|j| }| t |j} || |||j| q|jS) This is the implementation of `sparsity.prune_model`, for details please see explanation in `sparsity.prune_model`. NzCannot find {} variable, please call optimizer.minimize (paddle.sparsity.decorate(optimizer).minimize(loss) and initialization (exe.run(startup_program)) first!)rrrrX global_blockall_parametersr_is_supported_layernamerZfind_varZ get_tensornparray_get_prune_func_by_nameastypedtypeset_get_mask_namer2rGrQrI)rYr5rr r!r"r#rZparamZ weight_tensorweight_nparrayr6weight_pruned_nparrayweight_sparse_maskweight_mask_paramZweight_mask_tensorrrrr**s<          z ASPHelper.prune_model_by_programcCs trftj}||}|D]N} t|| jr`| } t | j} | | |||| j\} } | | j } | | |rY|j| jd}|dusTJdt| j| | || j| q|jSd}|D]} | jj}ql|dus{Jdtj||||||dS)rcNzZCannot find {} variable, please call sparsity.decorate() to decorate your optimizer first!z6Cannot get paddle.static.Program from Paddle.nn.Layer.r)rr]rrrX parametersrrfrgnumpyrjrkrl set_valuerOr0r2rnrGrQrIblockprogramr*)rYr5layerr r!r"r#rrZrorpr6rqrrrsZtarget_programrrrr)UsL           zASPHelper.prune_model_by_layercCs|dtjS)z Return mask name by given parameter name :attr:`param_name`. Args: param_name (string): The name of parameter. Returns: string: The mask name of :attr:`param_name`. .)rMASK_APPENDDED_NAME)rBrrrrns zASPHelper._get_mask_namecCs:g}|D]}|jd}tj|vr||q|S)aF Get all parameters's Variables in :attr:`main_program` but excluded ASP mask Variables. Args: main_program (Program): Program with model definition and its parameters. Returns: list: A list of parameter Variables in :attr:`main_program` (excluded ASP mask Variables). rz)rdrergsplitrr{append)rZvar_listroparam_name_listrrr_get_not_ASP_relevant_varss    z$ASPHelper._get_not_ASP_relevant_varscCs ||jvr t|j|<|j|Sr8)r[r7)rYrrrrrXs   zASPHelper._get_program_asp_infocCs|d}tj|vr dS||jD] }||vrdSq|tvr"dSt|dkr*dS|d}|d}|d|d}tj|vrBdS|tvsJ|tvrLdSdS)a Verify if given :attr:`param_name` is supported by ASP. Args: param_name (string): The name of parameter. Returns: bool: True if it is supported, else False. Examples: .. code-block:: python from paddle.static.sparsity.asp import ASPHelper main_program = paddle.static.Program() startup_program = paddle.static.Program() with paddle.static.program_guard(main_program, startup_program): input_data = paddle.static.data(name='data', shape=[None, 128]) fc = paddle.static.nn.fc(x=input_data, num_flatten_dims=-1, size=32, activation=None) for param in main_program.global_block().all_parameters(): ASPHelper._is_supported_layer(main_program, param.name) # fc_0.w_0 -> True # fc_0.b_0 -> False rzFTrrN_) r|rr{rXrRr lenrfindPADDLE_WEIGHT_SUFFIX)rYrrBr~ryparam_name_no_weight_suffixZparam_type_suffix layer_namerrrrfs,    zASPHelper._is_supported_layercCsXt|d}|dd}|durt|d}|dur*|d|d}t|t}|S)Nrzrr)r r0r|rr )rYrBfuncrrrrrrjs z!ASPHelper._get_prune_func_by_namec Csh|dur|jj}|durtj}|j||||d\}}dd|D} |||| ||| ||fS)a This function is a decorator of `minimize` function in `Optimizer`. There are three steps: 1. Call :attr:`optimizer`.minimize(:attr:`loss`) 2. Create sparse mask Tensors according to supported layers in :attr:`main_program`. 3. Insert masking ops in the end of parameters update. *Note*: Please use `ASP.decorate` instead when applying distributed training with `Fleet`. (Due to there is a invisiable graphs optimization in `Fleet.minimize()` which make training graph cannot be modified anymore.) Args: optimizer (Optimizer): A Optimizer used for training. loss (Variable): A Variable containing the value to minimize. main_program (Program, optional): Program with model definition and its parameters. Default is `loss.block.program`. startup_program (Program, optional): Program for initializing parameters in `parameter_list`. Default is `paddle.static.default_startup_program()`. parameter_list (Iterable, optional): Iterable of `Variable` or `Variable.name` to update to minimize `loss`. The default value is None, at this time all parameters will be updated. no_grad_set (set, optional): Set of `Variable or `Variable.name` that don't need to be updated. The default value is None. Returns: list: operators from :attr:`optimizer`.minimize(:attr:`loss`). list: pairs of parameters and their gradients. N) no_grad_setcSsg|]}|dqS)rr).0Zpgrrr sz'ASPHelper._minimize..)rwrxrrr^minimizer__insert_sparse_mask_ops) rYrlossrstartup_programparameter_listrZ optimizer_opsZparams_and_gradsZ params_onlyrrr _minimizes   zASPHelper._minimizecCsR|tj}tjjt||j WddS1s"wYdSa This function is a decorator of `step` function in `Optimizer`. There are three steps: 1. Call :attr:`optimizer`.step() 2. Mask parameters with sparse masks. *Note*: Please use `ASP.decorate` instead when applying distributed training with `Fleet`. (Due to there is a invisiable graphs optimization in `Fleet.minimize()` which make training graph cannot be modified anymore.) Args: optimizer (Optimizer): A Optimizer used for training. N) steprrrr,ZdygraphZno_gradrrr`)rYrrbrrr_step"s "zASPHelper._stepc Cs||}t||:|D].}t||jr;|j|jvr;tjt|j|j |j t ddd}d|_ d|_ ||j|q WddS1sGwYdS)a Create sparse mask Tensors according to supported layers in :attr:`main_program`. This function is called in second step of `ASPHelper._minimize` Args: main_program (Program): Program with model definition and its parameters. startup_program (Program): Program for initializing parameters. params (list): Variable parameters. g?)value)rgshaperlZdefault_initializerTFN)rXrrrfrgrOrZcreate_parameterrnrrlrZ stop_gradientZ trainablerD)rYrrparamsrZroZ mask_paramrrrr_9s"   "z ASPHelper._create_mask_variablesc Csb|}||}|D]#}|j|jvr.|jd||j|jdd|iddddtttjidq d S) a! Insert masking ops in the end of parameters update. This function is called in third step of `ASPHelper._minimize` Args: main_program (Program): Program with model definition and its parameters. params (list): Variable parameters. Zelementwise_mul)XYZOutZaxisZ use_mkldnnF)r3ZinputsZoutputsattrsN) rdrXrgrOZ append_op OP_ROLE_KEYr-OpRoleZOptimize)rYrrrwrZrorrrrRs     z!ASPHelper._insert_sparse_mask_opsr8)NNNN)rSrTrUrVr{rr[ classmethodr r staticmethodr rr%r&r*r)rnrrXrfrjrrrr_rrrrrrs^    * 5    6  ,  rc@sTeZdZdZddZddZ   dddZed d Zed d Z ed dZ dS)raa OptimizerWithSparsityGuarantee is a wrapper to decorate `minimize` function of given optimizer by `_minimize` of ASPHelper. The decorated `minimize` function would do three things (exactly same as `ASPHelper._minimize`): 1. Call `minimize` function of given optimizer. 2. Call `ASPHelper._create_mask_variables` to create mask Variables. 3. Call `ASPHelper._insert_sparse_mask_ops` to insert weight masking ops in the end of `loss`'s Program. cCs ||_dSr8) _optimizer)r=rrrrr>vrMz'OptimizerWithSparsityGuarantee.__init__cCs t|j|Sr8)getattrr)r=itemrrr __getattr__ys z*OptimizerWithSparsityGuarantee.__getattr__NcCstj|j||||dS)a0 This function is to call `ASPHelper.minimize()` and return its return Args: loss (Variable): A Variable containing the value to minimize. startup_program (Program, optional): Program for initializing parameters in `parameter_list`. Default is `paddle.static.default_startup_program()`. parameter_list (Iterable, optional): Iterable of `Variable` or `Variable.name` to update to minimize `loss`. The default value is None, at this time all parameters will be updated. no_grad_set (set, optional): Set of `Variable or `Variable.name` that don't need to be updated. The default value is None. Returns: list: operators from :attr:`optimizer`.minimize(:attr:`loss`). list: pairs of parameters and their gradients. )rrr)rrr)r=rrrrrrrr|s z'OptimizerWithSparsityGuarantee.minimizecCst|jdSr)rrrr<rrrrsz#OptimizerWithSparsityGuarantee.stepcCsF|j}ttj}|jD]\}}| t ||iq|S)z This function is a decorator of `state_dict` function in `Optimizer`. Returns: state_dict(dict) : dict contains all the Tensor used by optimizer ) r state_dictrrXrrrrOitemsupdatern)r=rrZrBrCrrrrs z)OptimizerWithSparsityGuarantee.state_dictcCsnttj}|jD]#\}}t|}||vs!Jd|| ||| || q |j |S)z This function is a decorator of `set_state_dict` function in `Optimizer`. Args: state_dict(dict) : Dict contains all the Tensor needed by optimizer Return: None zThe {} is not found.)rrXrrrrOrrnr2rvrGrurset_state_dict)r=rrZrBrCZparam_mask_namerrrrs    z-OptimizerWithSparsityGuarantee.set_state_dict)NNN) rSrTrUrVr>rrrrrrrrrrrams   rar8)rrrT)!rVr.rIrurhrZpaddle.fluid.frameworkrZ paddle.fluidrrrZpaddle.fluid.initializerrZpaddle.fluid.contribrrZ2paddle.fluid.contrib.sparsity.supported_layer_listr r Zop_proto_and_checker_makerrZkOpRoleAttrNamer__all__r rr r objectr7rrarrrrs2         VY V)&u