o Ne%u@s8ddlmZddlmZddlmZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlm Z ddl!m"Z"ddl!m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl(m*Z*ddl(m+Z+ddl,m-Z-ddl.m/Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z:m;Z;ddlda?d d!Z@d"d#ZAd$d%ZBd&d'ZCd(d)ZDd;d+d,ZEd-d.ZFd/d0ZGd._init_contextzOnly support CUDAPlace for now.T)rrFr CUDAPlaceZdev_idr dygraphparallelParallelStrategyrrrsrn_parallel_context_initializedr#rdisable_dygraphZenable_dygraph)rxrzr'rwr(prepare_distributed_contexts.       rcsd}d}ttrtjg}jg}||fStttfr0ddD}ddD}||fSttrKfddD}fddD}||fSdS)z=Get input shape list by given inputs in Model initialization.NcSsg|]}t|jqSr'r$shape.0inputr'r'r( z&_update_input_info..cSg|]}|jqSr'dtyperr'r'r(rcsg|] }t|jqSr'rrr-rar'r(rcsg|]}|jqSr'rrrr'r(rr)r#Inputr$rrr%dict)rashapesdtypesr'rr(_update_input_infos    rcseZdZdZfddZeddZejddZd!d d Zd"d d Z d dZ ddZ ddZ ddZ ddZddZd"ddZddZddZdd ZZS)#StaticGraphAdapterz6 Model traning/inference with a static graph. cstt|||_t|_t|_i|_ i|_ i|_ d|_ d|_ i|_i|_ddddd|_tj|_tj|_d|_i|_i|_d|_dS)NrZ eval_totalZ test_total eval_batchZ test_batchO0)superr__init__modelrdefault_startup_program _startup_progdefault_main_program _orig_prog _label_vars _input_vars _endpoints_loss_endpoint _executor_progs_compiled_progs _merge_countrrF_nranksrr _local_rank _amp_level _amp_configs_amp_custom_lists_use_fp16_guard)selfr __class__r'r(rs,     zStaticGraphAdapter.__init__cC|jjSr"rmoderr'r'r(rzStaticGraphAdapter.modecC ||j_dSr"rrr&r'r'r(r NTcCs2|jjsJdd|_|dusJd|||S)N4model not ready, please call `model.prepare()` firsttrainTz9Does not support `update == False` in static mode by now.)r _optimizerr_run)rralabelsupdater'r'r( train_batch#s  zStaticGraphAdapter.train_batchcCsd|_|||S)Nevalrr)rrarr'r'r(r* zStaticGraphAdapter.eval_batchcCsd|_||dS)Ntestr)rrar'r'r( predict_batch.rz StaticGraphAdapter.predict_batchcO|jjj|i|Sr"rnetwork parametersrargskwargsr'r'r(r2zStaticGraphAdapter.parametersc Csdd}tj|}|dksJdtj|}|r%tj|s%t||d}||jj||j dd}|dusC|jj durEdS|d}dd t t |D}|sYdS|||dS) NcSsV|sdSdd|D}t|d}t||WddS1s$wYdS)NcSs(i|]\}}|t|trt|n|qSr')r#r r3)rkvr'r'r( :sz:StaticGraphAdapter.save.._save..wb)itemsopenpickledump)statepathfr'r'r(_save7s "z&StaticGraphAdapter.save.._savez+path should be of 'dirname/filename' format .pdparamsr.pdoptcSsi|]}|j|qSr'r-)rpr'r'r(rMsz+StaticGraphAdapter.save..)rkrbasenamedirnameexistsmakedirsrr state_dictrgetrfilterr list_vars) rrrbasedir_nameZ param_pathprogZ optim_pathoptimr'r'r(save5s$    zStaticGraphAdapter.savecCsz|jdurttj}n|jj}tjdd|Dt||D] \}}|||q"|j j r3|s5dS| ||dS)NcSsg|]\}}|qSr'r')rparamrr'r'r(r_z+StaticGraphAdapter.load..) rrrtCPUPlace_default_executorr_create_loaded_parameterr _set_varrr_load_optimizer)rparam_state_pairs optim_stateexecutorrrr'r'r(loadWs   zStaticGraphAdapter.loadcCs|jdd}ttt|}|sdStj|t |t |}|D]}|j dvrHd|vr8t |ddn|dd}|durG|||j <n|j drU|j |vrTq#n|j |vr|jjj}|jjjj} d} |jjjD]r} |durv| n | t|dd} |jjj| D]V\} }| durt|ddd d D].}| d |dur| n|d }||r|t|dd t|}|t| d |} q| d | d | d }||||j <qqn|j |vsJd |j ||||j q#dS)Nr@LR_DECAY_COUNTER@ global_steprrrZlearning_rate_cSst|Sr"r5rEr'r'r(sz4StaticGraphAdapter._load_optimizer..Tkeyreverse__0z,variable [{}] is not in optimizer state file)rrr$rr rrrrr rr-r/r0pop startswithrr_namer__name__Z _accumulatorskeysr5rsortedfindformatr)rrrrrconverted_stater1Z state_valopt_name opt_cls_name opt_unq_namer- accum_name param_name state_varZ state_keyprefixZ prefix_offset dy_state_namer'r'r(rjs               z"StaticGraphAdapter._load_optimizercCstt|j}|}|rt}n|rt }ntj }| |t |}|||dSr")r r,r-r._placeZ is_cpu_placerrZis_cuda_pinned_placeZCUDAPinnedPlacerZPlaceZ set_placer{Z gpu_device_idset)rr1Zndarrayr2rrxr'r'r(rs   zStaticGraphAdapter._set_varcs|j|jd}|sJdt|}|durt|}t|t|j|jks*Jdi}dd|j|jD}dd|j|jD}t|D]D\}}||durV||||<|jdkr||tj j j krt ||tj ry||tj j j ||<qFt ||tjr||d||<qF|durt|j|jD] \}} |||| j<q|j|j} |jdkr| d } nt| d \} } | d | } t| d }g}d gt| }t| D]\}}|j|vr|j||<q||q|jj|||d d}t|D]\}}t|dkr ||||qdd|D}|jdkr |ddSt||d| }g}t|jj|D]z\}}|jdkr|jjdurt |jjt r|j!dkrt|jjj"|dj#d}|j$|jdd|krfdd|D}d|j$|jd<t%|j$|jd<n|j$|jd|7<||j$|jd<||j&|q2|rt|r|d||fS|r|d|S|S)N7Model is not ready, please call `model.prepare()` firstzGnumber of inputs does not match number of arguments of `forward` methodcSrr'rrrr'r'r(rrz+StaticGraphAdapter._run..cSrr'rrr'r'r(rrO2Zfloat16routputmetriclossrF)feed fetch_listZ return_numpyrcSg|]}t|qSr')r/r0rr'r'r(rrrr_totalcs$g|]}|dtdfqS)N.rT)rs current_count total_sizer'r(r s_batch)'rrrr)r5r enumeraterrrirjZFP16r#Z LoDTensorZ_as_typer/r0Zastyperr-rr:rr4rruinsertr<zipr_metrics_test_dataloaderrrdatasetrrrTr)rrar compiled_progr input_namesZ input_dtypesidxnrrWrZ metric_listZ metric_splitsZnum_lossZpruned_fetch_listZpruned_fetch_idx_name_mapiZ fetch_varZretsr-Z metric_statesmetricsrrsamplesr'r r(rs               zStaticGraphAdapter._runcCs2gd}|D]}||||j||qdS)N)rrr) _make_program_compile_and_initializer)rmodesrr'r'r(prepares  zStaticGraphAdapter.preparecsj|d}|dur dSj}|dkr't|jD] }|dq|dkrKjj rKjj j rKjj j j}|j |j }|jj j |<g}g}t |jjj}jjrdjjng} ddt|D}ddt| D} | j|<tjjj|} |dkrjjrjj| | }jdkr|dkrfdd| D} |dkrfd d| D} |dkr̈jjD]} |t| j| | q|dkrWjj rWt j|_jdkrtjd d } t !| t "} j#d krd | _$j%&| _'| j'(j)j#d k| j'd<t j*jj | dj_ n3j#d krOt+j,rOj)r5t-j.j$j/dij)nd}t-j.j$j0jj f|j#d kj1dj%j_ jj 2jWdn 1sbwY|dkrr|jd d}|j3|<|j|<| t||dj4|<dS)NrrcSg|]}|qSr'Z_create_feed_layerrrr'r'r(r:rz4StaticGraphAdapter._make_program..cSr5r'r6r7r'r'r(r;rrrcg|]}t|jqSr'rGrrorr'r(rCcr8r'r9rr6rr'r(rEr<T)Z is_collectiverr use_pure_fp16)ry) amp_listsr>use_fp16_guardZfor_test)rrrr')5rrrcloner$rfopsZ _remove_oprrZ_learning_rate_mapvarsr-rZ program_guardr_inputs_labelsr)rrforward_lossrr'r4computelayerssumrrZPaddleCloudRoleMakerrinitZDistributedStrategyramprcopy amp_configsrrZdistributed_optimizerrrgpaddleZstaticZAutoMixedPrecisionListsdecoraterminimizerr)rrropZlr_varZ new_lr_varlossesr/rarrbrZroleZ dist_strategyr?r'rr(r1 s           /   z StaticGraphAdapter._make_programc Cs|j|d}|dur |S|jjdusJd|jj}|jdurZt||_g}|jD]}t |j }|j dsF|rF| rFq-||q-|rZ|j|}|j||jdkrn|dkrntrn|jj||jdkryt|}n|}||j|<dS)Nz6device is not set, please call `model.prepare()` firstr]rrrI)rrrrrrrtrrr r,r-rr.Z_is_initializedr4Z_prunerurrrgrZamp_initrZCompiledProgram) rrrr*rxZ uninitializedZvar_pyr1Z startup_progr'r'r(r2rs4        z*StaticGraphAdapter._compile_and_initializerqr")r __module__ __qualname____doc__rpropertyrsetterrrrrrrrrrr4r1r2 __classcell__r'r'rr(rs&     "E ]RrcsteZdZfddZeddZejddZddd Zdd d Zd d Z ddZ ddZ dddZ ddZ ZS)DynamicGraphAdaptercstt|||_tj|_tj|_ddddd|_ d|_ d|_ i|_ i|_ d|_|jdkrZttjj}tj|_tj|_tj|_tj|_tjj|jj||_dSdS)NrrrTr)rr[rrrrFrrrrr _input_inforrrrrrr|r}r~rsrnZ DataParallelr ddp_model)rrZstradegyrr'r(rs2         zDynamicGraphAdapter.__init__cCrr"rrr'r'r(rrzDynamicGraphAdapter.modecCrr"rrr'r'r(rrNTc Cs|jjsJd|jjd|_t|}t||_|pg}ddt|D}|jdkr>|jj dur>t j j di|j |j_ t j jdd|jdki|jd|ji#|jdkrc|jd d|D}n |jjd d|D}Wdn1sxwY|jjt||}t|}tj|}|jdkr|jj |}||r|jj |jj||jjn||r|jj||jjg}|jjD]} | jt||} | jd dt| D} || qt|d krd d|D|fSdd|DS)NrrcSg|]}t|qSr'rr=r'r'r(rrz3DynamicGraphAdapter.train_batch..renablelevelrcSr^r'rrrEr'r'r(rrcSr^r'rrar'r'r(rrcSr^r'r3rmr'r'r(rrrcSr^r'rbr=r'r'r(rrcSr^r'rbr=r'r'r(rrr')rrrrrr)rr\r_scalerrPrMZ GradScalerrZ auto_castrrr]rHrrJrKscaleZbackwardrRZclear_gradientsr'rIrr4r5) rrarrrbrTZ final_lossZscaledr/r metric_outsrdr'r'r(rsX            zDynamicGraphAdapter.train_batchc sFjjd_t|}t|_|pg}ddt|D}jjdd|D}tj }t|D]}|j |dq3|D]}|j |dq>jj rYjj t||}t|}j dkrrfddt|D}fdd|D}g}jj D]} jjdurj dkrtjjtrtjjj|d jd } jjd d | krӇfd d|D}fd d|D}d jjd <tjjd <njjd | 7<| jjd <| jt||} | jddt| D} || qxjj rt|rdd|D|fSjj r!dd|DS|S)NrcSr^r'rr=r'r'r(rrz2DynamicGraphAdapter.eval_batch..cSr^r'rrar'r'r(rr)devicercr8r'r9r:rr'r(rr<cr8r'r9r=rr'r(rr<rrc g|] }|dtqSr"rr:r r'r(rcrir"rr=r r'r(rrjr#cSr^r'rbrcr'r'r(r rcSr^r'rbr=r'r'r(r$rcSr^r'rbr=r'r'r(r&r)rrrrr)rr\rPrhZ get_deviceZ_torHrr'r(r#rr5r)rrrrTrIrr4) rrarrbZexpected_devicer;r6rTr/rr0rgrdr')r!rr"r(rs^            zDynamicGraphAdapter.eval_batchcs|jjd_ddt|D}t|_jj|}jdkr5tjj t j r5fddt|D}ddt|DS)NrcSr^r'rrar'r'r(r-rz5DynamicGraphAdapter.predict_batch..rcr8r'r9r:rr'r(r1r<cSr^r'rbr:r'r'r(r3r) rrrrr)rr\rr#rrr{)rrarbr'rr(r*s   z!DynamicGraphAdapter.predict_batchcOrr"rrr'r'r(r5rzDynamicGraphAdapter.parameterscCs|jj}t|||jjdur$|jjr$|jj}t||t|jdrF|jjdurH|jjrJ|jj}t ||ddSdSdSdS)Nre .pdscaler) rrrrZ save_dygraphrr>rerPr)rrparamsrZscalerr'r'r(r8s        zDynamicGraphAdapter.savecCs|D] \}}||qt|jdr!|jjdur!|r!|jj||jjr'|s)dSt|}|jjj}|dur8d}|jjjj }|d| d} dd|jj D} t |dddd D]f\} } | d vrv| d krut|d d |d <q]| D]J} | | d| r| t| d| dd}n| | dr| |kr| t| dd}nqx|d| d}| d|d|d}| ||<qxq]t|jjdstd|jj|dS|jj|dS)NrerrcSrr'r)rrr'r'r(r^rz,DynamicGraphAdapter.load..cSs t|dS)Nrrrr'r'r(r`s z*DynamicGraphAdapter.load..Trrrrrrset_state_dictzUpaddle.fluid.optimizer is deprecated in API 2.0, please use paddle.optimizer instead.) set_valuer>rreZload_state_dictrrrrrrfindrrrrr/r0rrr5warningswarnZset_dictrm)rrr scaler_staterrr r r r Z param_namesvar_namerrr rr'r'r(rDsf      zDynamicGraphAdapter.loadcCs^|jdkr"|jjdkr"tr"tjj|jj|jj dd\|j_|j_ |jdkr-d|j_ dSdS)Nrr)modelsZ optimizersr`r) rrrrrgrPrMrQrrrerr'r'r(r4s  zDynamicGraphAdapter.preparerqr")rrUrVrrXrrYrrrrrrr4rZr'r'rr(r[s     /:  Cr[c@seZdZdZd1ddZd2ddZed3dd Zed d Zd4d d Z d5ddZ ddZ ddZ    d6ddZ               d7ddZ      d8ddZ     d9dd Zd!d"Zifd#d$Zd1d%d&Zd:d'd(Zd)d*Zd+d,Zd-d.Zd/d0ZdS);ModelaD An Model object is network with training and inference features. Dynamic graph and static graph are supported at the same time, switched by `paddle.enable_static()`. The usage is as follows. But note, the switching between dynamic and static should be before instantiating a Model. The input description, i.e, paddle.static.InputSpec, must be required for static graph. When training on GPU, auto mixed precision (AMP O1) and pure float16 (AMP O2) training are both supported in static mode and dynamic mode. In static graph mode, before training with pure float16 (AMP O2), `multi_precision` could be set to True when creating optimizer, which can avoid poor accuracy or slow convergence in a way, and inputs of dtype float should be cast to float16 by users. `paddle.static.amp.fp16_guard` API should be also used to limit the range of pure float16 training, otherwise, 'use_fp16_guard' should be set to False by users. However, limiting the range of is not supported during training using AMP. Args: network (paddle.nn.Layer): The network is an instance of paddle.nn.Layer. inputs (InputSpec|list|tuple|dict|None, optional): `inputs`, entry points of network, could be a InputSpec instance, or list/tuple of InputSpec instances, or dict ({name: InputSpec}), and it couldn't be None in static graph. Default: None. labels (InputSpec|list|tuple|None, optional): `labels`, entry points of network, could be a InputSpec instnace or list/tuple of InputSpec instances, or None. For static graph, if labels is required in loss, labels must be set. Otherwise, it could be None. Default: None. Examples: 1. A common example .. code-block:: python :name: code-example1 import paddle import paddle.nn as nn import paddle.vision.transforms as T from paddle.static import InputSpec device = paddle.set_device('cpu') # or 'gpu' net = nn.Sequential( nn.Flatten(1), nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10)) # inputs and labels are not required for dynamic graph. input = InputSpec([None, 784], 'float32', 'x') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model(net, input, label) optim = paddle.optimizer.SGD(learning_rate=1e-3, parameters=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss(), paddle.metric.Accuracy()) transform = T.Compose([ T.Transpose(), T.Normalize([127.5], [127.5]) ]) data = paddle.vision.datasets.MNIST(mode='train', transform=transform) model.fit(data, epochs=2, batch_size=32, verbose=1) 2. An example using mixed precision training. .. code-block:: python :name: code-example2 # required: gpu import paddle import paddle.nn as nn import paddle.vision.transforms as T def run_example_code(): device = paddle.set_device('gpu') net = nn.Sequential(nn.Flatten(1), nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10)) model = paddle.Model(net) optim = paddle.optimizer.SGD(learning_rate=1e-3, parameters=model.parameters()) amp_configs = { "level": "O1", "custom_white_list": {'conv2d'}, "use_dynamic_loss_scaling": True } model.prepare(optim, paddle.nn.CrossEntropyLoss(), paddle.metric.Accuracy(), amp_configs=amp_configs) transform = T.Compose([T.Transpose(), T.Normalize([127.5], [127.5])]) data = paddle.vision.datasets.MNIST(mode='train', transform=transform) model.fit(data, epochs=2, batch_size=32, verbose=1) # mixed precision training is only supported on GPU now. if paddle.is_compiled_with_cuda(): run_example_code() NcCsd|_||_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ t s2t |t tttfs1tdn|r9t||_|j|dd|_|||_t rRt||_dSt||_dS)NrFz='inputs' must be list or tuple or dict, and couldn't be None.T)is_input)rrrErFrHZ _loss_weightsrr\_is_shape_inferredr( stop_trainingrr#r$r%rr TypeErrorr _verify_specrr[_adapterr)rrrarr'r'r(rs0  zModel.__init__TcCs.|j|||}tr|jdur||S)a Run one training step on one batch of data. And using `update` indicates whether optimizer update gradients computing by this batch. Args: inputs (numpy.ndarray|Tensor|list): Batch of input data. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple inputs). labels (numpy.ndarray|Tensor|list, optional): Batch of labels. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple labels). If has no labels, set None. Default: None. update (bool, optional): Whether update parameters after loss.backward() computing. Set it to False to accumulate gradients. Default: True. Returns: A list of scalar training loss if the model has no metrics, or a tuple (list of scalar loss, list of metrics) if the model set metrics. Examples: .. code-block:: python import paddle import paddle.nn as nn from paddle.static import InputSpec device = paddle.set_device('cpu') # or 'gpu' net = nn.Sequential( nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10)) input = InputSpec([None, 784], 'float32', 'x') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model(net, input, label) optim = paddle.optimizer.SGD(learning_rate=1e-3, parameters=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss()) data = paddle.rand((4, 784), dtype="float32") label = paddle.randint(0, 10, (4, 1), dtype="int64") loss = model.train_batch([data], [label]) print(loss) # [array([2.192784], dtype=float32)] N)r{rrrr\_update_inputs)rrarrrr'r'r(rs0zModel.train_batchcCs,|j||}tr|jdur||S)aP Run one evaluating step on a batch of data. Args: inputs (numpy.ndarray|Tensor|list): Batch of input data. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple inputs). labels (numpy.ndarray|Tensor|list, optional): Batch of labels. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple labels). If has no labels, set None. Default: None. Returns: A list of scalar testing loss if the model has no metrics, or a tuple (list of scalar loss, list of metrics) if the model set metrics. Examples: .. code-block:: python import paddle import paddle.nn as nn from paddle.static import InputSpec device = paddle.set_device('cpu') # or 'gpu' net = nn.Sequential( nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10)) input = InputSpec([None, 784], 'float32', 'x') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model(net, input, label) optim = paddle.optimizer.SGD(learning_rate=1e-3, parameters=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss(), metrics=paddle.metric.Accuracy()) data = paddle.rand((4, 784), dtype="float32") label = paddle.randint(0, 10, (4, 1), dtype="int64") loss, acc = model.eval_batch([data], [label]) print(loss, acc) # [array([2.8825705], dtype=float32)] [0.0] N)r{rrrr\r|)rrarrr'r'r(rSs/zModel.eval_batchcCs*|j|}tr|jdur||S)a Run one predicting step on a batch of data. Args: inputs (numpy.ndarray|Tensor|list): Batch of input data. It could be a numpy array or paddle.Tensor, or a list of arrays or tensors (in case the model has multiple inputs). Returns: A list of numpy.ndarray of predictions, that is the outputs of Model forward. Examples: .. code-block:: python import paddle import paddle.nn as nn from paddle.static import InputSpec device = paddle.set_device('cpu') # or 'gpu' input = InputSpec([None, 784], 'float32', 'x') label = InputSpec([None, 1], 'int64', 'label') net = nn.Sequential( nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10), nn.Softmax()) model = paddle.Model(net, input, label) model.prepare() data = paddle.rand((1, 784), dtype="float32") out = model.predict_batch([data]) print(out) # [array([[0.08189095, 0.16740078, 0.06889386, 0.05085445, 0.10729759, # 0.02217775, 0.14518553, 0.1591538 , 0.01808308, 0.17906217]], # dtype=float32)] N)r{rrrr\r|)rrarr'r'r(rs *zModel.predict_batchcCs2tjdkr|s||dS|j|dSdS)aE This function saves parameters, optimizer information or model and paramters only for inference to path. It depends on the parameter `training`. If `training` is set to True, the parameters saved contain all the trainable Variable, will save to a file with suffix ".pdparams". The optimizer information contains all the variable used by optimizer. For Adam optimizer, contains beta1, beta2, momentum etc. All the information will save to a file with suffix ".pdopt". (If the optimizer have no variable need to save (like SGD), the fill will not generated). This function will silently overwrite existing file at the target location. If `training` is set to False, only inference model will be saved. Args: path (str): The file prefix to save model. The format is 'dirname/file_prefix' or 'file_prefix'. if empty str. A exception will be raised. training (bool, optional): Whether to save for training. If not, save for inference only. Default: True. Returns: None Examples: .. code-block:: python import paddle import paddle.nn as nn import paddle.vision.transforms as T from paddle.static import InputSpec class Mnist(nn.Layer): def __init__(self): super(Mnist, self).__init__() self.net = nn.Sequential( nn.Flatten(1), nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10), nn.Softmax()) def forward(self, x): return self.net(x) dynamic = True # False # if use static graph, do not set if not dynamic: paddle.enable_static() input = InputSpec([None, 784], 'float32', 'x') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model(Mnist(), input, label) optim = paddle.optimizer.SGD(learning_rate=1e-3, parameters=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss()) transform = T.Compose([ T.Transpose(), T.Normalize([127.5], [127.5]) ]) data = paddle.vision.datasets.MNIST(mode='train', transform=transform) model.fit(data, epochs=1, batch_size=32, verbose=0) model.save('checkpoint/test') # save for training model.save('inference_model', False) # save for inference rN)rrr_save_inference_modelr{r)rrZtrainingr'r'r(rs Gz Model.saveFc s*dd}fdd}dd}||}||dsJdg}|jD]6\}} z||| } Wn%tyW} z|rKtd |t| d }n| WYd } ~ nd } ~ ww|| q'|rbd n||d } t rd } t |d r|j d urt j |drt|d} |j|| | S|j|| S)a Load from files storing the model states and optimizer states. The file for optimizer states is not necessary if no need to restore the optimizer. NOTE: parameters are retrieved out from the file storing model states accoring to their structured names. For fine-tuning or transfer-learning models where some of the layers have changed, keep parameters needed to restore have same structured names in the pre-trained model and fine-tuning model. Args: path (str): The prefix of files storing the model states and optimizer states. The files would be `path.pdparams` and `path.pdopt` separately, and the latter is not necessary when no need to restore. skip_mismatch (bool, optional): Whether to skip the loading of mismatch parameter or raise an error when mismatch happens (not found the parameter in file storing model states of or receives a mismatch shape). Default: False. reset_optimizer (bool, optional): If True, ignore the providing file storing optimizer states and initialize optimizer states from scratch. Otherwise, restore optimizer states from `path.pdopt` if a optimizer has been set to the model. Default: False. Returns: None Examples: .. code-block:: python import paddle import paddle.nn as nn from paddle.static import InputSpec device = paddle.set_device('cpu') input = InputSpec([None, 784], 'float32', 'x') model = paddle.Model(nn.Sequential( nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10), nn.Softmax()), input) model.save('checkpoint/test') model.load('checkpoint/test') cSsLtj|sdSt|d}tj|ddWdS1swYdS)Nrblatin1)encoding)rkrrrrr)rrr'r'r(_load_state_from_path6s   $z)Model.load.._load_state_from_pathcs\|d}|durtd|t|jt|jkr*td|t|jt|j||fS)Nz&{} is not found in the providing file.z5{} receives a shape {}, but the expected shape is {}.)r ValueErrorrr$r)rrrZ param_stater'r( _check_match<s z Model.load.._check_matchcSs*tj|\}}|dvsJd||S)N)rrrz.pdmodelzUnknown postfix {} from weights)rkrsplitextr)rextr'r'r(_strip_postfixGs  z"Model.load.._strip_postfixrz-Failed to load parameters, please check path.zSkip loading for {}. TNrrerk)rrrrrprqrstrr4rr>rerkrrrPrr{)rrZ skip_mismatchZreset_optimizerrrrZmatched_param_staterrZ match_reserrrrrr'rr(rsB3        z Model.loadcOs |jS)a Returns a list of parameters of the model. Returns: A list of Parameter in static graph. A list of ParamBase in dynamic graph. Examples: .. code-block:: python import paddle import paddle.nn as nn from paddle.static import InputSpec input = InputSpec([None, 784], 'float32', 'x') model = paddle.Model(nn.Sequential( nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10)), input) params = model.parameters() )r{rrr'r'r(rns zModel.parameterscs6fdd}ij_ij_sdj_dSttr,dvr#tdj_|dSdvr5dj_nddvr?tddj_tdh}|rVjjdkrXdSd vr`td |jjdkr|rd D]}||vr|jj|<||h8}qmfd d }||}|D] }|jj|<qdS)NcsBjjdkrjjrtjjtjjtjjfsJddSdSdS)NrzkOnly GradientClipByNorm and GradientClipByGlobalNorm are supported in amp training with level=O2 currently.) r{rrZ _grad_clipr#rPnnZClipGradByGlobalNormZClipGradByNormr'rr'r(_check_pure_fp16_configss z4Model._prepare_amp.._check_pure_fp16_configsr)rO1rz6The level of amp_configs should be 'O0', 'O1' or 'O2'.r`rz2amp_configs['level'] should be 'O0', 'O1' or 'O2'.r>zl'use_pure_fp16' is an invalid parameter, the level of mixed precision training only depends on 'O1' or 'O2'.)Zcustom_white_listZcustom_black_listZcustom_black_varnamescsVhd}||rtdt||d|vr)trtddj_|d|S)N>Zinit_loss_scalingZincr_every_n_stepsZ incr_ratioZ decr_ratior@Zdecr_every_n_nan_or_infZuse_dynamic_loss_scalingzwExcept for 'level', the keys of 'amp_configs' must be accepted by mixed precision APIs, but {} could not be recognized.r@z2'use_fp16_guard' is supported in static mode only.)rrr%rr{rre)amp_config_key_setZaccepted_param_setrOrr'r(_check_amp_configss   z.Model._prepare_amp.._check_amp_configs) r{rrrr#rrrr)rrOrrrrZamp_configs_setrr'rr( _prepare_ampsT      zModel._prepare_ampcCst|_t|jtjr=tjdkr=ts=tr6t j }t j }t t |j|t _ |t _ nt|jda||_|durSt|t jjsSt|sStd||_|pYg}t|D]}t|tsnJd|jjq^t||_|||jdS)a Configures the model before runing. Args: optimizer (Optimizer|None, optional): Optimizer must be set in training and should be a Optimizer instance. It can be None in eval and test mode. Default: None. loss (Loss|Callable|None, optional): Loss function can be a `paddle.nn.Layer` instance or any callable function taken the predicted values and ground truth values as input. It can be None when there is no loss. Default: None. metrics (Metric|list[Metric]|None, optional): If metrics is set, all metrics will be calculated and output in train/eval mode. Default: None. amp_configs (str|dict|None, optional): AMP configurations. If AMP or pure float16 training is used, the key 'level' of 'amp_configs' should be set to 'O1' or 'O2' respectively. Otherwise, the value of 'level' defaults to 'O0', which means float32 training. In addition to 'level', parameters consistent with mixed precision API could also be passed in. The supported keys are: 'init_loss_scaling', 'incr_ratio', 'decr_ratio', 'incr_every_n_steps', 'decr_every_n_nan_or_inf', 'use_dynamic_loss_scaling', 'custom_white_list', 'custom_black_list', and 'custom_black_varnames'or 'use_fp16_guard' is only supported in static mode. Mixed precision API documentations :ref:`api_paddle_amp_auto_cast` and :ref:`api_paddle_amp_GradScaler` could be referenced for details. For convenience, 'amp_configs' could be set to 'O1' or 'O2' if no more parameters are needed. 'amp_configs' could be None in float32 training. Default: None. Returns: None rTNzI'loss' must be sub classes of `paddle.nn.Layer` or any callable function.z{} is not sub class of Metric) _get_devicerr#rr{rrFrrrZ random_seedrrrPZdisable_staticrrrZLayercallableryrHr)rrrrr'rr{r4)rZ optimizerrr/rOZmain_prog_seedZstartup_prog_seedrr'r'r(r4sB&        z Model.preparer rIrc Cs|dusJdt|tr t||| | d}t|||j| dd}n|}|dur}|||||d}||||r||dkr||}|d|| d|||d}|d||jrnq|d|d|_dS)af Trains the model for a fixed number of epochs. If `eval_data` is set, evaluation will be done at the end of each epoch. Args: train_data (Dataset|DataLoader, optional): An iterable data loader is used for train. An instance of paddle paddle.io.Dataset or paddle.io.Dataloader is recomended. Default: None. eval_data (Dataset|DataLoader, optional): An iterable data loader is used for evaluation at the end of epoch. If None, will not do evaluation. An instance of paddle.io.Dataset or paddle.io.Dataloader is recomended. Default: None. batch_size (int, optional): The batch size of train_data and eval_data. When train_data and eval_data are both the instance of Dataloader, this parameter will be ignored. Default: 1. epochs (int, optional): The number of epochs to train the model. Default: 1. eval_freq (int, optional): The frequency, in number of epochs, an evalutation is performed. Default: 1. log_freq (int, optional): The frequency, in number of steps, the training logs are printed. Default: 10. save_dir(str|None, optional): The directory to save checkpoint during training. If None, will not save checkpoint. Default: None. save_freq (int, optional): The frequency, in number of epochs, to save checkpoint. Default: 1. verbose (int, optional): The verbosity mode, should be 0, 1, or 2. 0 = silent, 1 = progress bar, 2 = one line per epoch. Default: 2. drop_last (bool, optional): Whether drop the last incomplete batch of train_data when dataset size is not divisible by the batch size. When train_data is an instance of Dataloader, this parameter will be ignored. Default: False. shuffle (bool, optional): Whther to shuffle train_data. When train_data is an instance of Dataloader, this parameter will be ignored. Default: True. num_workers (int, optional): The number of subprocess to load data, 0 for no subprocess used and loading data in main process. When train_data and eval_data are both the instance of Dataloader, this parameter will be ignored. Default: 0. callbacks (Callback|None, optional): A list of `Callback` instances to apply during training. If None, :ref:`api_paddle_callbacks_ProgBarLogger` and :ref:`api_paddle_callbacks_ModelCheckpoint` are automatically inserted. Default: None. accumulate_grad_batches (int, optional): The number of batches to accumulate gradident during training process before optimizer updates. It can mimic large batch size. Default: 1. num_iters (int|None, optional): The number of iterations to evaluate the model. If None, evaluate on whole input dataset, otherwise, evaluate `num_iters` times. Default: None. Returns: None Examples: 1. An example use Dataset and set batch size, shuffle in fit. How to make a batch is done internally. .. code-block:: python :name: code-example1 import paddle import paddle.vision.transforms as T from paddle.vision.datasets import MNIST from paddle.static import InputSpec dynamic = True if not dynamic: paddle.enable_static() transform = T.Compose([ T.Transpose(), T.Normalize([127.5], [127.5]) ]) train_dataset = MNIST(mode='train', transform=transform) val_dataset = MNIST(mode='test', transform=transform) input = InputSpec([None, 1, 28, 28], 'float32', 'image') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model( paddle.vision.models.LeNet(), input, label) optim = paddle.optimizer.Adam( learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, paddle.nn.CrossEntropyLoss(), paddle.metric.Accuracy(topk=(1, 2))) model.fit(train_dataset, val_dataset, epochs=2, batch_size=64, save_dir='mnist_checkpoint') 2. An example use DataLoader, batch size and shuffle is set in DataLoader. .. code-block:: python :name: code-example2 import paddle import paddle.vision.transforms as T from paddle.vision.datasets import MNIST from paddle.static import InputSpec dynamic = True if not dynamic: paddle.enable_static() transform = T.Compose([ T.Transpose(), T.Normalize([127.5], [127.5]) ]) train_dataset = MNIST(mode='train', transform=transform) train_loader = paddle.io.DataLoader(train_dataset, batch_size=64) val_dataset = MNIST(mode='test', transform=transform) val_loader = paddle.io.DataLoader(val_dataset, batch_size=64) input = InputSpec([None, 1, 28, 28], 'float32', 'image') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model( paddle.vision.models.LeNet(), input, label) optim = paddle.optimizer.Adam( learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, paddle.nn.CrossEntropyLoss(), paddle.metric.Accuracy(topk=(1, 2))) model.fit(train_loader, val_loader, epochs=2, save_dir='mnist_checkpoint') Nztrain_data must be given!) batch_sizeshuffle drop_lastTZ batch_samplerZplaces num_workersZ return_listrr!num_iters must be greater than 0!r)repochsstepslog_freq save_freqsave_dirverboser/css|]}t|tVqdSr")r#r r7r'r'r( szModel.fit..z$EarlyStopping needs validation data.rrrr/)r#rrrrr( _accumulate_len_data_loader num_itersrTminr _metrics_nameanyrprqon_beginrangeZon_epoch_begin_run_one_epochZ on_epoch_endon_endrx)rZ train_data eval_datarrZ eval_freqrrrrrrr callbacksZaccumulate_grad_batchesrZ train_samplerZ train_loader eval_sampler eval_loaderZdo_evalrcbksepochlogs eval_stepsZ eval_logsr'r'r(fit%s              z Model.fitcCs|durt|trt||d}t|||j|dd} n|} | |_t|||||d} || } ||_ |durPt|t rPt| t rP|dksHJdt || } | |_ | d| |d | | | d} | d| d|_i} |D]}| || |<qq| S) a Evaluate the loss and metrics of the model on input dataset. Args: eval_data (Dataset|DataLoader): An iterable data loader is used for evaluation. An instance of paddle.io.Dataset or paddle.io.Dataloader is recomended. batch_size (int, optional): The batch size of train_data and eval_data. When eval_data is the instance of Dataloader, this argument will be ignored. Default: 1. log_freq (int, optional): The frequency, in number of steps, the eval logs are printed. Default: 10. verbose (int, optional): The verbosity mode, should be 0, 1, or 2. 0 = silent, 1 = progress bar, 2 = one line per epoch. Default: 2. num_workers (int, optional): The number of subprocess to load data, 0 for no subprocess used and loading data in main process. When train_data and eval_data are both the instance of Dataloader, this parameter will be ignored. Default: 0. callbacks (Callback|None, optional): A list of `Callback` instances to apply during training. If None, `ProgBarLogger` and `ModelCheckpoint` are automatically inserted. Default: None. num_iters (int|None, optional): The number of iterations to evaluate the model. If None, evaluate on whole input dataset, otherwise, evaluate `num_iters` times. Default: None. Returns: dict: Result of metric. The key is the names of Metric, value is a scalar or numpy.array. Examples: .. code-block:: python import paddle import paddle.vision.transforms as T from paddle.static import InputSpec # declarative mode transform = T.Compose([ T.Transpose(), T.Normalize([127.5], [127.5]) ]) val_dataset = paddle.vision.datasets.MNIST(mode='test', transform=transform) input = InputSpec([-1, 1, 28, 28], 'float32', 'image') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model(paddle.vision.models.LeNet(), input, label) model.prepare(metrics=paddle.metric.Accuracy()) result = model.evaluate(val_dataset, batch_size=64) print(result) # {'acc': 0.0699} NrTr)rrrr/rrrr)r#rrrrr(rrrrrTrrrr)rrrrrrrrrrrrrZ eval_resultrr'r'r(evaluate sL<    zModel.evaluatec Cs|durt|trt||d}t|||j|dd}n|}||_t|||d} ||} d| i} | d| g} | || d\} } t t | } |rOdd | D} d|_| d| | S) a Compute the output predictions on testing data. Args: test_data (Dataset|DataLoader): An iterable data loader is used for predict. An instance of paddle.io.Dataset or paddle.io.Dataloader is recomended. batch_size (int, optional): The batch size of test_data. When test_data is the instance of Dataloader, this argument will be ignored. Default: 1. num_workers (int, optional): The number of subprocess to load data, 0 for no subprocess used and loading data in main process. When test_data is the instance of Dataloader, this argument will be ignored. Default: 0. stack_outputs (bool, optional): Whether stack output field like a batch, as for an output field of a sample is in shape [X, Y], test_data contains N samples, predict output field will be in shape [N, X, Y] if stack_output is True, and will be a length N list in shape [[X, Y], [X, Y], ..., [X, Y]] if stack_outputs is False. stack_outputs as False is used for LoDTensor output situation, it is recommended set as True if outputs contains no LoDTensor. Default: False. verbose (int, optional): The verbosity mode, should be 0, 1, or 2. 0 = silent, 1 = progress bar, 2 = one line per batch. Default: 1. callbacks(Callback, optional): A Callback instance, Default: None. Returns: list: output of models. Examples: .. code-block:: python import numpy as np import paddle from paddle.static import InputSpec class MnistDataset(paddle.vision.datasets.MNIST): def __init__(self, mode, return_label=True): super(MnistDataset, self).__init__(mode=mode) self.return_label = return_label def __getitem__(self, idx): img = np.reshape(self.images[idx], [1, 28, 28]) if self.return_label: return img, np.array(self.labels[idx]).astype('int64') return img, def __len__(self): return len(self.images) test_dataset = MnistDataset(mode='test', return_label=False) # imperative mode input = InputSpec([-1, 1, 28, 28], 'float32', 'image') model = paddle.Model(paddle.vision.models.LeNet(), input) model.prepare() result = model.predict(test_dataset, batch_size=64) print(len(result[0]), result[0][0].shape) # 157 (64, 10) # declarative mode device = paddle.set_device('cpu') paddle.enable_static() input = InputSpec([-1, 1, 28, 28], 'float32', 'image') model = paddle.Model(paddle.vision.models.LeNet(), input) model.prepare() result = model.predict(test_dataset, batch_size=64) print(len(result[0]), result[0][0].shape) # 157 (64, 10) NrTr)rrrpredictcSrr')r/Zvstack)routsr'r'r(rrz!Model.predict..) r#rrrrr(rrrrr$r&r) rZ test_datarrZ stack_outputsrrZ test_samplerZ test_loaderrZ test_stepsrrbr'r'r(rrs0L    z Model.predictc CsFtr@tjd,|j}|jdurtd|jr$t d|jdt j j |||j dWddS1s9wYdStj|}|dkrNtdtj|}|ratj|sat||}|t}|t}|jjdd}|syJd |jd d } d d |jjdD} |jjdd} tjj|| | |jj| ||ddS)z Save inference model can be used in static or dynamic mode. Args: path (str): The path prefix to save model. The format is ``dirname/file_prefix`` or ``file_prefix``. Returns: None NzSaving inference model needs 'inputs' or running before saving. Please specify 'inputs' in Model initialization or input training data and perform a training for shape derivation.a'inputs' was not specified when Model initialization, so the input shape to be saved will be the shape derived from the user's actual inputs. The input shape to be saved is %s. For saving correct input shapes, please provide 'inputs' for Model initialization.r)Z input_specrzThe input path MUST be format of dirname/file_prefix [dirname\file_prefix in Windows system], but received file_prefix is empty string.rrTrAcSrr'rrr'r'r(rrz/Model._save_inference_model..r)Z main_programmodel_filenameparams_filename) rrZ frameworkZ_dygraph_guardrr\ RuntimeErrorrwrprqrPZjitrrErkrrrrrrrrr{rrrBrrioZsave_inference_modelr) rrlayerZ file_prefixrZ model_pathrrrZ infer_progr+rWr'r'r(r}sR  "     zModel._save_inference_modelcCsTg}t|D]\}}t|}t|djr|ddn|djd}|||||dkr|dt|j|t|jdg} |dkrZ| |d|jdkpX|dt|kt ||d| } |j rt|j rtdd| dDg} n|j rdd| Dg} ng} |j D] } | } | t| qt|t| ksJt|| D]\}}|||<qn|jdur||dt|j} n||} || ||d <|dks|jj|dddkr|tj|d <n |jj|d|d <||||t|d r|jdur|jd8_|jdkrd |_|`nq||dkr(||fS|S) Nrrrrr#cSg|]}|dqSrr'r=r'r'r(rCrz(Model._run_one_epoch..cSrrr'r=r'r'r(rErsteprrT)r$rrrZon_batch_beginr5rEr4rgetattrr'rH accumulateextendr)rr&rr{rrrrFZ on_batch_endr>rrx_reset_metrics)r data_loaderrrrrbrdatarrErr/rresrrr'r'r(rsh  $        zModel._run_one_epochcCs>|dus |jdus Jd|dur|}n|j}t|j||dS)aPrints a string summary of the network. Args: input_size (tuple|InputSpec|list[tuple|InputSpec], optional): size of input tensor. if not set, input_size will get from ``self._inputs`` if network only have one input, input_size can be tuple or InputSpec. if model have multiple input, input_size must be a list which contain every input's shape. Default: None. dtype (str, optional): if dtype is None, 'float32' will be used, Default: None. Returns: Dict: a summary of the network including total params and total trainable params. Examples: .. code-block:: python import paddle from paddle.static import InputSpec input = InputSpec([None, 1, 28, 28], 'float32', 'image') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model(paddle.vision.models.LeNet(), input, label) optim = paddle.optimizer.Adam( learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, paddle.nn.CrossEntropyLoss()) params_info = model.summary() print(params_info) # {'total_params': 61610, 'trainable_params': 61610} Nz)'input_size' or 'self._input' must be set)r)rEr!r)rZ input_sizerZ _input_sizer'r'r(r!ms $z Model.summaryc sg}dur8|r3t|jjdd}dur+dur+tr+fddt|D}n*dd|D}n"t}nttrQ|dusCJfddt|jjD}nt}|durvt|D]\}}t|t shJ|j durut d ||q]|S)Nrcs&g|]\}}t|||dqS))r-rrr)rr.r-)rrr'r(rsz&Model._verify_spec..cSsg|] }t|dgdqS)N)r-rrrr-r'r'r(rrFcsg|] }|dkr|qSrr'r)specsr'r(rs zsr                              8'&y