o NeBw@sddlmZmZddlZddlZddlZddlZddlZddlZ ddl m Z ddl m Z ddl mZddlmZddlZdd lmZddlZdd l mZmZmZddlZddlZgd Zd ad dZddZddZeeZed.ddZeddZ da!eddZ"ddZ#ddZ$d/ddZ%d d!Z&ed.d"d#Z'd/d$d%Z(Gd&d'd'Z)ed/d(d)Z*e j+    d0d*d+Z,e j-d1d,d-Z.dS)2)signature_safe_contextmanagerwrap_decoratorN)core) framework)CleanupFuncRegistrar)Tracer) convert_dtype)_get_paddle_place_in_legacy_dygraph_in_eager_without_dygraph_check)no_gradno_grad_gradguardenable_dygraphdisable_dygraphenabled to_variableFcCstS)zW Return a bool value that indicates whether running code under `@declarative` _in_declarative_mode_rrID:\Projects\ConvertPro\env\Lib\site-packages\paddle/fluid/dygraph/base.pyin_declarative_mode)srcCs>t|||D]\}}}||krt|d|dqdS)z Warning if inputs do not elementwisely equals to support_values. It's a utility function for dy2static when dygraph interface have more inputs than static interface such as paddle.grad. z# has unsupported parameter in jit: z, jit will discard itN)zipwarningswarn) func_nameZ input_namesinputsZsupport_valuesnameinpsuprrr&declarative_unsupport_argument_warning1s r#csfdd}|S)Ncs<td|i|WdS1swYdSN)r_dygraph_guard)argskwargsfuncrr__impl__As  $z*_switch_to_static_graph_..__impl__rr)r*rr(r_switch_to_static_graph_?s r,Tccst}|adV|adSr$r)Zis_declarative original_valrrr_switch_declarative_mode_guard_Ks r.ccsBt}|r |j}||_z dVW|r||_dSdS|r ||_wr$)r_dygraph_tracerZ_enable_program_desc_tracing)enabletracerr-rrrprogram_desc_tracing_guardUs r2ccsvtr6ts6|r6|}|D]\}}t|tr#dd|D}nt|}|||<qdV||dSdVdS)NcSsg|]}t|qSr)_convert_into_variable).0varrrr mszparam_guard..) rr_non_static_modecopyitems isinstancelistr3update) parametersZorigin_parametersr Zvar_basenew_varrrr param_guardes   r?cCst|tjjtjfr@|j|j}|durt|tj sJ|St|tj tj fr0|j dd}|St |jdk}|j d|d}|S|S)z( Convert Varbase into Variable. NT) to_parameterrF)r@ persistable)r:reagerTensorVarBaseblockZ_find_var_recursiver rVariableZEagerParamBaseZ ParamBaseZ_to_static_varlenshape)Ztensorr>Zis_persistablerrrr3ws   r3cCstS)aL This function checks whether the program runs in dynamic graph mode or not. You can enter dynamic graph mode with :ref:`api_fluid_dygraph_guard` api, or enable and disable dynamic graph mode with :ref:`api_fluid_dygraph_enable_dygraph` and :ref:`api_fluid_dygraph_disable_dygraph` api . **Note**: ``fluid.dygraph.enabled`` is the alias of ``fluid.in_dygraph_mode``, and ``fluid.in_dygraph_mode`` is recommended to use for now. Returns: bool: Whether the program is running in dynamic graph mode. Examples: .. code-block:: python import paddle.fluid as fluid fluid.enable_dygraph() # Now we are in dygragh mode print(fluid.dygraph.enabled()) # True fluid.disable_dygraph() print(fluid.dygraph.enabled()) # False )rr7rrrrrsrcCs0tdurtt|datttdSdS)a .. note:: Dynamic graph mode is turn ON by default since paddle 2.0.0 This API turn OFF static graph mode. You can turn ON static graph mode by `enable_static <./disable_dygraph_en.html>`_ . Parameters: place(paddle.CPUPlace|paddle.CUDAPlace|str, optional): Place to run dynamic graph. Default: None. Which means that the running place will be determined according to the way of paddle compilation. If ``place`` is string, It can be ``cpu``, and ``gpu:x``, where ``x`` is the index of the GPUs. return: None Examples: .. code-block:: python import paddle print(paddle.in_dynamic_mode()) # True, dynamic mode is turn ON by default since paddle 2.0.0 paddle.enable_static() print(paddle.in_dynamic_mode()) # False, Now we are in static mode paddle.disable_static() print(paddle.in_dynamic_mode()) # True, Now we are in dynamic mode Nplace)#_functional_dygraph_context_managerrr __enter__rregisterrrIrrrrsrcCs"tdurtjtdadSdS)a .. note:: Dynamic graph mode is turn ON by default since paddle 2.0.0 This API turn ON static graph mode. You can turn ON static graph mode by `disable_static <./enable_dygraph_en.html>`_ . return: None Examples: .. code-block:: python import paddle print(paddle.in_dynamic_mode()) # True, dynamic mode is turn ON by default since paddle 2.0.0 paddle.enable_static() print(paddle.in_dynamic_mode()) # False, Now we are in static mode paddle.disable_static() print(paddle.in_dynamic_mode()) # True, Now we are in dynamic mode N)rK__exit__sysexc_inforrrrrsrccs@t}|r|j}||_z dVW||_dS||_wdVdSr$)rr/ _has_grad)is_trainr1Zhas_gradrrr_switch_tracer_mode_guard_s rScCs8trtd|durtddStjdd}||S)a :api_attr: imperative Create a context which disables dygraph gradient calculation. In this mode, the result of every computation will have `stop_gradient=True`. Also functions as a decorator. (Make sure to instantiate without parenthesis.) Examples: .. code-block:: python import numpy as np import paddle.fluid as fluid # use as generator data = np.array([[2, 3], [4, 5]]).astype('float32') with fluid.dygraph.guard(): l0 = fluid.Linear(2, 2) # l0.weight.gradient() is None l1 = fluid.Linear(2, 2) with fluid.dygraph.no_grad(): # l1.weight.stop_gradient is False tmp = l1.weight * 2 # tmp.stop_gradient is True x = fluid.dygraph.to_variable(data) y = l0(x) + tmp o = l1(y) o.backward() print(tmp.gradient() is None) # True print(l0.weight.gradient() is None) # False # use as decorator @fluid.dygraph.no_grad def test_layer(): with fluid.dygraph.guard(): inp = np.ones([3, 1024], dtype='float32') t = fluid.dygraph.base.to_variable(inp) linear1 = fluid.Linear(1024, 4, bias_attr=False) linear2 = fluid.Linear(4, 4) ret = linear1(t) dy_ret = linear2(ret) test_layer() zfpaddle.no_grad is only supported for inference model, and not supported for training under @to_static.NFrRc_s<tdd||i|WdS1swYdS)NFrT)rSr)r&r'rrrr*<s  $zno_grad..__impl__)rrrrS decoratorr+rrrrs/  rc@s(eZdZdZddZddZddZdS) ra :api_attr: imperative Create a context which disables dygraph gradient calculation. In this mode, the result of every computation will have `stop_gradient` set to `True`. Also functions as a decorator. (Make sure to use an instance.) Examples: .. code-block:: python import numpy as np import paddle # use as generator data = np.array([[2, 3], [4, 5]]).astype('float32') l0 = paddle.nn.Linear(2, 2) # l0.weight.gradient() is None l1 = paddle.nn.Linear(2, 2) with paddle.no_grad(): # l1.weight.stop_gradient is False tmp = l1.weight * 2 # tmp.stop_gradient is True x = paddle.to_tensor(data) y = l0(x) + tmp o = l1(y) o.backward() print(tmp.gradient() is None) # True print(l0.weight.gradient() is None) # False # use as decorator @paddle.no_grad() def test_layer(): inp = np.ones([3, 1024], dtype='float32') t = paddle.to_tensor(inp) linear1 = paddle.nn.Linear(1024, 4, bias_attr=False) linear2 = paddle.nn.Linear(4, 4) ret = linear1(t) dy_ret = linear2(ret) test_layer() cs>tjfdd}tjfdd}t|r||S||S)Ncs6||i|WdS1swYdSr$rrUselfrr_decorate_functionts $z-no_grad_.__call__.._decorate_functionc?sJ||i|}|D]}|Vq WddS1swYdSr$r)r)r&r'genxrWrr_decorate_generatorys"z.no_grad_.__call__.._decorate_generator)rVinspectisgeneratorfunction)rXr)rYr\rrWr__call__rs zno_grad_.__call__cCs"t}|r|j|_d|_dSdS)NF)rr/rQorig)rXr1rrrrLs  zno_grad_.__enter__cGst}|r |j|_dSdSr$)rr/r`rQ)rXr&r1rrrrNs zno_grad_.__exit__N)__name__ __module__ __qualname____doc__r_rLrNrrrrrDs - rc cst}t}t}tj}|durt|}nt}t||Stj 5t | t | dVWdn1sBwYWdn1sQwYWdn1s`wYWddSWddS1sxwYdS)a :api_attr: imperative This context will create a dygraph context for dygraph to run, using python ``with`` statement. Parameters: place(fluid.CPUPlace| fluid.CUDAPlace|str, optional): Place to execute dygraph. If None, the running place will be determined according to the way of paddle compilation. If ``place`` is string, It can be ``cpu``, ``gpu:x`` and ``xpu:x``, where ``x`` is the index of the GPUs or XPUs. Default: None return: None Examples: .. code-block:: python import numpy as np import paddle.fluid as fluid with fluid.dygraph.guard(): inp = np.ones([3, 1024], dtype='float32') t = fluid.dygraph.base.to_variable(inp) linear1 = fluid.Linear(1024, 4, bias_attr=False) linear2 = fluid.Linear(4, 4) ret = linear1(t) dy_ret = linear2(ret) N) rZProgramr rrDr _current_expected_placeZ program_guardZ unique_namerr%Z_dygraph_place_guard)rJtrainZstartupr1rDZexpected_placerrrrs*     "rc CsBtrddlm}tdgd||||ggd|||||Sdd} | |d}| |d }|d ur^t|ttfs;|g}|D]} | d ur\trRt| tj j sQJd q=t| tj s\Jd q=ng}t |dkrrt |t |ksrJd |d uryg}nOt|tj tj j fr|g}nAt|tj j r|g}n6t|ttt frt|}|D]} trt| tj j sJd qt| tj sJdqn trtdtdt|tsJd|d ur|}t|tsJdt|tsJdt|tsJd|sJdtr tj ||||||||St} | tt||||| |||| S)a .. note:: **This API is ONLY available in imperative mode.** This API computes the sum of gradients of `outputs` with respect to each `inputs` . Parameters: outputs (Tensor|list(Tensor)|tuple(Tensor)): the output Tensor or Tensor list/tuple of the graph to compute gradients. inputs (Tensor|list(Tensor)|tuple(Tensor)): the input Tensor or Tensor list/tuple of the graph to compute gradients. The returned values of this API are the gradients of `inputs` . grad_outputs (Tensor|list(Tensor|None)|tuple(Tensor|None), optional): initial gradient values of `outputs` . If `grad_outputs` is None, the initial gradient values of `outputs` would be Tensors filled with 1; if `grad_outputs` is not None, it must have the same length as `outputs` , and in this case, the initial gradient value of the i-th `outputs` would be: (1) a Tensor filled with 1 when the i-th element of `grad_outputs` is None; (2) the i-th element of `grad_outputs` when the i-th element of `grad_outputs` is a Tensor. Default None. retain_graph (bool, optional): whether to retain the forward graph which is used to calculate the gradient. When it is True, the graph would be retained, in which way users can calculate backward twice for the same graph. When it is False, the graph would be freed. Default None, which means it is equal to `create_graph` . create_graph (bool, optional): whether to create the gradient graphs of the computing process. When it is True, higher order derivatives are supported to compute; when it is False, the gradient graphs of the computing process would be discarded. Default False. only_inputs (bool, optional): whether to only compute the gradients of `inputs` . If it is False, the gradients of all remaining leaf Tensors in the graph would be also computed and accumulated. If it is True, only the gradients of `inputs` would be computed. Default True. only_inputs=False is under development, and it is not supported yet. allow_unused (bool, optional): whether to raise error or return None if some Tensors of `inputs` are unreachable in the graph. If some Tensors of `inputs` are unreachable in the graph (i.e., their gradients are None), error would be raised if allow_unused=False, or None would be returned as their gradients if allow_unused=True. Default False. no_grad_vars (Tensor|list(Tensor)|tuple(Tensor)|set(Tensor), optional): the Tensors whose gradients are not needed to compute. Default None. Returns: list: a list of Tensors, whose length is the same as the Tensor number inside `inputs`, and the i-th returned Tensor is the sum of gradients of `outputs` with respect to the i-th `inputs`. Examples: .. code-block:: python :name: code-example-1 import paddle def test_dygraph_grad(create_graph): x = paddle.ones(shape=[1], dtype='float32') x.stop_gradient = False y = x * x # Since y = x * x, dx = 2 * x dx = paddle.grad( outputs=[y], inputs=[x], create_graph=create_graph, retain_graph=True)[0] z = y + dx # If create_graph = False, the gradient of dx # would not be backpropagated. Therefore, # z = x * x + dx, and x.gradient() = 2 * x = 2.0 # If create_graph = True, the gradient of dx # would be backpropagated. Therefore, # z = x * x + dx = x * x + 2 * x, and # x.gradient() = 2 * x + 2 = 4.0 z.backward() return x.gradient() print(test_dygraph_grad(create_graph=False)) # [2.] print(test_dygraph_grad(create_graph=True)) # [4.] .. code-block:: python :name: code-example-2 import paddle def test_dygraph_grad(grad_outputs=None): x = paddle.to_tensor(2.0) x.stop_gradient = False y1 = x * x y2 = x * 3 # If grad_outputs=None, dy1 = [1], dy2 = [1]. # If grad_outputs=[g1, g2], then: # - dy1 = [1] if g1 is None else g1 # - dy2 = [1] if g2 is None else g2 # Since y1 = x * x, dx = 2 * x * dy1. # Since y2 = x * 3, dx = 3 * dy2. # Therefore, the final result would be: # dx = 2 * x * dy1 + 3 * dy2 = 4 * dy1 + 3 * dy2. dx = paddle.grad( outputs=[y1, y2], inputs=[x], grad_outputs=grad_outputs)[0] return dx.numpy() grad_value = paddle.to_tensor(4.0) # dy1 = [1], dy2 = [1] print(test_dygraph_grad(None)) # [7.] # dy1 = [1], dy2 = [4] print(test_dygraph_grad([None, grad_value])) # [16.] # dy1 = [4], dy2 = [1] print(test_dygraph_grad([grad_value, None])) # [19.] # dy1 = [3], dy2 = [4] grad_y1 = paddle.to_tensor(3.0) print(test_dygraph_grad([grad_y1, grad_value])) # [24.] r) gradientsz paddle.grad) retain_graphZ create_grad only_inputs allow_unused)NFTFcSs|dus Jd|t|ttfrEt|dksJd||D]!}tr5t|tjjs4Jd|q!t|tj sBJd|q!|StrYt|tjjsVJd||gSt|tj sfJd||gS)Nz{} should not be Nonerz{} cannot be emptyzElements of {} must be TensorzElements of {} must be Variablez#{} must be Tensor or list of Tensorz'{} must be Variable or list of Variable) formatr:r;tuplerGr rrBrCrD)Z in_out_listr each_varrrr check_in_outTs@zgrad..check_in_outoutputsrNzLgrad_outputs must be None, a Variable or a list containing None or Variablesz3The length of grad_outputs must be equal to outputsz%no_grad_vars can only contains Tensorz'no_grad_vars can only contains Variablez>no_grad_vars must be None, Tensor or list/tuple/set of TensorszBno_grad_vars must be None, Variable or list/tuple/set of Variablesz"create_graph must be True or Falsez(retain_graph must be None, True or Falsez"allow_unused must be True or Falsez!only_inputs must be True or Falsez&only_inputs=False is not supported yet)rZ paddle.staticrgr#r:r;rlr rrBrCrDrGsetAssertionErrorboolZrun_partial_gradZPlaceZ set_placerreZdygraph_partial_grad) rorZ grad_outputsrhZ create_graphrirjZ no_grad_varsrgrnrmr5rJrrrrs         rcCs8tttjtjjtjtj tjtj f}t ||s t d|t |ft |tjjtjtj fr.|St |tjtj frsf            '  ?M 0 {