Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 2 additions & 14 deletions paddle/fluid/eager/api/manual/eager_manual/nodes/multiply_node.cc
Original file line number Diff line number Diff line change
Expand Up @@ -150,16 +150,10 @@ MultiplyGradNode::operator()(
paddle::prim::StaticCompositeContext::Instance().CheckSkipCompOps(
grad_op_name);
if (paddle::prim::PrimCommonUtils::IsEagerPrimEnabled() && !need_skip) {
bool original_global_grad = egr::Controller::Instance().HasGrad();
if (!create_graph) {
egr::Controller::Instance().SetHasGrad(create_graph);
}
egr::HasGradGuard has_grad_guard(create_graph);
paddle::prim::multiply_grad<paddle::Tensor>(
x, y, grad_out, axis, api_output_0, api_output_1);
VLOG(4) << "Composite api multiply_grad is called ";
if (!create_graph) {
egr::Controller::Instance().SetHasGrad(original_global_grad);
}
} else {
paddle::experimental::multiply_grad(
x, y, grad_out, axis, api_output_0, api_output_1);
Expand Down Expand Up @@ -440,10 +434,7 @@ MultiplyDoubleGradNode::operator()(
paddle::prim::StaticCompositeContext::Instance().CheckSkipCompOps(
grad_op_name);
if (!need_skip) {
bool original_global_grad = egr::Controller::Instance().HasGrad();
if (!create_graph) {
egr::Controller::Instance().SetHasGrad(create_graph);
}
egr::HasGradGuard has_grad_guard(create_graph);
paddle::prim::multiply_double_grad<paddle::Tensor>(x,
y,
fwd_grad_out,
Expand All @@ -454,9 +445,6 @@ MultiplyDoubleGradNode::operator()(
api_output_1,
api_output_2);
VLOG(4) << "Composite api multiply_double_grad is called ";
if (!create_graph) {
egr::Controller::Instance().SetHasGrad(original_global_grad);
}
} else {
paddle::experimental::multiply_double_grad(x,
y,
Expand Down
25 changes: 25 additions & 0 deletions paddle/fluid/eager/api/utils/global_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,31 @@ class EagerBackwardStateGuard {

~EagerBackwardStateGuard() { Controller::Instance().SetIsInBackward(false); }
};

// NOTE: Keeps grad recording on for this scope only if `keep_grad`
// asks for it and it was already on, and restores the previous value on scope
// exit, including when the scope is left by an exception. Restoring by hand is
// not exception safe: a throw between the two statements skips the restore and
// the whole thread silently stops recording the computation graph.
//
// `keep_grad` is never allowed to turn recording back on, so passing
// `create_graph` is enough to express "run with grad recording off unless the
// caller asked for a graph", without re-opening the switch inside no_grad.
class HasGradGuard {
public:
explicit HasGradGuard(bool keep_grad)
: original_(Controller::Instance().HasGrad()) {
Controller::Instance().SetHasGrad(original_ && keep_grad);
}

~HasGradGuard() { Controller::Instance().SetHasGrad(original_); }

HasGradGuard(const HasGradGuard&) = delete;
HasGradGuard& operator=(const HasGradGuard&) = delete;

private:
bool original_;
};
class EagerBackwardSubGraphNodeRecorder {
public:
/**
Expand Down
22 changes: 6 additions & 16 deletions paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -3333,30 +3333,20 @@ def _gen_api_call_code_block(
"""
if in_prim_white_list:
code = f"""
bool original_global_grad = egr::Controller::Instance().HasGrad();
if (!create_graph) {{
{indent}egr::Controller::Instance().SetHasGrad(create_graph);
}}
{composite_grad_api_namespace}{composite_grad_api_name}{composite_template_name}({composite_grad_api_args_str});
VLOG(4) << "Composite api {composite_grad_api_name} is called";
if (!create_graph) {{
{indent}egr::Controller::Instance().SetHasGrad(original_global_grad);
{{
{indent}egr::HasGradGuard has_grad_guard(create_graph);
{indent}{composite_grad_api_namespace}{composite_grad_api_name}{composite_template_name}({composite_grad_api_args_str});
{indent}VLOG(4) << "Composite api {composite_grad_api_name} is called";
}}
"""
else:
code = f"""
std::string grad_op_name = "{composite_grad_api_name}";
auto need_skip = paddle::prim::StaticCompositeContext::Instance().CheckSkipCompOps(grad_op_name);
if (paddle::prim::PrimCommonUtils::IsEagerPrimEnabled() && !need_skip) {{
{indent}bool original_global_grad = egr::Controller::Instance().HasGrad();
{indent}if (!create_graph) {{
{indent}{indent}egr::Controller::Instance().SetHasGrad(create_graph);
{indent}}}
{indent}egr::HasGradGuard has_grad_guard(create_graph);
{indent}{composite_grad_api_namespace}{composite_grad_api_name}{composite_template_name}({composite_grad_api_args_str});
{indent}VLOG(4) << "Composite api {composite_grad_api_name} is called";
{indent}if (!create_graph) {{
{indent}{indent}egr::Controller::Instance().SetHasGrad(original_global_grad);
{indent}}}"""
{indent}VLOG(4) << "Composite api {composite_grad_api_name} is called";"""
if has_kernel_impl:
code = (
code
Expand Down
15 changes: 8 additions & 7 deletions paddle/fluid/eager/pylayer/py_layer_node.cc
Original file line number Diff line number Diff line change
Expand Up @@ -168,15 +168,16 @@ GradNodePyLayer::operator()(
PADDLE_THROW(common::errors::InvalidArgument(
"%s:Get backward function failed.", name()));
}
bool need_grad_tmp = egr::Controller::Instance().HasGrad();
egr::Controller::Instance().SetHasGrad(create_graph && need_grad_tmp);
PyObject* outputs = nullptr;
{
egr::HasGradGuard has_grad_guard(create_graph);
#ifdef PADDLE_WITH_CUDA
for (auto& functor : ctx->reload_functors) {
functor.Reload();
}
for (auto& functor : ctx->reload_functors) {
functor.Reload();
}
#endif
auto outputs = PyObject_CallObject(backward_fn, backward_args);
egr::Controller::Instance().SetHasGrad(need_grad_tmp);
outputs = PyObject_CallObject(backward_fn, backward_args);
}
if (!outputs) {
std::string err_msg =
FormatPyLayerBackwardErrorMsg(this, pybind11::detail::error_string());
Expand Down
8 changes: 5 additions & 3 deletions paddle/fluid/pybind/eager_py_layer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -422,9 +422,11 @@ PyObject* pylayer_method_apply(PyObject* cls,
common::errors::InvalidArgument("Get forward function failed."));
}
bool trace_backward = egr::Controller::Instance().HasGrad();
egr::Controller::Instance().SetHasGrad(false);
auto outputs = PyObject_Call(forward_fn, forward_args, kwargs);
egr::Controller::Instance().SetHasGrad(trace_backward);
PyObject* outputs = nullptr;
{
egr::HasGradGuard has_grad_guard(false);
outputs = PyObject_Call(forward_fn, forward_args, kwargs);
}
if (!outputs) {
Py_XDECREF(forward_args);
Py_XDECREF(kwargs_value_list);
Expand Down
20 changes: 5 additions & 15 deletions paddle/fluid/pybind/eager_utils.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3090,8 +3090,7 @@ PackHook::~PackHook() { // NOLINT

std::shared_ptr<egr::PyObjectHolderBase> PackHook::operator()(
const Tensor& tensor) {
bool grad_tmp = egr::Controller::Instance().HasGrad();
egr::Controller::Instance().SetHasGrad(false);
egr::HasGradGuard has_grad_guard(false);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P3

这里修复的是一条用户可直接触发的泄漏路径:saved_tensors_hooks 的 pack hook 在 Python 中抛异常时,PyObject_Call 返回 NULL,旧代码在 PADDLE_ENFORCE_NOT_NULL(ret, ...) 处抛出而跳过 SetHasGrad(grad_tmp),此后线程内 grad 记录被关闭。新增测试只覆盖了 composite 反向(MatmulDoubleGradNode)路径,hook 这条路径没有回归覆盖。

建议在 test_grad_enabled_restore_on_backward_error.py 补一个用例:在 paddle.autograd.saved_tensors_hooks(pack_hook, unpack_hook) 中让 pack_hook 抛异常,执行 x * x(x.stop_gradient=False)并 assertRaises,之后断言 paddle.is_grad_enabled() 仍为 True 且新前向的 stop_gradient 为 False;unpack hook 抛异常的情况同理可选。

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已在 77214f6 中修复:新增 test_saved_tensors_pack_hook_error 与 test_saved_tensors_unpack_hook_error,分别覆盖 pack/unpack hook 抛异常后 grad 开关恢复。

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已补,感谢指出 —— 这条路径比我原先判断的更容易触发,实测确认是用户可直接触发的泄漏。

在 test_grad_enabled_restore_on_backward_error.py 中新增两个用例:

  • test_saved_tensors_pack_hook_error:pack hook 抛异常,覆盖 PackHook::operator()
  • test_saved_tensors_unpack_hook_error:unpack hook 在反向期间抛异常,覆盖 UnPackHook::operator()

注意异常类型是 OSError(errors::External),不是 composite 路径的 ValueError,因此把 check_restored 改成接受期望的异常类型。

修复前后对比(同一份测试):未修复构建 5 failed / 1 passed,修复后 6 passed。两条新用例都在未修复构建上失败,确认有回归价值。

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已核对:两条新用例分别覆盖 PackHook 与 UnPackHook 的抛异常路径,期望 OSError 与 errors::External 的映射一致(paddle/fluid/pybind/exception.cc:73)。此问题已解决。

::pybind11::gil_scoped_acquire gil;
PyObject* args = PyTuple_New(1);
PADDLE_ENFORCE_NOT_NULL(
Expand All @@ -3101,13 +3100,11 @@ std::shared_ptr<egr::PyObjectHolderBase> PackHook::operator()(
PADDLE_ENFORCE_NOT_NULL(
ret, common::errors::External(pybind11::detail::error_string().c_str()));
Py_XDECREF(args);
egr::Controller::Instance().SetHasGrad(grad_tmp);
return std::make_shared<PyObjectHolder>(ret);
}

void* PackHook::operator()(void* py_tensor) {
bool grad_tmp = egr::Controller::Instance().HasGrad();
egr::Controller::Instance().SetHasGrad(false);
egr::HasGradGuard has_grad_guard(false);
::pybind11::gil_scoped_acquire gil;
PyObject* args = PyTuple_New(1);
PADDLE_ENFORCE_NOT_NULL(
Expand All @@ -3122,7 +3119,6 @@ void* PackHook::operator()(void* py_tensor) {
PADDLE_ENFORCE_NOT_NULL(
ret, common::errors::External(pybind11::detail::error_string().c_str()));
Py_XDECREF(args);
egr::Controller::Instance().SetHasGrad(grad_tmp);
return reinterpret_cast<void*>(ret);
}

Expand All @@ -3135,8 +3131,7 @@ UnPackHook::~UnPackHook() { // NOLINT

Tensor UnPackHook::operator()(
std::shared_ptr<egr::PyObjectHolderBase> packed_value) {
bool grad_tmp = egr::Controller::Instance().HasGrad();
egr::Controller::Instance().SetHasGrad(false);
egr::HasGradGuard has_grad_guard(false);
::pybind11::gil_scoped_acquire gil;
PyObject* args = PyTuple_New(1);
PADDLE_ENFORCE_NOT_NULL(
Expand All @@ -3151,7 +3146,6 @@ Tensor UnPackHook::operator()(
// in it to be decremented by one, so no need to call
// Py_XDECREF(py_packed_value)
Py_XDECREF(args);
egr::Controller::Instance().SetHasGrad(grad_tmp);

PADDLE_ENFORCE_EQ(paddle::pybind::PyCheckTensor(ret),
true,
Expand All @@ -3165,8 +3159,7 @@ Tensor UnPackHook::operator()(
}

void* UnPackHook::operator()(void* packed_value, void* other) {
bool grad_tmp = egr::Controller::Instance().HasGrad();
egr::Controller::Instance().SetHasGrad(false);
egr::HasGradGuard has_grad_guard(false);
::pybind11::gil_scoped_acquire gil;
PyObject* args = PyTuple_New(1);
PADDLE_ENFORCE_NOT_NULL(
Expand All @@ -3181,7 +3174,6 @@ void* UnPackHook::operator()(void* packed_value, void* other) {
PADDLE_ENFORCE_NOT_NULL(
ret, common::errors::External(pybind11::detail::error_string().c_str()));
Py_XDECREF(args);
egr::Controller::Instance().SetHasGrad(grad_tmp);

PADDLE_ENFORCE_EQ(paddle::pybind::PyCheckTensor(ret),
true,
Expand Down Expand Up @@ -3264,8 +3256,7 @@ NodePostHook::operator()(
grad_outputs,
const paddle::small_vector<std::vector<Tensor>, egr::kSlotSmallVectorSize>&
grad_inputs) {
bool grad_tmp = egr::Controller::Instance().HasGrad();
egr::Controller::Instance().SetHasGrad(false);
egr::HasGradGuard has_grad_guard(false);
::pybind11::gil_scoped_acquire gil;
PyObject* args = PyTuple_New(2);
PADDLE_ENFORCE_NOT_NULL(
Expand All @@ -3276,7 +3267,6 @@ NodePostHook::operator()(
PADDLE_ENFORCE_NOT_NULL(
ret, common::errors::External(pybind11::detail::error_string().c_str()));
Py_XDECREF(args);
egr::Controller::Instance().SetHasGrad(grad_tmp);
return CastPyArg2SmallVectorOfVectorOfTensor(ret);
}

Expand Down
134 changes: 134 additions & 0 deletions test/legacy_test/test_grad_enabled_restore_on_backward_error.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import paddle


def build_failing_graph():
"""Build a graph whose second order backward is doomed to fail.

``dout`` has an illegal shape (it should be [4, 0]): matmul_grad itself
succeeds because of the 0-size fast return, but taking grad of its output
runs MatmulDoubleGradNode, whose composite implementation calls the matmul
forward api and fails the K dimension check in InferMeta.
"""
x = paddle.randn([4, 8])
y = paddle.randn([8, 0])
dout = paddle.randn([4, 3])
for tensor in (x, y, dout):
tensor.stop_gradient = False
dx, _ = paddle._C_ops.matmul_grad(x, y, dout, False, False)
return [x, y, dout], dx


class TestGradEnabledRestoreOnBackwardError(unittest.TestCase):
"""Raising from a backward node must not leak the global grad switch.

Composite (prim) backward nodes and the saved_tensors_hooks turn grad
recording off before running and turn it back on afterwards. When the code
in between throws, the restore must still happen, otherwise the whole
thread silently stops recording the computation graph.
"""

def check_restored(self, action, expected_error=ValueError):
paddle.set_grad_enabled(True)

with self.assertRaises(expected_error):
action()

self.assertTrue(paddle.is_grad_enabled())
probe = paddle.randn([2, 2])
probe.stop_gradient = False
self.assertFalse((probe * 2).stop_gradient)

def test_paddle_grad_without_create_graph(self):
def action():
inputs, dx = build_failing_graph()
paddle.grad(
[dx],
inputs,
grad_outputs=[paddle.randn(dx.shape)],
create_graph=False,
allow_unused=True,
)

self.check_restored(action)

def test_paddle_grad_with_create_graph(self):
def action():
inputs, dx = build_failing_graph()
paddle.grad(
[dx],
inputs,
grad_outputs=[paddle.randn(dx.shape)],
create_graph=True,
allow_unused=True,
)

self.check_restored(action)

def test_tensor_backward(self):
def action():
_, dx = build_failing_graph()
dx.backward(paddle.randn(dx.shape))

self.check_restored(action)

def test_tensor_backward_with_retain_graph(self):
def action():
_, dx = build_failing_graph()
dx.backward(paddle.randn(dx.shape), retain_graph=True)

self.check_restored(action)

def test_saved_tensors_pack_hook_error(self):
"""The pack hook runs in forward and reports failure by returning NULL."""

def pack_hook(tensor):
raise RuntimeError("pack hook fails on purpose")

def unpack_hook(value):
return value

def action():
x = paddle.randn([2, 2])
x.stop_gradient = False
with paddle.autograd.saved_tensors_hooks(pack_hook, unpack_hook):
return x * x

self.check_restored(action, OSError)

def test_saved_tensors_unpack_hook_error(self):
"""The unpack hook runs while the backward graph is being walked."""

def pack_hook(tensor):
return tensor

def unpack_hook(value):
raise RuntimeError("unpack hook fails on purpose")

def action():
x = paddle.randn([2, 2])
x.stop_gradient = False
with paddle.autograd.saved_tensors_hooks(pack_hook, unpack_hook):
out = (x * x).sum()
out.backward()

self.check_restored(action, OSError)


if __name__ == '__main__':
unittest.main()
Loading