Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 25 additions & 12 deletions python/paddle/distributed/fsdp/fully_shard_fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -2496,18 +2496,31 @@ def _forward_post_hook(layer, inputs, outputs):
exposed = _find_exposed_param_names(outputs, own_param_names)

if isinstance(outputs, dict):
for key, value in outputs.items():
if (
isinstance(value, paddle.Tensor)
and not value.stop_gradient
and value.name not in exposed
):
outputs[key] = FusionBackwardHook.apply(
value,
layer=layer,
comm_manager=self.comm_manager,
recursive=recursive,
)
# Route every hookable value through ONE identity PyLayer, as
# the tuple branch does. With a PyLayer per value, the first
# value whose gradient arrives (e.g. an entry a later layer
# passes through untouched) all-gathers this unit early, and
# _release_params_after_unit then frees later units whose
# recompute backward has not run yet.
keys = [
key
for key, value in outputs.items()
if isinstance(value, paddle.Tensor)
and not value.stop_gradient
and value.name not in exposed
]
if not keys:
return outputs
hooked = FusionBackwardHook.apply(
*[outputs[key] for key in keys],
layer=layer,
comm_manager=self.comm_manager,
recursive=recursive,
)
if not isinstance(hooked, (tuple, list)):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 可维护性建议(逻辑清晰度,不阻塞合入):FusionBackwardHook.forward(1484 行)只会返回 tuple 或单个 Tensor,list 分支走不到,而且与紧邻 tuple 分支 2546 行的 isinstance(hooked, tuple) 写法不一致,读者容易以为两处的返回约定不同。建议统一为 if not isinstance(hooked, tuple):。

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个不影响正确性,暂时可以先推进合入

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

收到,可维护性建议不阻塞合入,已标记为接受。

hooked = (hooked,)
for key, value in zip(keys, hooked):
outputs[key] = value
return outputs
elif isinstance(outputs, tuple):
# Exposed parameters are leaf vars that require grad, which the
Expand Down
70 changes: 70 additions & 0 deletions test/legacy_test/test_fsdp_fully_shard_fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
* the fused-kernel fp32 gradient hand-off inside the grad hook.
* the tuple branch of the forward post hook that splices exposed parameters
back untouched while routing the rest through the backward all_gather hook.
* the dict branch of the forward post hook, which must route all hookable
values through a single backward hook so the unit is all-gathered once.
* ``unshard_params`` -- the synchronous gather / reshard context manager and
the double-buffer bookkeeping it restores on exit.
* the grads buffer ``grad_div`` under context parallel.
Expand Down Expand Up @@ -224,6 +226,74 @@ def test_no_exposed_params_routes_all_through_hook(self):
np.testing.assert_array_equal(result[1].numpy(), b.numpy())


class _GatherCountingCommManager(_NoopCommManager):
"""Counts the backward all_gather calls issued by ``FusionBackwardHook``."""

def __init__(self):
super().__init__()
self.backward_gathers = 0

def all_gather_params(self, params, is_backward=False):
if is_backward:
self.backward_gathers += 1


class TestForwardPostHookDictBranch(unittest.TestCase):
"""Covers the dict branch of the forward post hook built by
``_register_fusion_layer_hooks``."""

def _capture_post_hook(self, param_names):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 可维护性建议(AI 生成债务,不阻塞合入):这个 _capture_post_hook 与 TestForwardPostHookTupleBranch._capture_post_hook(192 行)逐行相同,只有 comm_manager 类型和返回值不同。_register_fusion_layer_hooks 的签名变化时需要同步改两处。建议抽成一个模块级 helper,例如 _make_hooked_layer(param_names, comm_manager) 返回 (fsdp, layer),两个测试类共用。

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个不影响正确性,暂时可以先推进合入

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

收到,可维护性建议不阻塞合入,已标记为接受。

fsdp = object.__new__(fsf.FullyShardFusion)
fsdp.comm_manager = _GatherCountingCommManager()
layer = _FakeLayer([_NamedParam(n) for n in param_names])
fsdp._register_fusion_layer_hooks(layer, recursive=False)
self.assertIsNotNone(layer.post_hook)
return fsdp, layer

def test_multiple_values_all_gather_once_in_backward(self):
# A PyLayer per value would all-gather the unit once per value, the
# first time as soon as any single value's gradient arrives.
fsdp, layer = self._capture_post_hook([])
x = paddle.to_tensor([1.0, 2.0], stop_gradient=False)
outputs = {"a": x * 1.0, "b": x * 2.0}
result = layer.post_hook(layer, (), outputs)
(result["a"].sum() + result["b"].sum()).backward()
self.assertEqual(fsdp.comm_manager.backward_gathers, 1)
np.testing.assert_allclose(x.grad.numpy(), [3.0, 3.0])

def test_unhookable_values_kept_untouched(self):
hidden = paddle.to_tensor([1.0, 2.0], stop_gradient=False) * 1.0
frozen = paddle.to_tensor([5.0]) # stop_gradient=True
weight = paddle.to_tensor([3.0, 4.0]) # exposed LM-head weight

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P3 这里的 weight 由 paddle.to_tensor 创建,默认 stop_gradient=True,dict 分支第一个条件 not value.stop_gradient(fully_shard_fusion.py:2509)就已把它排除。所以即使删掉 value.name not in exposed 这个按名字过滤 exposed 参数的条件,assertIs(result["w"], weight) 依然成立,用例没有覆盖 PR 描述中“exposed 参数保持不变”的行为。实际的 exposed 参数(如 LM head weight)是 stop_gradient=False 的叶子变量,送进 PyLayer 会报错。

建议改为 weight = paddle.to_tensor([3.0, 4.0], stop_gradient=False),让按名字过滤成为该断言成立的唯一原因。

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个不影响正确性,暂时可以先推进合入

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

收到。这是 P3 的测试覆盖问题,确实不影响修复本身的正确性,接受先合入,建议后续顺手补上 stop_gradient=False 的用例。

_, layer = self._capture_post_hook([weight.name])
outputs = {"h": hidden, "frozen": frozen, "w": weight, "n": 7}
result = layer.post_hook(layer, (), outputs)
self.assertEqual(list(result.keys()), ["h", "frozen", "w", "n"])
self.assertIs(result["frozen"], frozen)
self.assertIs(result["w"], weight)
self.assertEqual(result["n"], 7)
np.testing.assert_array_equal(result["h"].numpy(), [1.0, 2.0])

def test_nothing_hookable_returns_outputs_without_hook(self):
_, layer = self._capture_post_hook([])
outputs = {"frozen": paddle.to_tensor([1.0]), "n": 3}
with mock.patch.object(fsf.FusionBackwardHook, "apply") as apply:
result = layer.post_hook(layer, (), outputs)
apply.assert_not_called()
self.assertIs(result, outputs)

def test_single_hookable_value_written_back(self):
# FusionBackwardHook.forward returns a bare tensor for one input.
fsdp, layer = self._capture_post_hook([])
x = paddle.to_tensor([1.0, 2.0], stop_gradient=False)
outputs = {"a": x * 2.0, "n": 1}
result = layer.post_hook(layer, (), outputs)
self.assertIsInstance(result["a"], paddle.Tensor)
result["a"].sum().backward()
self.assertEqual(fsdp.comm_manager.backward_gathers, 1)
np.testing.assert_allclose(x.grad.numpy(), [2.0, 2.0])


class _FakeParamsBuffer:
def __init__(self, status, is_sharded=True):
self.status = status
Expand Down
Loading