Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion python/paddle/distributed/fsdp/fully_shard_fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ class BufferGroup:
grads_use_sum: int = 0
grads_use_cnt: int = 0
grads_ready_cnt: int = 0
cp_replicated: bool = False


def _whole_3d_shard_numel(params, fsdp_group):
Expand Down Expand Up @@ -447,6 +448,9 @@ def __init__(
if use_ep
else self._fsdp_group
)
cp_degree = getattr(
self.hcg, "get_context_parallel_world_size", lambda: 1
)()
self.main_grad_dtype = (
main_grad_dtype if main_grad_dtype is not None else paddle.float32
)
Expand Down Expand Up @@ -514,6 +518,15 @@ def __init__(
)
),
getattr(param, "_fsdp_fp8_quantizer", None),
# Grad is replicated (not partial) across cp ranks.
Comment thread
zhengshengning marked this conversation as resolved.
Resolved
cp_degree > 1
and bool(
getattr(param, "expert_cp_replicated", False)
if is_expert
else getattr(
param, "context_parallel_disable_scale_grad", False
)
),
)
keyed_params.setdefault(key, []).append(param)

Expand All @@ -531,6 +544,7 @@ def sort_key(item):
_,
_,
_,
_,
) = item[0]
return (
0 if is_tie else (1 if not trainable else 2),
Expand All @@ -552,6 +566,7 @@ def sort_key(item):
keep_whole=keep_whole,
hyperball_sharded=hyperball_name is not None,
clip_attrs=clip_attrs,
cp_replicated=cp_replicated,
)
for (
dtype,
Expand All @@ -566,6 +581,7 @@ def sort_key(item):
hyperball_name,
clip_attrs,
_,
cp_replicated,
), params in sorted(keyed_params.items(), key=sort_key)
]

Expand Down Expand Up @@ -621,7 +637,8 @@ def sort_key(item):
main_grad_dtype=paddle.float32
if group.is_expert_param or group.dtype == paddle.float32
else self.main_grad_dtype,
grad_div=self._fsdp_group.nranks,
grad_div=self._fsdp_group.nranks
// (1 if group.cp_replicated else cp_degree),
Comment on lines +640 to +641

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 这一行改变了 cp>1 时所有 grads buffer 的梯度除数,属于精度关键路径,但 PR 没有补测试;test/legacy_test/test_fsdp_fully_shard_fusion.py 和 test/collective/fleet/dygraph_group_fsdp.py 都没有覆盖 cp_degree > 1。建议补一个单进程用例:用 fake hcg(get_context_parallel_world_size() 返回 2、sharding 组 nranks=4)构造 FSDPBufferManager,断言未标记参数的 grads_buffer.grad_div == 2、带 context_parallel_disable_scale_grad / expert_cp_replicated 的参数落在独立 buffer 且 grad_div == 4,防止后续改动把系数改回去。

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已在 90896eb 补充 TestCpGradDiv:cp=2 时校验部分梯度 grad_div == 2、两类复制梯度 grad_div == 4 且落在独立 buffer,cp=1 时校验分组不变、grad_div == 4。

shard_buffer=grads_shard_buffer,
shard_align_numel=shard_align,
grad_comm_dtype=self.grad_comm_dtype,
Expand Down
97 changes: 97 additions & 0 deletions test/legacy_test/test_fsdp_fully_shard_fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,14 @@
back untouched while routing the rest through the backward all_gather hook.
* ``unshard_params`` -- the synchronous gather / reshard context manager and
the double-buffer bookkeeping it restores on exit.
* the grads buffer ``grad_div`` under context parallel.
* ``full`` -- export through CPU copies of the released params.
"""

import contextlib
import types
import unittest
from unittest import mock

import numpy as np

Expand Down Expand Up @@ -973,5 +976,99 @@ def test_after_step_keeps_through_backward(self):
)


class _FakeGroup:
def __init__(self, nranks):
self.nranks = nranks
self.id = 0


class _FakeHcg:
def __init__(self, sharding_nranks, cp_degree):
self._group = _FakeGroup(sharding_nranks)
self._cp_degree = cp_degree

def get_sharding_parallel_group(self):
return self._group

def get_expert_parallel_world_size(self):
return 1

def get_context_parallel_world_size(self):
return self._cp_degree


class _RecordingFusionBuffer:
"""Records grad_div instead of allocating buffers."""

def __init__(self, gid, params, fsdp_group, dtype, **kwargs):
self.grad_div = kwargs.get("grad_div", 1)
self.is_sharded = True
self.total_buffer_size = 0
self.shard_align_numel = None
self.data_buffer = types.SimpleNamespace()

def param_shard(self, param):
return True


class TestCpGradDiv(unittest.TestCase):
"""The grads buffer divisor must sum cp-partial grads over cp."""

def _build(self, cp_degree, sharding_nranks=4):
model = paddle.nn.Layer()
params = {}
for name in ("plain", "dense_rep", "expert", "expert_rep"):
params[name] = model.create_parameter([4, 4])
setattr(model, name, params[name])
params["dense_rep"].context_parallel_disable_scale_grad = True
params["expert"].is_moe_param = True
params["expert_rep"].is_moe_param = True
params["expert_rep"].expert_cp_replicated = True

hcg = _FakeHcg(sharding_nranks, cp_degree)
with contextlib.ExitStack() as stack:
stack.enter_context(
mock.patch.object(
fsf.fleet, "get_hybrid_communicate_group", return_value=hcg
)
)
stack.enter_context(
mock.patch.object(
fsf, "TensorFusionBuffer", _RecordingFusionBuffer
)
)
stack.enter_context(
mock.patch.object(
fsf.FSDPBufferManager, "_comm_stream", return_value=None
)
)
manager = fsf.FSDPBufferManager(model)

def group_of(name):
gid = manager.param_to_buffer_id[params[name].name]
return manager.buffer_groups[gid]

return manager.buffer_groups, group_of

def test_cp_partial_grads_summed_over_cp(self):
groups, group_of = self._build(cp_degree=2)
# cp-partial grads: divide by sharding / cp.
self.assertEqual(group_of("plain").grads_buffer.grad_div, 2)
self.assertEqual(group_of("expert").grads_buffer.grad_div, 2)
# cp-replicated grads: average over the whole sharding group.
self.assertEqual(group_of("dense_rep").grads_buffer.grad_div, 4)
self.assertEqual(group_of("expert_rep").grads_buffer.grad_div, 4)
self.assertEqual(len(groups), 4)

def test_cp_one_keeps_grouping(self):
groups, group_of = self._build(cp_degree=1)
# cp marks must not split buffers when cp is disabled.
self.assertIs(group_of("plain"), group_of("dense_rep"))
self.assertIs(group_of("expert"), group_of("expert_rep"))
self.assertEqual(len(groups), 2)
for group in groups:
self.assertEqual(group.grads_buffer.grad_div, 4)


if __name__ == "__main__":
unittest.main()
Loading