From e059190ceba7e279b44656ecaca02c229bf54336 Mon Sep 17 00:00:00 2001 From: zhengshengning Date: Fri, 13 Nov 2026 11:35:09 +0800 Subject: [PATCH 1/3] [FSDP] Sum cp-partial grads over cp in the sharding reduce The sharding group spans cp, so averaging over it divided cp-partial grads by cp too. Divide by sharding_degree / cp instead; grads replicated across cp (context_parallel_disable_scale_grad, or expert_cp_replicated for experts) keep the full sharding_degree and get their own buffer. --- .../distributed/fsdp/fully_shard_fusion.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/python/paddle/distributed/fsdp/fully_shard_fusion.py b/python/paddle/distributed/fsdp/fully_shard_fusion.py index 7b7daca8b962fd..f936994f88f8dc 100644 --- a/python/paddle/distributed/fsdp/fully_shard_fusion.py +++ b/python/paddle/distributed/fsdp/fully_shard_fusion.py @@ -98,6 +98,7 @@ class BufferGroup: grads_use_sum: int = 0 grads_use_cnt: int = 0 grads_ready_cnt: int = 0 + cp_replicated: bool = False def _whole_3d_shard_numel(params, fsdp_group): @@ -447,6 +448,9 @@ def __init__( if use_ep else self._fsdp_group ) + cp_degree = getattr( + self.hcg, "get_context_parallel_world_size", lambda: 1 + )() self.main_grad_dtype = ( main_grad_dtype if main_grad_dtype is not None else paddle.float32 ) @@ -514,6 +518,14 @@ def __init__( ) ), getattr(param, "_fsdp_fp8_quantizer", None), + # Grad is replicated (not partial) across cp ranks. + bool( + getattr(param, "expert_cp_replicated", False) + if is_expert + else getattr( + param, "context_parallel_disable_scale_grad", False + ) + ), ) keyed_params.setdefault(key, []).append(param) @@ -531,6 +543,7 @@ def sort_key(item): _, _, _, + _, ) = item[0] return ( 0 if is_tie else (1 if not trainable else 2), @@ -552,6 +565,7 @@ def sort_key(item): keep_whole=keep_whole, hyperball_sharded=hyperball_name is not None, clip_attrs=clip_attrs, + cp_replicated=cp_replicated, ) for ( dtype, @@ -566,6 +580,7 @@ def sort_key(item): hyperball_name, clip_attrs, _, + cp_replicated, ), params in sorted(keyed_params.items(), key=sort_key) ] @@ -621,7 +636,8 @@ def sort_key(item): main_grad_dtype=paddle.float32 if group.is_expert_param or group.dtype == paddle.float32 else self.main_grad_dtype, - grad_div=self._fsdp_group.nranks, + grad_div=self._fsdp_group.nranks + // (1 if group.cp_replicated else cp_degree), shard_buffer=grads_shard_buffer, shard_align_numel=shard_align, grad_comm_dtype=self.grad_comm_dtype, From ebee2172cf72cc9f973d8c68b5100966b632e80f Mon Sep 17 00:00:00 2001 From: zhengshengning Date: Fri, 13 Nov 2026 14:05:26 +0800 Subject: [PATCH 2/3] [FSDP] Only split buffers by cp replication when cp > 1 At cp = 1 both kinds of grads use the same grad_div, so splitting buffers only changes buffer count and fuse_params_ numbering. Keep the grouping identical to before when cp is disabled. --- python/paddle/distributed/fsdp/fully_shard_fusion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/distributed/fsdp/fully_shard_fusion.py b/python/paddle/distributed/fsdp/fully_shard_fusion.py index f936994f88f8dc..629e1cad94634e 100644 --- a/python/paddle/distributed/fsdp/fully_shard_fusion.py +++ b/python/paddle/distributed/fsdp/fully_shard_fusion.py @@ -519,7 +519,8 @@ def __init__( ), getattr(param, "_fsdp_fp8_quantizer", None), # Grad is replicated (not partial) across cp ranks. - bool( + cp_degree > 1 + and bool( getattr(param, "expert_cp_replicated", False) if is_expert else getattr( From 90896ebb158cca1bbaf2347873124755a1d6f68c Mon Sep 17 00:00:00 2001 From: zhengshengning Date: Fri, 13 Nov 2026 14:08:37 +0800 Subject: [PATCH 3/3] [FSDP] Add unit test for grads buffer grad_div under cp Check that cp-partial grads divide by sharding / cp, cp-replicated grads divide by sharding in their own buffers, and cp = 1 keeps the grouping. --- .../test_fsdp_fully_shard_fusion.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/test/legacy_test/test_fsdp_fully_shard_fusion.py b/test/legacy_test/test_fsdp_fully_shard_fusion.py index 109bc2dc31a396..fb6141b854ab09 100644 --- a/test/legacy_test/test_fsdp_fully_shard_fusion.py +++ b/test/legacy_test/test_fsdp_fully_shard_fusion.py @@ -24,10 +24,13 @@ back untouched while routing the rest through the backward all_gather hook. * ``unshard_params`` -- the synchronous gather / reshard context manager and the double-buffer bookkeeping it restores on exit. +* the grads buffer ``grad_div`` under context parallel. """ +import contextlib import types import unittest +from unittest import mock import numpy as np @@ -947,5 +950,99 @@ def test_after_step_keeps_through_backward(self): ) +class _FakeGroup: + def __init__(self, nranks): + self.nranks = nranks + self.id = 0 + + +class _FakeHcg: + def __init__(self, sharding_nranks, cp_degree): + self._group = _FakeGroup(sharding_nranks) + self._cp_degree = cp_degree + + def get_sharding_parallel_group(self): + return self._group + + def get_expert_parallel_world_size(self): + return 1 + + def get_context_parallel_world_size(self): + return self._cp_degree + + +class _RecordingFusionBuffer: + """Records grad_div instead of allocating buffers.""" + + def __init__(self, gid, params, fsdp_group, dtype, **kwargs): + self.grad_div = kwargs.get("grad_div", 1) + self.is_sharded = True + self.total_buffer_size = 0 + self.shard_align_numel = None + self.data_buffer = types.SimpleNamespace() + + def param_shard(self, param): + return True + + +class TestCpGradDiv(unittest.TestCase): + """The grads buffer divisor must sum cp-partial grads over cp.""" + + def _build(self, cp_degree, sharding_nranks=4): + model = paddle.nn.Layer() + params = {} + for name in ("plain", "dense_rep", "expert", "expert_rep"): + params[name] = model.create_parameter([4, 4]) + setattr(model, name, params[name]) + params["dense_rep"].context_parallel_disable_scale_grad = True + params["expert"].is_moe_param = True + params["expert_rep"].is_moe_param = True + params["expert_rep"].expert_cp_replicated = True + + hcg = _FakeHcg(sharding_nranks, cp_degree) + with contextlib.ExitStack() as stack: + stack.enter_context( + mock.patch.object( + fsf.fleet, "get_hybrid_communicate_group", return_value=hcg + ) + ) + stack.enter_context( + mock.patch.object( + fsf, "TensorFusionBuffer", _RecordingFusionBuffer + ) + ) + stack.enter_context( + mock.patch.object( + fsf.FSDPBufferManager, "_comm_stream", return_value=None + ) + ) + manager = fsf.FSDPBufferManager(model) + + def group_of(name): + gid = manager.param_to_buffer_id[params[name].name] + return manager.buffer_groups[gid] + + return manager.buffer_groups, group_of + + def test_cp_partial_grads_summed_over_cp(self): + groups, group_of = self._build(cp_degree=2) + # cp-partial grads: divide by sharding / cp. + self.assertEqual(group_of("plain").grads_buffer.grad_div, 2) + self.assertEqual(group_of("expert").grads_buffer.grad_div, 2) + # cp-replicated grads: average over the whole sharding group. + self.assertEqual(group_of("dense_rep").grads_buffer.grad_div, 4) + self.assertEqual(group_of("expert_rep").grads_buffer.grad_div, 4) + self.assertEqual(len(groups), 4) + + def test_cp_one_keeps_grouping(self): + groups, group_of = self._build(cp_degree=1) + # cp marks must not split buffers when cp is disabled. + self.assertIs(group_of("plain"), group_of("dense_rep")) + self.assertIs(group_of("expert"), group_of("expert_rep")) + self.assertEqual(len(groups), 2) + for group in groups: + self.assertEqual(group.grads_buffer.grad_div, 4) + + if __name__ == "__main__": unittest.main()