Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions python/paddle/distributed/flex_checkpoint/dcp/full_param.py
Original file line number Diff line number Diff line change
Expand Up @@ -764,6 +764,8 @@ def _process_one_batch(self) -> list[str]:
)
else:
buffer = paddle.empty(read_item.slice_shape, dtype=read_item.dtype)
if buffer.place.is_cpu_place(): # NCCL needs GPU; outputs stay on GPU
buffer = buffer.cuda()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 可维护性建议(逻辑清晰度,不阻塞合入):同文件 _run_distributed(第 525-531 行)对 CPU 源张量是 cuda() 通信后再 .cpu() 搬回;这里只做了搬上 GPU,结果会以 GPU 张量的形式缓存进 sharded_desc_to_tensor。两条路径的设备语义不一致,读者无法判断这是有意为之还是遗漏。建议对齐 on_cpu 的写法,或者加一行注释说明 HV 路径的输出固定在 GPU 上,由 memory_growth_threshold 控制峰值。


if op_type == OperationType.BROADCAST_ALLGATHER:
if not self.using_1d_comm_group:
Expand Down
77 changes: 77 additions & 0 deletions python/paddle/distributed/fsdp/fully_shard_fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -1540,6 +1540,8 @@ def __init__(
# Redirect the model's own API; the original stays as our description base.
self._model_sharded_state_dict = self.model.sharded_state_dict
self.model.sharded_state_dict = self.sharded_state_dict
self._model_full = self.model.full
self.model.full = self.full

def _base_sharded_state_dict(self, structured_name_prefix):
"""Model descriptions of every param; fused params get a scratch view since their storage is cleared."""
Expand Down Expand Up @@ -1668,6 +1670,81 @@ def sharded_state_dict(self, structured_name_prefix=""):
self._shard_descs[params_buffer.data_buffer.name] = param_slice_info
return result

def _model_description(self, *args, **kwargs):
"""The model's own description, with experts folded as in sharded_state_dict."""
result = self._model_sharded_state_dict(*args, **kwargs)
hcg = self.buffer_manager.hcg
if not hasattr(hcg, "get_expert_parallel_world_size"):
return result
ep_nranks = hcg.get_expert_parallel_world_size()
if ep_nranks <= 1:
return result
ep_rank = hcg.get_expert_parallel_rank()
experts = {
param.name
for group in self.buffer_manager.buffer_groups
if group.is_expert_param
for param in group.params
}
for sharded_weight in result.values():
local_shape = tuple(sharded_weight.local_shape)
if (
sharded_weight.local_tensor.name not in experts
or tuple(sharded_weight.global_shape) != local_shape
):
continue
# Expert described as replicated: fold EP into axis 0 to keep keys distinct.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 可维护性建议(中层优雅度,不阻塞合入):这里的 EP 折叠公式和注释与 sharded_state_dict 第 1632-1641 行逐行相同。两处必须保持一致,full() 导出的 key 才能和 checkpoint 描述对齐,以后只改一处就会让导出与保存悄悄分叉。建议抽一个小函数,如 _fold_expert_desc(local_shape, ep_nranks, ep_rank) -> (global_shape, global_offset),两处共用;ep_nranks/ep_rank 的获取也可以一并复用。

sharded_weight.global_shape = (
local_shape[0] * ep_nranks,
*local_shape[1:],
)
sharded_weight.global_offset = (
ep_rank * local_shape[0],
*(0 for _ in local_shape[1:]),
)
return result

def full(self, aoa_config=None, **kwargs):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 可维护性建议(契约一致性,不阻塞合入):Layer.full 是普通函数,调用时会立即校验 h_group/v_group、解析 kwargs,并返回 __iter__ 可重复调用的 assembler。这里替换成生成器函数之后,参数错误要等到第一次 next() 才抛出,返回值也只能迭代一次。建议改为先完成参数绑定,再 return 一个负责收尾的迭代器;如果保留生成器写法,请在 docstring 中写明返回只能迭代一次,并说明 model.sharded_state_dict 会被临时替换,同时补上与 Layer.full 一致的类型标注。

"""Gather released params to CPU and run the model's own full; iterate once to the end or close()."""
released = []
try:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2

try/finally 从第 1704 行才开始,第 1676-1702 行的收集循环不在保护范围内。循环里已经对前面若干 group 执行了 value._share_buffer_to(param),如果后面某个 group 的 paddle.empty([total_buffer_size]) 申请显存失败(大 buffer 时可能 OOM),或者 all_gather、build_ckpt_weight 抛错,前面这些参数会一直绑在 CPU 数据上(_is_initialized() 为 True),后续 forward 的 all_gather_params 会在这份 CPU 存储上做 _share_data_with,状态不一致。_base_sharded_state_dict 同样是"先绑定、再还原",但它的回滚覆盖了完整流程。

另外 full() 现在是生成器:调用方如果中途 break,或者没迭代完就丢弃,finally 要等到 GC 才会执行,在此之前参数保持 CPU 绑定,model.sharded_state_dict 也还停在原始实现上。

建议把 released = [] 之后的收集循环和第 1703 行的替换一起放进 try,finally 只还原 released 里已经绑定的参数;同时在 docstring 中说明调用方必须迭代完或显式 close()。

for group in self.buffer_manager.buffer_groups:
params = [p for p in group.params if not p._is_initialized()]
if not params:
continue
params_buffer = group.params_buffer
weight = params_buffer.data_buffer
if params_buffer.fp8 is not None:
weight = build_ckpt_weight(params_buffer, group.dtype)
gathered = weight
if params_buffer.is_sharded:
gathered = paddle.empty(
[params_buffer.total_buffer_size], dtype=weight.dtype
)
paddle.distributed.stream.all_gather(
gathered,
weight,
group=params_buffer.fsdp_group,
sync_op=True,
use_calc_stream=True,
)
for param in params:
offset = params_buffer.param_offsets[param.name]
value = gathered._slice(offset, offset + param._numel())
value = value.reshape(param.shape).cpu()
released.append((param, param.shape))
value._share_buffer_to(param)
del gathered, weight
self.model.sharded_state_dict = self._model_description
yield from self._model_full(aoa_config, **kwargs)
finally:
self.model.sharded_state_dict = self.sharded_state_dict
for param, shape in released:
stop_gradient = param.stop_gradient

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 可维护性建议(底层优雅度,不阻塞合入):第 1708-1712 行"保存 stop_gradient → _clear_data() → 恢复 stop_gradient → _set_dims(shape)"与本类 _base_sharded_state_dict 的 finally(第 1568-1573 行)、shard_params、_release_params_after_unit 逐行相同,测试里又新增了一个 _release() 副本。建议抽一个模块级小函数(如 _release_param(param, shape)),这次新增的代码和测试先复用,以后释放语义有调整时只需改一处。

param._clear_data()
param.stop_gradient = stop_gradient
param.get_tensor()._set_dims(shape)

def bind_decay_param_fun(self, optimizer, buffer_groups=None):
"""Answer ``apply_decay_param_fun`` for the fused buffers.

Expand Down
54 changes: 54 additions & 0 deletions test/collective/fleet/dygraph_group_fsdp.py
Original file line number Diff line number Diff line change
Expand Up @@ -1463,8 +1463,62 @@ def run_grad_comm_dtype():
np.testing.assert_allclose(new_losses, ref_losses, rtol=1e-2)


def run_full_export():
"""model.full() exports the trained params over the EP groups, then releases them."""
hcg = fleet.get_hybrid_communicate_group()
ep_group = hcg.get_expert_parallel_group()
_, model = build_moe_models(hcg)
expert_names = {
p.name for layer in model.layers for p in layer.moe.experts.parameters()
}
key_to_name = {k: v.name for k, v in model.state_dict().items()}
opt = paddle.optimizer.AdamW(
learning_rate=0.001, parameters=model.parameters(), weight_decay=0.0
)
model = fully_shard(model, optimizer=opt)
ctx = model._fsdp_context
paddle.seed(11)
x = paddle.randn([TOKENS, HIDDEN])

def step():
model(x).mean().backward()
opt.step()
opt.clear_grad()

step()
params = list(model.parameters())
with ctx.unshard_params(params):
expected = {p.name: p.numpy() for p in params}
released = [p for p in params if not p._is_initialized()]
assert released, "no released param to export"

out = dict(
model.full(
h_group=ep_group,
v_group=hcg.get_pipe_parallel_group(),
num_splits=1,
shard_idx=0,
)
)
for key, name in key_to_name.items():
value = expected[name]
if name in expert_names:
# Experts are described as replicated: EP folds into axis 0.
parts = []
paddle.distributed.all_gather(
parts, paddle.to_tensor(value), group=ep_group
)
value = np.concatenate([part.numpy() for part in parts], axis=0)
np.testing.assert_array_equal(out[key].numpy(), value, err_msg=key)

assert all(not p._is_initialized() for p in released)
assert model.sharded_state_dict == ctx.sharded_state_dict
step()


if __name__ == '__main__':
run_moe(2)
run_full_export()
run_fused_linear()
run_grad_comm_dtype()
run_muon_shard()
Expand Down
26 changes: 26 additions & 0 deletions test/legacy_test/test_fsdp_fully_shard_fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
back untouched while routing the rest through the backward all_gather hook.
* ``unshard_params`` -- the synchronous gather / reshard context manager and
the double-buffer bookkeeping it restores on exit.
* ``full`` -- export through CPU copies of the released params.
"""

import types
Expand Down Expand Up @@ -452,6 +453,31 @@ def _fake_grads_buffer(numel, waited, i):
)


class TestFullToCpu(unittest.TestCase):
def test_model_full_sees_cpu_params_then_they_are_released(self):
w = paddle.create_parameter([2, 3], "float32")
w._clear_data()
w.get_tensor()._set_dims([2, 3])
fsdp = object.__new__(fsf.FullyShardFusion)
params_buffer = types.SimpleNamespace(
data_buffer=paddle.arange(6, dtype="float32"),
fp8=None,
is_sharded=False,
param_offsets={w.name: 0},
)
group = types.SimpleNamespace(params=[w], params_buffer=params_buffer)
fsdp.buffer_manager = types.SimpleNamespace(
buffer_groups=[group], hcg=None
)
fsdp.model = types.SimpleNamespace()
fsdp._model_sharded_state_dict = dict
fsdp._model_full = lambda aoa_config=None: iter([(w.name, w.numpy())])
((name, value),) = fsdp.full()
np.testing.assert_array_equal(value, np.arange(6).reshape(2, 3))
self.assertFalse(w._is_initialized())
self.assertEqual(fsdp.model.sharded_state_dict, fsdp.sharded_state_dict)


class TestPrefetchByLayer(unittest.TestCase):
"""Covers the default per-unit prefetch of FSDPCommManager."""

Expand Down
Loading