From 03ef16adbd1c66bd98e2807cdb27677c29d24c79 Mon Sep 17 00:00:00 2001 From: lizhenxing02 Date: Thu, 12 Nov 2026 12:23:30 +0800 Subject: [PATCH 1/2] supp_save_hf_ckpt --- .../flex_checkpoint/dcp/full_param.py | 2 + .../distributed/fsdp/fully_shard_fusion.py | 43 +++++++++++++++ .../test_fsdp_fully_shard_fusion.py | 52 +++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/python/paddle/distributed/flex_checkpoint/dcp/full_param.py b/python/paddle/distributed/flex_checkpoint/dcp/full_param.py index e677cb50a4aa6c..032a816dbe6ecc 100644 --- a/python/paddle/distributed/flex_checkpoint/dcp/full_param.py +++ b/python/paddle/distributed/flex_checkpoint/dcp/full_param.py @@ -764,6 +764,8 @@ def _process_one_batch(self) -> list[str]: ) else: buffer = paddle.empty(read_item.slice_shape, dtype=read_item.dtype) + if buffer.place.is_cpu_place(): + buffer = buffer.cuda() if op_type == OperationType.BROADCAST_ALLGATHER: if not self.using_1d_comm_group: diff --git a/python/paddle/distributed/fsdp/fully_shard_fusion.py b/python/paddle/distributed/fsdp/fully_shard_fusion.py index 7b7daca8b962fd..13ac8561192d77 100644 --- a/python/paddle/distributed/fsdp/fully_shard_fusion.py +++ b/python/paddle/distributed/fsdp/fully_shard_fusion.py @@ -1540,6 +1540,8 @@ def __init__( # Redirect the model's own API; the original stays as our description base. self._model_sharded_state_dict = self.model.sharded_state_dict self.model.sharded_state_dict = self.sharded_state_dict + self._model_full = self.model.full + self.model.full = self.full def _base_sharded_state_dict(self, structured_name_prefix): """Model descriptions of every param; fused params get a scratch view since their storage is cleared.""" @@ -1668,6 +1670,47 @@ def sharded_state_dict(self, structured_name_prefix=""): self._shard_descs[params_buffer.data_buffer.name] = param_slice_info return result + def full(self, aoa_config=None, **kwargs): + """Gather released params to CPU, then run the model's own full.""" + released = [] + for group in self.buffer_manager.buffer_groups: + params = [p for p in group.params if not p._is_initialized()] + if not params: + continue + params_buffer = group.params_buffer + full = params_buffer.data_buffer + if params_buffer.fp8 is not None: + full = build_ckpt_weight(params_buffer, group.dtype) + if params_buffer.is_sharded: + shard = full + full = paddle.empty( + [params_buffer.total_buffer_size], dtype=shard.dtype + ) + paddle.distributed.stream.all_gather( + full, + shard, + group=params_buffer.fsdp_group, + sync_op=True, + use_calc_stream=True, + ) + for param in params: + offset = params_buffer.param_offsets[param.name] + value = full._slice(offset, offset + param._numel()) + value = value.reshape(param.shape).cpu() + released.append((param, param.shape)) + value._share_buffer_to(param) + del full + self.model.sharded_state_dict = self._model_sharded_state_dict + try: + yield from self._model_full(aoa_config, **kwargs) + finally: + self.model.sharded_state_dict = self.sharded_state_dict + for param, shape in released: + stop_gradient = param.stop_gradient + param._clear_data() + param.stop_gradient = stop_gradient + param.get_tensor()._set_dims(shape) + def bind_decay_param_fun(self, optimizer, buffer_groups=None): """Answer ``apply_decay_param_fun`` for the fused buffers. diff --git a/test/legacy_test/test_fsdp_fully_shard_fusion.py b/test/legacy_test/test_fsdp_fully_shard_fusion.py index 109bc2dc31a396..ed48c452e6a0b2 100644 --- a/test/legacy_test/test_fsdp_fully_shard_fusion.py +++ b/test/legacy_test/test_fsdp_fully_shard_fusion.py @@ -24,6 +24,7 @@ back untouched while routing the rest through the backward all_gather hook. * ``unshard_params`` -- the synchronous gather / reshard context manager and the double-buffer bookkeeping it restores on exit. +* ``full`` -- export through CPU copies of the released params. """ import types @@ -452,6 +453,57 @@ def _fake_grads_buffer(numel, waited, i): ) +def _release(param): + shape = param.shape + param._clear_data() + param.get_tensor()._set_dims(shape) + + +class TestFullToCpu(unittest.TestCase): + def test_model_full_sees_cpu_params_then_they_are_released(self): + w0 = paddle.create_parameter([2, 3], "float32") + w1 = paddle.create_parameter([4], "float32") + _release(w0) + _release(w1) + params_buffer = types.SimpleNamespace( + data_buffer=paddle.arange(10, dtype="float32"), + fp8=None, + is_sharded=False, + param_offsets={w0.name: 0, w1.name: 6}, + ) + group = types.SimpleNamespace( + params=[w0, w1], params_buffer=params_buffer + ) + fsdp = object.__new__(fsf.FullyShardFusion) + fsdp.buffer_manager = types.SimpleNamespace(buffer_groups=[group]) + fsdp.model = types.SimpleNamespace() + fsdp._model_sharded_state_dict = lambda: "model description" + seen = {} + + def model_full(aoa_config=None, **kwargs): + seen["description"] = fsdp.model.sharded_state_dict() + seen["kwargs"] = kwargs + for p in (w0, w1): + yield p.name, (p.numpy().copy(), p.place.is_cpu_place()) + + fsdp._model_full = model_full + out = dict(fsdp.full(num_splits=2, shard_idx=1)) + + self.assertEqual(seen["description"], "model description") + self.assertEqual(seen["kwargs"], {"num_splits": 2, "shard_idx": 1}) + np.testing.assert_array_equal( + out[w0.name][0], np.arange(6, dtype="float32").reshape(2, 3) + ) + np.testing.assert_array_equal( + out[w1.name][0], np.arange(6, 10, dtype="float32") + ) + self.assertTrue(out[w0.name][1] and out[w1.name][1]) + self.assertEqual(fsdp.model.sharded_state_dict, fsdp.sharded_state_dict) + for p, shape in ((w0, [2, 3]), (w1, [4])): + self.assertFalse(p._is_initialized()) + self.assertEqual(p.shape, shape) + + class TestPrefetchByLayer(unittest.TestCase): """Covers the default per-unit prefetch of FSDPCommManager.""" From 41f10d584510f807832a0c595a761c398267f77d Mon Sep 17 00:00:00 2001 From: lizhenxing02 Date: Thu, 12 Nov 2026 13:56:30 +0800 Subject: [PATCH 2/2] fix --- .../flex_checkpoint/dcp/full_param.py | 2 +- .../distributed/fsdp/fully_shard_fusion.py | 92 +++++++++++++------ test/collective/fleet/dygraph_group_fsdp.py | 54 +++++++++++ .../test_fsdp_fully_shard_fusion.py | 54 +++-------- 4 files changed, 132 insertions(+), 70 deletions(-) diff --git a/python/paddle/distributed/flex_checkpoint/dcp/full_param.py b/python/paddle/distributed/flex_checkpoint/dcp/full_param.py index 032a816dbe6ecc..2dd5535eaa6f39 100644 --- a/python/paddle/distributed/flex_checkpoint/dcp/full_param.py +++ b/python/paddle/distributed/flex_checkpoint/dcp/full_param.py @@ -764,7 +764,7 @@ def _process_one_batch(self) -> list[str]: ) else: buffer = paddle.empty(read_item.slice_shape, dtype=read_item.dtype) - if buffer.place.is_cpu_place(): + if buffer.place.is_cpu_place(): # NCCL needs GPU; outputs stay on GPU buffer = buffer.cuda() if op_type == OperationType.BROADCAST_ALLGATHER: diff --git a/python/paddle/distributed/fsdp/fully_shard_fusion.py b/python/paddle/distributed/fsdp/fully_shard_fusion.py index 13ac8561192d77..54906bc20540cd 100644 --- a/python/paddle/distributed/fsdp/fully_shard_fusion.py +++ b/python/paddle/distributed/fsdp/fully_shard_fusion.py @@ -1670,38 +1670,72 @@ def sharded_state_dict(self, structured_name_prefix=""): self._shard_descs[params_buffer.data_buffer.name] = param_slice_info return result + def _model_description(self, *args, **kwargs): + """The model's own description, with experts folded as in sharded_state_dict.""" + result = self._model_sharded_state_dict(*args, **kwargs) + hcg = self.buffer_manager.hcg + if not hasattr(hcg, "get_expert_parallel_world_size"): + return result + ep_nranks = hcg.get_expert_parallel_world_size() + if ep_nranks <= 1: + return result + ep_rank = hcg.get_expert_parallel_rank() + experts = { + param.name + for group in self.buffer_manager.buffer_groups + if group.is_expert_param + for param in group.params + } + for sharded_weight in result.values(): + local_shape = tuple(sharded_weight.local_shape) + if ( + sharded_weight.local_tensor.name not in experts + or tuple(sharded_weight.global_shape) != local_shape + ): + continue + # Expert described as replicated: fold EP into axis 0 to keep keys distinct. + sharded_weight.global_shape = ( + local_shape[0] * ep_nranks, + *local_shape[1:], + ) + sharded_weight.global_offset = ( + ep_rank * local_shape[0], + *(0 for _ in local_shape[1:]), + ) + return result + def full(self, aoa_config=None, **kwargs): - """Gather released params to CPU, then run the model's own full.""" + """Gather released params to CPU and run the model's own full; iterate once to the end or close().""" released = [] - for group in self.buffer_manager.buffer_groups: - params = [p for p in group.params if not p._is_initialized()] - if not params: - continue - params_buffer = group.params_buffer - full = params_buffer.data_buffer - if params_buffer.fp8 is not None: - full = build_ckpt_weight(params_buffer, group.dtype) - if params_buffer.is_sharded: - shard = full - full = paddle.empty( - [params_buffer.total_buffer_size], dtype=shard.dtype - ) - paddle.distributed.stream.all_gather( - full, - shard, - group=params_buffer.fsdp_group, - sync_op=True, - use_calc_stream=True, - ) - for param in params: - offset = params_buffer.param_offsets[param.name] - value = full._slice(offset, offset + param._numel()) - value = value.reshape(param.shape).cpu() - released.append((param, param.shape)) - value._share_buffer_to(param) - del full - self.model.sharded_state_dict = self._model_sharded_state_dict try: + for group in self.buffer_manager.buffer_groups: + params = [p for p in group.params if not p._is_initialized()] + if not params: + continue + params_buffer = group.params_buffer + weight = params_buffer.data_buffer + if params_buffer.fp8 is not None: + weight = build_ckpt_weight(params_buffer, group.dtype) + gathered = weight + if params_buffer.is_sharded: + gathered = paddle.empty( + [params_buffer.total_buffer_size], dtype=weight.dtype + ) + paddle.distributed.stream.all_gather( + gathered, + weight, + group=params_buffer.fsdp_group, + sync_op=True, + use_calc_stream=True, + ) + for param in params: + offset = params_buffer.param_offsets[param.name] + value = gathered._slice(offset, offset + param._numel()) + value = value.reshape(param.shape).cpu() + released.append((param, param.shape)) + value._share_buffer_to(param) + del gathered, weight + self.model.sharded_state_dict = self._model_description yield from self._model_full(aoa_config, **kwargs) finally: self.model.sharded_state_dict = self.sharded_state_dict diff --git a/test/collective/fleet/dygraph_group_fsdp.py b/test/collective/fleet/dygraph_group_fsdp.py index 44d44d3061cf73..abc07d6dc1f5cf 100644 --- a/test/collective/fleet/dygraph_group_fsdp.py +++ b/test/collective/fleet/dygraph_group_fsdp.py @@ -1463,8 +1463,62 @@ def run_grad_comm_dtype(): np.testing.assert_allclose(new_losses, ref_losses, rtol=1e-2) +def run_full_export(): + """model.full() exports the trained params over the EP groups, then releases them.""" + hcg = fleet.get_hybrid_communicate_group() + ep_group = hcg.get_expert_parallel_group() + _, model = build_moe_models(hcg) + expert_names = { + p.name for layer in model.layers for p in layer.moe.experts.parameters() + } + key_to_name = {k: v.name for k, v in model.state_dict().items()} + opt = paddle.optimizer.AdamW( + learning_rate=0.001, parameters=model.parameters(), weight_decay=0.0 + ) + model = fully_shard(model, optimizer=opt) + ctx = model._fsdp_context + paddle.seed(11) + x = paddle.randn([TOKENS, HIDDEN]) + + def step(): + model(x).mean().backward() + opt.step() + opt.clear_grad() + + step() + params = list(model.parameters()) + with ctx.unshard_params(params): + expected = {p.name: p.numpy() for p in params} + released = [p for p in params if not p._is_initialized()] + assert released, "no released param to export" + + out = dict( + model.full( + h_group=ep_group, + v_group=hcg.get_pipe_parallel_group(), + num_splits=1, + shard_idx=0, + ) + ) + for key, name in key_to_name.items(): + value = expected[name] + if name in expert_names: + # Experts are described as replicated: EP folds into axis 0. + parts = [] + paddle.distributed.all_gather( + parts, paddle.to_tensor(value), group=ep_group + ) + value = np.concatenate([part.numpy() for part in parts], axis=0) + np.testing.assert_array_equal(out[key].numpy(), value, err_msg=key) + + assert all(not p._is_initialized() for p in released) + assert model.sharded_state_dict == ctx.sharded_state_dict + step() + + if __name__ == '__main__': run_moe(2) + run_full_export() run_fused_linear() run_grad_comm_dtype() run_muon_shard() diff --git a/test/legacy_test/test_fsdp_fully_shard_fusion.py b/test/legacy_test/test_fsdp_fully_shard_fusion.py index ed48c452e6a0b2..2243a43519d5fa 100644 --- a/test/legacy_test/test_fsdp_fully_shard_fusion.py +++ b/test/legacy_test/test_fsdp_fully_shard_fusion.py @@ -453,55 +453,29 @@ def _fake_grads_buffer(numel, waited, i): ) -def _release(param): - shape = param.shape - param._clear_data() - param.get_tensor()._set_dims(shape) - - class TestFullToCpu(unittest.TestCase): def test_model_full_sees_cpu_params_then_they_are_released(self): - w0 = paddle.create_parameter([2, 3], "float32") - w1 = paddle.create_parameter([4], "float32") - _release(w0) - _release(w1) + w = paddle.create_parameter([2, 3], "float32") + w._clear_data() + w.get_tensor()._set_dims([2, 3]) + fsdp = object.__new__(fsf.FullyShardFusion) params_buffer = types.SimpleNamespace( - data_buffer=paddle.arange(10, dtype="float32"), + data_buffer=paddle.arange(6, dtype="float32"), fp8=None, is_sharded=False, - param_offsets={w0.name: 0, w1.name: 6}, + param_offsets={w.name: 0}, ) - group = types.SimpleNamespace( - params=[w0, w1], params_buffer=params_buffer + group = types.SimpleNamespace(params=[w], params_buffer=params_buffer) + fsdp.buffer_manager = types.SimpleNamespace( + buffer_groups=[group], hcg=None ) - fsdp = object.__new__(fsf.FullyShardFusion) - fsdp.buffer_manager = types.SimpleNamespace(buffer_groups=[group]) fsdp.model = types.SimpleNamespace() - fsdp._model_sharded_state_dict = lambda: "model description" - seen = {} - - def model_full(aoa_config=None, **kwargs): - seen["description"] = fsdp.model.sharded_state_dict() - seen["kwargs"] = kwargs - for p in (w0, w1): - yield p.name, (p.numpy().copy(), p.place.is_cpu_place()) - - fsdp._model_full = model_full - out = dict(fsdp.full(num_splits=2, shard_idx=1)) - - self.assertEqual(seen["description"], "model description") - self.assertEqual(seen["kwargs"], {"num_splits": 2, "shard_idx": 1}) - np.testing.assert_array_equal( - out[w0.name][0], np.arange(6, dtype="float32").reshape(2, 3) - ) - np.testing.assert_array_equal( - out[w1.name][0], np.arange(6, 10, dtype="float32") - ) - self.assertTrue(out[w0.name][1] and out[w1.name][1]) + fsdp._model_sharded_state_dict = dict + fsdp._model_full = lambda aoa_config=None: iter([(w.name, w.numpy())]) + ((name, value),) = fsdp.full() + np.testing.assert_array_equal(value, np.arange(6).reshape(2, 3)) + self.assertFalse(w._is_initialized()) self.assertEqual(fsdp.model.sharded_state_dict, fsdp.sharded_state_dict) - for p, shape in ((w0, [2, 3]), (w1, [4])): - self.assertFalse(p._is_initialized()) - self.assertEqual(p.shape, shape) class TestPrefetchByLayer(unittest.TestCase):