Repository navigation
[FSDP] Supp FSDP save HF ckpt #79946
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1540,6 +1540,8 @@ def __init__( | |
| # Redirect the model's own API; the original stays as our description base. | ||
| self._model_sharded_state_dict = self.model.sharded_state_dict | ||
| self.model.sharded_state_dict = self.sharded_state_dict | ||
| self._model_full = self.model.full | ||
| self.model.full = self.full | ||
|
|
||
| def _base_sharded_state_dict(self, structured_name_prefix): | ||
| """Model descriptions of every param; fused params get a scratch view since their storage is cleared.""" | ||
|
|
@@ -1668,6 +1670,81 @@ def sharded_state_dict(self, structured_name_prefix=""): | |
| self._shard_descs[params_buffer.data_buffer.name] = param_slice_info | ||
| return result | ||
|
|
||
| def _model_description(self, *args, **kwargs): | ||
| """The model's own description, with experts folded as in sharded_state_dict.""" | ||
| result = self._model_sharded_state_dict(*args, **kwargs) | ||
| hcg = self.buffer_manager.hcg | ||
| if not hasattr(hcg, "get_expert_parallel_world_size"): | ||
| return result | ||
| ep_nranks = hcg.get_expert_parallel_world_size() | ||
| if ep_nranks <= 1: | ||
| return result | ||
| ep_rank = hcg.get_expert_parallel_rank() | ||
| experts = { | ||
| param.name | ||
| for group in self.buffer_manager.buffer_groups | ||
| if group.is_expert_param | ||
| for param in group.params | ||
| } | ||
| for sharded_weight in result.values(): | ||
| local_shape = tuple(sharded_weight.local_shape) | ||
| if ( | ||
| sharded_weight.local_tensor.name not in experts | ||
| or tuple(sharded_weight.global_shape) != local_shape | ||
| ): | ||
| continue | ||
| # Expert described as replicated: fold EP into axis 0 to keep keys distinct. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💡 可维护性建议(中层优雅度,不阻塞合入):这里的 EP 折叠公式和注释与 |
||
| sharded_weight.global_shape = ( | ||
| local_shape[0] * ep_nranks, | ||
| *local_shape[1:], | ||
| ) | ||
| sharded_weight.global_offset = ( | ||
| ep_rank * local_shape[0], | ||
| *(0 for _ in local_shape[1:]), | ||
| ) | ||
| return result | ||
|
|
||
| def full(self, aoa_config=None, **kwargs): | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💡 可维护性建议(契约一致性,不阻塞合入): |
||
| """Gather released params to CPU and run the model's own full; iterate once to the end or close().""" | ||
| released = [] | ||
| try: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
另外 建议把 |
||
| for group in self.buffer_manager.buffer_groups: | ||
| params = [p for p in group.params if not p._is_initialized()] | ||
| if not params: | ||
| continue | ||
| params_buffer = group.params_buffer | ||
| weight = params_buffer.data_buffer | ||
| if params_buffer.fp8 is not None: | ||
| weight = build_ckpt_weight(params_buffer, group.dtype) | ||
| gathered = weight | ||
| if params_buffer.is_sharded: | ||
| gathered = paddle.empty( | ||
| [params_buffer.total_buffer_size], dtype=weight.dtype | ||
| ) | ||
| paddle.distributed.stream.all_gather( | ||
| gathered, | ||
| weight, | ||
| group=params_buffer.fsdp_group, | ||
| sync_op=True, | ||
| use_calc_stream=True, | ||
| ) | ||
| for param in params: | ||
| offset = params_buffer.param_offsets[param.name] | ||
| value = gathered._slice(offset, offset + param._numel()) | ||
| value = value.reshape(param.shape).cpu() | ||
| released.append((param, param.shape)) | ||
| value._share_buffer_to(param) | ||
| del gathered, weight | ||
| self.model.sharded_state_dict = self._model_description | ||
| yield from self._model_full(aoa_config, **kwargs) | ||
| finally: | ||
| self.model.sharded_state_dict = self.sharded_state_dict | ||
| for param, shape in released: | ||
| stop_gradient = param.stop_gradient | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💡 可维护性建议(底层优雅度,不阻塞合入):第 1708-1712 行"保存 |
||
| param._clear_data() | ||
| param.stop_gradient = stop_gradient | ||
| param.get_tensor()._set_dims(shape) | ||
|
|
||
| def bind_decay_param_fun(self, optimizer, buffer_groups=None): | ||
| """Answer ``apply_decay_param_fun`` for the fused buffers. | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
💡 可维护性建议(逻辑清晰度,不阻塞合入):同文件
_run_distributed(第 525-531 行)对 CPU 源张量是cuda()通信后再.cpu()搬回;这里只做了搬上 GPU,结果会以 GPU 张量的形式缓存进sharded_desc_to_tensor。两条路径的设备语义不一致,读者无法判断这是有意为之还是遗漏。建议对齐on_cpu的写法,或者加一行注释说明 HV 路径的输出固定在 GPU 上,由memory_growth_threshold控制峰值。