Repository navigation
[FSDP] Sum cp-partial grads over cp in the sharding reduce #79948
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
e059190
ebee217
90896eb
f42bc76
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -98,6 +98,7 @@ class BufferGroup: | |
| grads_use_sum: int = 0 | ||
| grads_use_cnt: int = 0 | ||
| grads_ready_cnt: int = 0 | ||
| cp_replicated: bool = False | ||
|
|
||
|
|
||
| def _whole_3d_shard_numel(params, fsdp_group): | ||
|
|
@@ -447,6 +448,9 @@ def __init__( | |
| if use_ep | ||
| else self._fsdp_group | ||
| ) | ||
| cp_degree = getattr( | ||
| self.hcg, "get_context_parallel_world_size", lambda: 1 | ||
| )() | ||
| self.main_grad_dtype = ( | ||
| main_grad_dtype if main_grad_dtype is not None else paddle.float32 | ||
| ) | ||
|
|
@@ -514,6 +518,15 @@ def __init__( | |
| ) | ||
| ), | ||
| getattr(param, "_fsdp_fp8_quantizer", None), | ||
| # Grad is replicated (not partial) across cp ranks. | ||
| cp_degree > 1 | ||
| and bool( | ||
| getattr(param, "expert_cp_replicated", False) | ||
| if is_expert | ||
| else getattr( | ||
| param, "context_parallel_disable_scale_grad", False | ||
| ) | ||
| ), | ||
| ) | ||
| keyed_params.setdefault(key, []).append(param) | ||
|
|
||
|
|
@@ -531,6 +544,7 @@ def sort_key(item): | |
| _, | ||
| _, | ||
| _, | ||
| _, | ||
| ) = item[0] | ||
| return ( | ||
| 0 if is_tie else (1 if not trainable else 2), | ||
|
|
@@ -552,6 +566,7 @@ def sort_key(item): | |
| keep_whole=keep_whole, | ||
| hyperball_sharded=hyperball_name is not None, | ||
| clip_attrs=clip_attrs, | ||
| cp_replicated=cp_replicated, | ||
| ) | ||
| for ( | ||
| dtype, | ||
|
|
@@ -566,6 +581,7 @@ def sort_key(item): | |
| hyperball_name, | ||
| clip_attrs, | ||
| _, | ||
| cp_replicated, | ||
| ), params in sorted(keyed_params.items(), key=sort_key) | ||
| ] | ||
|
|
||
|
|
@@ -621,7 +637,8 @@ def sort_key(item): | |
| main_grad_dtype=paddle.float32 | ||
| if group.is_expert_param or group.dtype == paddle.float32 | ||
| else self.main_grad_dtype, | ||
| grad_div=self._fsdp_group.nranks, | ||
| grad_div=self._fsdp_group.nranks | ||
| // (1 if group.cp_replicated else cp_degree), | ||
|
Comment on lines
+640
to
+641
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 已在 90896eb 补充 |
||
| shard_buffer=grads_shard_buffer, | ||
| shard_align_numel=shard_align, | ||
| grad_comm_dtype=self.grad_comm_dtype, | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.