Skip to content

Remove no exp usage from logical rule Part II #3603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NuojCheng wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Remove no exp usage from logical rule Part II #3603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions src/maxtext/common/common_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,10 @@
BATCH = "activation_batch"

ATTN_LENGTH = "activation_attn_length"
ATTN_LENGTH_NO_EXP = "activation_attn_length_no_exp"

LENGTH = "activation_length"
PREFILL_LENGTH = "prefill_activation_length"
Q_LENGTH = "activation_q_length"
Q_LENGTH_NO_EXP = "activation_q_length_no_exp"
Q_LORA_UP_PROJ = "q_lora_up_proj"
KV_LENGTH = "activation_kv_length"
KV_LORA_UP_PROJ = "kv_lora_up_proj"
Expand All @@ -48,7 +46,6 @@
HEAD = "activation_heads"
PREFILL_KV_BATCH = "activation_prefill_kv_batch"
KV_BATCH = "activation_kv_batch"
KV_BATCH_NO_EXP = "activation_kv_batch_no_exp"
KV_HEAD = "activation_kv_heads"
KV_HEAD_DIM = "activation_kv_head_dim"
D_KV = "activation_kv"
Expand Down
6 changes: 2 additions & 4 deletions src/maxtext/configs/base.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,8 +468,7 @@ logical_axis_rules: [
['activation_length_moe', ['context']],
['activation_norm_length', ['tensor_sequence', 'context', 'sequence']],
['activation_norm_length_moe', ['tensor_sequence', 'context', 'sequence']],
['activation_q_length', ['context', 'expert']],
['activation_q_length_no_exp', ['context']],
['activation_q_length', ['context']],
['prefill_activation_length', ['sequence', 'context']],
['prefill_activation_norm_length', ['tensor_sequence', 'context', 'sequence']],
['activation_kv_length', []],
Expand All @@ -479,8 +478,7 @@ logical_axis_rules: [
['activation_mlp', ['tensor', 'tensor_transpose', 'tensor_sequence']],
['activation_kv', ['tensor', 'tensor_transpose', 'tensor_sequence']],
['activation_prefill_kv_batch', ['data', 'fsdp', 'fsdp_transpose', 'expert']],
['activation_kv_batch', ['data', 'fsdp', 'fsdp_transpose', 'expert']],
['activation_kv_batch_no_exp', ['data', 'fsdp', 'fsdp_transpose']],
['activation_kv_batch', ['data', 'fsdp', 'fsdp_transpose']],
['activation_kv_head_dim', ['tensor', 'tensor_transpose', 'tensor_sequence']],
['activation_vocab', ['tensor', 'tensor_transpose', 'tensor_sequence']],
['activation_vocab', ['tensor', 'tensor_transpose']],
Expand Down
9 changes: 4 additions & 5 deletions src/maxtext/configs/custom_mesh_and_rule/pipeline-large-moe.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,17 @@ logical_axis_rules: [
['activation_embed_and_logits_batch_sequence', ['data', 'stage', 'fsdp', 'context', 'expert']],
['activation_heads', ['tensor']],
['activation_kv_heads', ['tensor']],
['activation_length', ['context', 'expert']],
['activation_attn_length', ['context', 'expert']],
['activation_q_length', ['context', 'expert']],
['activation_length', ['context']],
['activation_attn_length', ['context']],
['activation_q_length', ['context']],
['activation_attn_embed', ['tensor']],
['activation_norm_length', ['context']],
['activation_norm_length_moe', ['context']],
['activation_embed', ['tensor']],
['activation_embed_moe', ['tensor']],
['activation_mlp', ['tensor']],
['activation_kv', ['tensor']],
['activation_kv_batch', ['data', 'fsdp', 'expert']],
['activation_kv_batch_no_exp', ['data', 'fsdp']],
['activation_kv_batch', ['data', 'fsdp']],
['activation_kv_head_dim', ['tensor']],
['activation_vocab', ['tensor']],
['activation_stage', 'stage'],
Expand Down
1 change: 0 additions & 1 deletion src/maxtext/configs/custom_mesh_and_rule/pure-fsdp.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ logical_axis_rules: [
['activation_embed_and_logits_batch_sequence', ['fsdp']],
['activation_prefill_kv_batch', ['fsdp']],
['activation_kv_batch', ['fsdp']],
['activation_kv_batch_no_exp', ['fsdp']],
['decode_batch', ['fsdp']],
['embed', ['fsdp']],
['embed_no_exp', ['fsdp']],
Expand Down
6 changes: 2 additions & 4 deletions src/maxtext/configs/inference/vllm.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ logical_axis_rules: [
['activation_embed_and_logits_batch_sequence', ['data', 'expert']],
['activation_heads', ['model', 'expert']],
['activation_kv_heads', ['model', 'expert']],
['activation_attn_length', ['expert']],
['activation_attn_length_no_exp', []],
['activation_attn_length', []],
['activation_length', ['data']],
['activation_length_moe', ['data', 'expert']],
['activation_length_moe', 'data'],
Expand All @@ -48,8 +47,7 @@ logical_axis_rules: [
['activation_mlp', ['model', 'attn_dp']],
['activation_kv', ['model']],
['activation_prefill_kv_batch', ['expert', 'attn_dp_expert']],
['activation_kv_batch', ['data', 'expert', 'attn_dp_expert']],
['activation_kv_batch_no_exp', ['data']],
['activation_kv_batch', ['data']],
['activation_kv_head_dim', ['model']],
['activation_vocab', ['model', 'attn_dp']],
['activation_norm_length', []],
Expand Down
10 changes: 5 additions & 5 deletions src/maxtext/layers/attention_op.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
MODEL_MODE_PREFILL,
MODEL_MODE_TRAIN,
PREFILL_LENGTH,
Q_LENGTH_NO_EXP,
Q_LENGTH,
)
from maxtext.inference import page_manager
from maxtext.inference.kvcache import KVQuant, KVTensor
Expand Down Expand Up @@ -1134,13 +1134,13 @@ def tpu_flash_attention(
segment_axis_names_kv = None
sink_axis_names = self._logical_to_mesh_axes((HEAD,))
if decoder_segment_ids is not None:
segment_axis_names_q = self._logical_to_mesh_axes((BATCH, Q_LENGTH_NO_EXP))
segment_axis_names_q = self._logical_to_mesh_axes((BATCH, Q_LENGTH))
segment_axis_names_kv = self._logical_to_mesh_axes((BATCH, KV_LENGTH))

axis_names_splash_kernel = self._logical_to_mesh_axes(self.flash_axis_names_splash_kernel)
axis_names_q = self._logical_to_mesh_axes(self.flash_axis_names_q)
axis_names_kv = self._logical_to_mesh_axes(self.flash_axis_names_kv)
indexer_mask_axis_names = self._logical_to_mesh_axes((BATCH, Q_LENGTH_NO_EXP, KV_LENGTH))
indexer_mask_axis_names = self._logical_to_mesh_axes((BATCH, Q_LENGTH, KV_LENGTH))

global global_block_q, global_block_kv, global_block_kv_compute, global_block_q_dkv, global_block_kv_dkv
global global_block_kv_dkv_compute, global_block_q_dq, global_block_kv_dq, global_use_fused_bwd_kernel
Expand Down Expand Up @@ -1269,11 +1269,11 @@ def wrap_splash_kernel(single_head_mask):
return splash_kernel

splash_kernel = wrap_splash_kernel(single_head_mask)
segment_axis_names_splash_kernel = self._logical_to_mesh_axes((Q_LENGTH_NO_EXP,))
segment_axis_names_splash_kernel = self._logical_to_mesh_axes((Q_LENGTH,))
elif self.config.use_jax_splash and self.config.expert_shard_attention_option == EP_AS_FSDP:
if self.config.use_max_logit_estimate > 0:
sa_config = dataclasses.replace(sa_config, max_logit_const=self.config.use_max_logit_estimate)
segment_axis_names_splash_kernel = nn.logical_to_mesh_axes((Q_LENGTH_NO_EXP,))
segment_axis_names_splash_kernel = nn.logical_to_mesh_axes((Q_LENGTH,))
else:
# Create multi-head mask
multi_head_mask = splash_attention_mask.MultiHeadMask(masks=(mask,) * query.shape[1])
Expand Down
39 changes: 10 additions & 29 deletions src/maxtext/layers/attentions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
AxisNames,
AxisIdxes,
ATTN_LENGTH,
ATTN_LENGTH_NO_EXP,
DType,
Config,
Array,
Expand All @@ -44,12 +43,10 @@
KV_HEAD,
KV_HEAD_DIM,
KV_BATCH,
KV_BATCH_NO_EXP,
ATTN_EMBED,
MODEL_MODE_AUTOREGRESSIVE,
MODEL_MODE_TRAIN,
MODEL_MODE_PREFILL,
EP_AS_CONTEXT,
AttentionType,
)
from maxtext.layers import nnx_wrappers
Expand Down Expand Up @@ -141,14 +138,11 @@ def attention_as_linen(
prefill_query_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, KV_HEAD, KV_HEAD_DIM),
prefill_key_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, KV_HEAD, KV_HEAD_DIM),
prefill_value_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, KV_HEAD, KV_HEAD_DIM),
query_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH_NO_EXP, KV_HEAD, KV_HEAD_DIM),
key_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH_NO_EXP, KV_HEAD, KV_HEAD_DIM),
value_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH_NO_EXP, KV_HEAD, KV_HEAD_DIM),
ep_query_axis_names: AxisNames = (KV_BATCH_NO_EXP, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
ep_key_axis_names: AxisNames = (KV_BATCH_NO_EXP, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
ep_value_axis_names: AxisNames = (KV_BATCH_NO_EXP, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
input_axis_names: AxisNames = (BATCH, ATTN_LENGTH_NO_EXP, ATTN_EMBED),
out_axis_names: AxisNames = (BATCH, ATTN_LENGTH_NO_EXP, HEAD, D_KV),
query_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
key_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
value_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
input_axis_names: AxisNames = (BATCH, ATTN_LENGTH, ATTN_EMBED),
out_axis_names: AxisNames = (BATCH, ATTN_LENGTH, HEAD, D_KV),
prefill_input_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, ATTN_EMBED),
decode_input_axis_names: AxisNames = (DECODE_BATCH, DECODE_LENGTH, ATTN_EMBED),
prefill_out_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, HEAD, D_KV),
Expand Down Expand Up @@ -208,9 +202,6 @@ def attention_as_linen(
query_axis_names=query_axis_names,
key_axis_names=key_axis_names,
value_axis_names=value_axis_names,
ep_query_axis_names=ep_query_axis_names,
ep_key_axis_names=ep_key_axis_names,
ep_value_axis_names=ep_value_axis_names,
input_axis_names=input_axis_names,
out_axis_names=out_axis_names,
prefill_input_axis_names=prefill_input_axis_names,
Expand Down Expand Up @@ -304,14 +295,11 @@ def __init__(
prefill_query_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, KV_HEAD, KV_HEAD_DIM),
prefill_key_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, KV_HEAD, KV_HEAD_DIM),
prefill_value_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, KV_HEAD, KV_HEAD_DIM),
query_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH_NO_EXP, KV_HEAD, KV_HEAD_DIM),
key_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH_NO_EXP, KV_HEAD, KV_HEAD_DIM),
value_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH_NO_EXP, KV_HEAD, KV_HEAD_DIM),
ep_query_axis_names: AxisNames = (KV_BATCH_NO_EXP, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
ep_key_axis_names: AxisNames = (KV_BATCH_NO_EXP, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
ep_value_axis_names: AxisNames = (KV_BATCH_NO_EXP, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
input_axis_names: AxisNames = (BATCH, ATTN_LENGTH_NO_EXP, ATTN_EMBED),
out_axis_names: AxisNames = (BATCH, ATTN_LENGTH_NO_EXP, HEAD, D_KV),
query_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
key_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
value_axis_names: AxisNames = (KV_BATCH, ATTN_LENGTH, KV_HEAD, KV_HEAD_DIM),
input_axis_names: AxisNames = (BATCH, ATTN_LENGTH, ATTN_EMBED),
out_axis_names: AxisNames = (BATCH, ATTN_LENGTH, HEAD, D_KV),
prefill_input_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, ATTN_EMBED),
decode_input_axis_names: AxisNames = (DECODE_BATCH, DECODE_LENGTH, ATTN_EMBED),
prefill_out_axis_names: AxisNames = (PREFILL_KV_BATCH, PREFILL_LENGTH, HEAD, D_KV),
Expand Down Expand Up @@ -413,9 +401,6 @@ def __init__(
self.query_axis_names = query_axis_names
self.key_axis_names = key_axis_names
self.value_axis_names = value_axis_names
self.ep_query_axis_names = ep_query_axis_names
self.ep_key_axis_names = ep_key_axis_names
self.ep_value_axis_names = ep_value_axis_names
self.input_axis_names = input_axis_names
self.out_axis_names = out_axis_names
self.prefill_input_axis_names = prefill_input_axis_names
Expand Down Expand Up @@ -1161,10 +1146,6 @@ def __call__(
query = self._maybe_shard_with_logical(query, (DECODE_BATCH, DECODE_LENGTH, HEAD, D_KV))
key = self._maybe_shard_with_logical(key, (DECODE_BATCH, DECODE_LENGTH, KV_HEAD, D_KV))
value = self._maybe_shard_with_logical(value, (DECODE_BATCH, DECODE_LENGTH, KV_HEAD, D_KV))
elif model_mode == MODEL_MODE_TRAIN and self.config.expert_shard_attention_option == EP_AS_CONTEXT:
query = self._maybe_shard_with_logical(query, self.ep_query_axis_names)
key = self._maybe_shard_with_logical(key, self.ep_key_axis_names)
value = self._maybe_shard_with_logical(value, self.ep_value_axis_names)
else:
query = self._maybe_shard_with_logical(query, self.query_axis_names)
key = self._maybe_shard_with_logical(key, self.key_axis_names)
Expand Down
12 changes: 6 additions & 6 deletions tests/utils/sharding_info/gpt-oss-20b/tpu7x-16/slice_1/input_shardings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,31 @@
"Activation Sharding Dump": [
{
"attentions/inputs_q: bfloat16[192,2048,2880]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_attn_embed')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_attn_embed')",
"PartitionSpec": "P('fsdp', None, None)"
}
},
{
"attentions/inputs_kv: bfloat16[192,2048,2880]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_attn_embed')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_attn_embed')",
"PartitionSpec": "P('fsdp', None, None)"
}
},
{
"attentions/query: bfloat16[192,2048,64,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
{
"attentions/key: bfloat16[192,2048,8,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
{
"attentions/value: bfloat16[192,2048,8,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
Expand All @@ -50,7 +50,7 @@
},
{
"attentions/out: bfloat16[192,2048,64,64]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_heads', 'activation_kv')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_heads', 'activation_kv')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
Expand Down
12 changes: 6 additions & 6 deletions tests/utils/sharding_info/gpt-oss-20b/tpu7x-16/slice_4/input_shardings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,31 @@
"Activation Sharding Dump": [
{
"attentions/inputs_q: bfloat16[768,2048,2880]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_attn_embed')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_attn_embed')",
"PartitionSpec": "P(('data', 'fsdp'), None, None)"
}
},
{
"attentions/inputs_kv: bfloat16[768,2048,2880]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_attn_embed')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_attn_embed')",
"PartitionSpec": "P(('data', 'fsdp'), None, None)"
}
},
{
"attentions/query: bfloat16[768,2048,64,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P(('data', 'fsdp'), None, None, None)"
}
},
{
"attentions/key: bfloat16[768,2048,8,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P(('data', 'fsdp'), None, None, None)"
}
},
{
"attentions/value: bfloat16[768,2048,8,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P(('data', 'fsdp'), None, None, None)"
}
},
Expand All @@ -50,7 +50,7 @@
},
{
"attentions/out: bfloat16[768,2048,64,64]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_heads', 'activation_kv')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_heads', 'activation_kv')",
"PartitionSpec": "P(('data', 'fsdp'), None, None, None)"
}
},
Expand Down
12 changes: 6 additions & 6 deletions tests/utils/sharding_info/gpt-oss-20b/v5p-16/slice_1/input_shardings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,31 @@
"Activation Sharding Dump": [
{
"attentions/inputs_q: bfloat16[96,2048,2880]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_attn_embed')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_attn_embed')",
"PartitionSpec": "P('fsdp', None, None)"
}
},
{
"attentions/inputs_kv: bfloat16[96,2048,2880]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_attn_embed')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_attn_embed')",
"PartitionSpec": "P('fsdp', None, None)"
}
},
{
"attentions/query: bfloat16[96,2048,64,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
{
"attentions/key: bfloat16[96,2048,8,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
{
"attentions/value: bfloat16[96,2048,8,64]": {
"logic_axes": "('activation_kv_batch', 'activation_attn_length_no_exp', 'activation_kv_heads', 'activation_kv_head_dim')",
"logic_axes": "('activation_kv_batch', 'activation_attn_length', 'activation_kv_heads', 'activation_kv_head_dim')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
Expand All @@ -50,7 +50,7 @@
},
{
"attentions/out: bfloat16[96,2048,64,64]": {
"logic_axes": "('activation_batch', 'activation_attn_length_no_exp', 'activation_heads', 'activation_kv')",
"logic_axes": "('activation_batch', 'activation_attn_length', 'activation_heads', 'activation_kv')",
"PartitionSpec": "P('fsdp', None, None, None)"
}
},
Expand Down
Loading
Loading