Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions simplyblock_core/utils/__init__.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -1459,6 +1459,7 @@ def build_unisolated_stride(
num_unisolated: int,
client_qpair_count: int,
pool_stride: int = 2,
nodes_per_socket: int = 1,
) -> List[int]:
"""
Build a list of 'unisolated' CPUs by picking from per-qpair pools.
Expand All @@ -1473,6 +1474,10 @@ def build_unisolated_stride(
If hyper_thread=True, append sibling right after each selected core,
where sibling is defined by *index pairing* across halves of the sorted list:
sibling(cores[i]) = cores[i + half] if i < half else cores[i - half]

When hyper_thread=True and nodes_per_socket >= 2, the remaining SPDK cores
(all_cores - unisolated) are trimmed to a multiple of 4 so that when split
across two nodes each node gets an even (paired) count.
"""
hyper_thread = is_hyperthreading_enabled_via_siblings()

Expand All @@ -1496,9 +1501,27 @@ def build_unisolated_stride(
raise ValueError(f"hyper_thread=True but total logical CPUs ({total}) is not even")
half = total // 2

# If you REQUIRE strict pairing (cpu+sibling always together), uncomment:
# if hyper_thread and (num_unisolated % 2 != 0):
# raise ValueError("num_unisolated must be even when hyper_thread=True")
# Cores are always selected in complete HT pairs (cpu + sibling), so
# num_unisolated must be even — round down if needed.
if num_unisolated % 2 != 0:
num_unisolated -= 1

if nodes_per_socket >= 2:
# When splitting across 2 nodes each SPDK core must pair with its HT
# sibling, so the SPDK pool (total - num_unisolated) must be a multiple
# of 4. Both values are already even, so the only possible misalignment
# is a remainder of 2 — corrected by adjusting num_unisolated by one
# HT pair (2 cores).
spdk_count = total - num_unisolated
if spdk_count % 4 != 0:
# spdk_count % 4 == 2: give one extra HT pair to SPDK
if num_unisolated >= 2:
num_unisolated -= 2
else:
# No room to shrink unisolated; take one pair from SPDK instead
num_unisolated += 2

num_unisolated = max(0, min(num_unisolated, total))

core_to_idx = {c: i for i, c in enumerate(cores)}

Expand Down Expand Up @@ -1568,7 +1591,7 @@ def add_cpu(cpu: int) -> bool:

break

return out[:num_unisolated]
return out[:num_unisolated], num_unisolated

def generate_core_allocation(cores_by_numa, sockets_to_use, nodes_per_socket, cores_percentage=0):
node_distribution: dict = {}
Expand All @@ -1578,7 +1601,9 @@ def generate_core_allocation(cores_by_numa, sockets_to_use, nodes_per_socket, co
continue
all_cores = sorted(cores_by_numa[numa_node])
num_unisolated = calculate_unisolated_cores(all_cores, cores_percentage)
unisolated = build_unisolated_stride(all_cores, num_unisolated, constants.CLIENT_QPAIR_COUNT)
unisolated, num_unisolated = build_unisolated_stride(
all_cores, num_unisolated, constants.CLIENT_QPAIR_COUNT, nodes_per_socket=nodes_per_socket
)

available_cores = [c for c in all_cores if c not in unisolated]
q1 = len(available_cores) // 4
Expand Down
1 change: 1 addition & 0 deletions simplyblock_web/api/internal/storage_node/kubernetes.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,7 @@ def spdk_process_start(body: SPDKParams):
'NSOCKET': body.socket,
'FW_PORT': body.firewall_port,
'CPU_TOPOLOGY_ENABLED': cpu_topology_enabled,
'HT_ENABLED': core_utils.is_hyperthreading_enabled_via_siblings(),
'RESERVED_SYSTEM_CPUS': reserved_system_cpus
}

Expand Down
23 changes: 17 additions & 6 deletions simplyblock_web/templates/storage_deploy_spdk.yaml.j2
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,16 @@ spec:
mountPath: /etc/foundationdb
resources:
requests:
cpu: "100m"
{% if CPU_TOPOLOGY_ENABLED %}
cpu: {{ 2 if HT_ENABLED else 1 }}
numa-align/numa-{{ NSOCKET }}: 1
{% endif %}
memory: "64Mi"
limits:
cpu: "100m"
{% if CPU_TOPOLOGY_ENABLED %}
cpu: {{ 2 if HT_ENABLED else 1 }}
numa-align/numa-{{ NSOCKET }}: 1
{% endif %}
memory: "64Mi"

containers:
Expand Down Expand Up @@ -138,6 +144,7 @@ spec:
{% endif %}
requests:
hugepages-2Mi: {{ MEM_MEGA }}Mi
cpu: {{ CORES }}
{% if CPU_TOPOLOGY_ENABLED %}
memory: {{ MEM2_MEGA }}Mi
numa-align/numa-{{ NSOCKET }}: 1
Expand All @@ -163,15 +170,19 @@ spec:
value: "True"
- name: TIMEOUT
value: "300"
{% if CPU_TOPOLOGY_ENABLED %}
resources:
limits:
cpu: 1
{% if CPU_TOPOLOGY_ENABLED %}
cpu: {{ 2 if HT_ENABLED else 1 }}
numa-align/numa-{{ NSOCKET }}: 1
{% endif %}
memory: "128Mi"
requests:
cpu: 1
{% if CPU_TOPOLOGY_ENABLED %}
cpu: {{ 2 if HT_ENABLED else 1 }}
numa-align/numa-{{ NSOCKET }}: 1
{% endif %}
memory: "128Mi"
{% endif %}

{% if MODE == "docker" %}
---
Expand Down
Loading