NVMe-TCP on Debian/Ubuntu - Performance Tuning

Sysctl Tuning

Create sysctl configuration for NVMe-TCP:

sudo tee /etc/sysctl.d/90-nvme-tcp-storage.conf > /dev/null <<'EOF'
# Network buffer sizes
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
net.core.rmem_default = 16777216
net.core.wmem_default = 16777216
net.ipv4.tcp_rmem = 4096 87380 67108864
net.ipv4.tcp_wmem = 4096 65536 67108864

# Network performance
net.core.netdev_max_backlog = 30000
net.core.somaxconn = 4096
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_sack = 1

# VM tuning
vm.dirty_ratio = 10
vm.dirty_background_ratio = 5
vm.swappiness = 10

# ARP cache
net.ipv4.neigh.default.gc_thresh1 = 4096
net.ipv4.neigh.default.gc_thresh2 = 8192
net.ipv4.neigh.default.gc_thresh3 = 16384

# ARP settings for same-subnet multipath (CRITICAL)
# Prevents ARP responses on wrong interface when multiple NICs share same subnet
# See: Network Concepts documentation for detailed explanation
net.ipv4.conf.all.arp_ignore = 2
net.ipv4.conf.default.arp_ignore = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.default.arp_announce = 2
# Interface-specific (adjust interface names as needed)
net.ipv4.conf.ens1f0.arp_ignore = 2
net.ipv4.conf.ens1f1.arp_ignore = 2
net.ipv4.conf.ens1f0.arp_announce = 2
net.ipv4.conf.ens1f1.arp_announce = 2
EOF

# Apply settings
sudo sysctl -p /etc/sysctl.d/90-nvme-tcp-storage.conf

NIC Tuning

Create systemd service for NIC optimization:

sudo tee /etc/systemd/system/tune-storage-nics.service > /dev/null <<'EOF'
[Unit]
Description=Tune storage network interfaces
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/bin/tune-storage-nics.sh

[Install]
WantedBy=multi-user.target
EOF

# Create tuning script
sudo tee /usr/local/bin/tune-storage-nics.sh > /dev/null <<'EOF'
#!/bin/bash

# Storage interfaces to tune
INTERFACES="ens1f0 ens1f1"

for iface in $INTERFACES; do
    if [ -d "/sys/class/net/$iface" ]; then
        echo "Tuning $iface..."

        # Ring buffers
        ethtool -G $iface rx 4096 tx 4096 2>/dev/null || true

        # Interrupt coalescing
        ethtool -C $iface rx-usecs 50 tx-usecs 50 2>/dev/null || true

        # Offloads
        ethtool -K $iface tso on gso on gro on 2>/dev/null || true
        ethtool -K $iface lro off 2>/dev/null || true

        # Flow control
        ethtool -A $iface rx on tx on 2>/dev/null || true

        echo "$iface tuned successfully"
    fi
done
EOF

sudo chmod +x /usr/local/bin/tune-storage-nics.sh

# Enable and start
sudo systemctl enable --now tune-storage-nics.service

IRQ Affinity

Install irqbalance:

sudo apt install -y irqbalance

# Configure
sudo tee /etc/default/irqbalance > /dev/null <<EOF
# Ban CPU 0 from IRQ handling
IRQBALANCE_BANNED_CPUS=00000001
EOF

# Restart
sudo systemctl restart irqbalance

Manual IRQ affinity (for specific control):

# Create IRQ affinity script
sudo tee /usr/local/bin/set-storage-irq-affinity.sh > /dev/null <<'EOF'
#!/bin/bash

INTERFACE="ens1f0"
CPU_START=2

for IRQ in $(grep $INTERFACE /proc/interrupts | awk '{print $1}' | sed 's/://'); do
    MASK=$(printf "%x" $((1 << $CPU_START)))
    echo $MASK > /proc/irq/$IRQ/smp_affinity
    echo "IRQ $IRQ -> CPU $CPU_START (mask: $MASK)"
    CPU_START=$((CPU_START + 1))
done
EOF

sudo chmod +x /usr/local/bin/set-storage-irq-affinity.sh

# Run at boot
echo "@reboot root /usr/local/bin/set-storage-irq-affinity.sh" | sudo tee -a /etc/crontab

CPU Governor

Set performance governor:

# Install cpufrequtils
sudo apt install -y cpufrequtils

# Set to performance
echo 'GOVERNOR="performance"' | sudo tee /etc/default/cpufrequtils

# Apply
sudo systemctl restart cpufrequtils

# Verify
cpufreq-info

I/O Scheduler

Set I/O scheduler for NVMe devices:

# Check current scheduler
cat /sys/block/nvme0n1/queue/scheduler

# Set to 'none' for NVMe (best performance)
echo none | sudo tee /sys/block/nvme*/queue/scheduler

# Make persistent
sudo tee /etc/udev/rules.d/60-nvme-scheduler.rules > /dev/null <<'EOF'
# Set I/O scheduler to 'none' for NVMe devices
ACTION=="add|change", KERNEL=="nvme[0-9]n[0-9]", ATTR{queue/scheduler}="none"
EOF

# Reload udev rules
sudo udevadm control --reload-rules
sudo udevadm trigger