NVMe-TCP on RHEL/Rocky/AlmaLinux - Quick Start Guide

sudo dnf install -y nvme-cli
nvme version

# Enable multipath BEFORE connecting (requires reboot to take effect)
echo 'options nvme_core multipath=Y' | sudo tee /etc/modprobe.d/nvme-tcp.conf
sudo reboot

Configure dedicated storage interfaces with jumbo frames and no default route:

# First storage interface
sudo nmcli connection add type ethernet \
    con-name storage-1 \
    ifname <INTERFACE_NAME_1> \
    ipv4.method manual \
    ipv4.addresses <HOST_IP_1>/<CIDR> \
    ipv4.never-default yes \
    802-3-ethernet.mtu 9000 \
    connection.autoconnect yes

# Second storage interface
sudo nmcli connection add type ethernet \
    con-name storage-2 \
    ifname <INTERFACE_NAME_2> \
    ipv4.method manual \
    ipv4.addresses <HOST_IP_2>/<CIDR> \
    ipv4.never-default yes \
    802-3-ethernet.mtu 9000 \
    connection.autoconnect yes

# Activate
sudo nmcli connection up storage-1
sudo nmcli connection up storage-2

Add storage interfaces to trusted zone (recommended for dedicated storage networks):

# Check if firewalld is running (skip if not installed, e.g., minimal/cloud images)
if systemctl is-active --quiet firewalld; then
    sudo firewall-cmd --permanent --zone=trusted --add-interface=<INTERFACE_NAME_1>
    sudo firewall-cmd --permanent --zone=trusted --add-interface=<INTERFACE_NAME_2>
    sudo firewall-cmd --reload
else
    echo "firewalld not running - skipping firewall configuration"
fi

For port filtering options, see Firewall Configuration Best Practices.

sudo mkdir -p /etc/nvme
sudo nvme gen-hostnqn | sudo tee /etc/nvme/hostnqn
cat /etc/nvme/hostnqn

Register this NQN with your FlashArray's allowed hosts list.

Option A: Connect-all (simplest method)

Discovers and connects to all available subsystems from a single portal:

# Replace <PORTAL_IP_1> with any storage portal IP
sudo nvme connect-all -t tcp -a <PORTAL_IP_1> -s 4420

# Verify connections
sudo nvme list-subsys

Option B: Individual connections (for specific interface binding)

Use when you need to bind connections to specific host interfaces:

# Replace values: <PORTAL_IP_X>, <SUBSYSTEM_NQN>, <INTERFACE_NAME_X>, <HOST_IP_X>
sudo nvme connect -t tcp -a <PORTAL_IP_1> -s 4420 -n <SUBSYSTEM_NQN> \
    --host-iface=<INTERFACE_NAME_1> --host-traddr=<HOST_IP_1> \
    --ctrl-loss-tmo=1800 --reconnect-delay=10

sudo nvme connect -t tcp -a <PORTAL_IP_2> -s 4420 -n <SUBSYSTEM_NQN> \
    --host-iface=<INTERFACE_NAME_1> --host-traddr=<HOST_IP_1> \
    --ctrl-loss-tmo=1800 --reconnect-delay=10

# Repeat for all portal/interface combinations to create full mesh

# Verify connections
sudo nvme list-subsys

# Create udev rule for queue-depth IO policy
sudo tee /etc/udev/rules.d/99-nvme-iopolicy.rules > /dev/null <<'EOF'
ACTION=="add|change", SUBSYSTEM=="nvme-subsystem", ATTR{iopolicy}="queue-depth"
EOF

sudo udevadm control --reload-rules
sudo udevadm trigger

# Create discovery configuration
sudo tee /etc/nvme/discovery.conf > /dev/null <<EOF
-t tcp -a <PORTAL_IP_1> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>
-t tcp -a <PORTAL_IP_2> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>
-t tcp -a <PORTAL_IP_3> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>
-t tcp -a <PORTAL_IP_4> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>

-t tcp -a <PORTAL_IP_1> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
-t tcp -a <PORTAL_IP_2> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
-t tcp -a <PORTAL_IP_3> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
-t tcp -a <PORTAL_IP_4> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
EOF

# Enable autoconnect service
sudo systemctl enable --now nvmf-autoconnect.service

# Find NVMe device
sudo nvme list
# Example: /dev/nvme0n1

# Create LVM
sudo pvcreate /dev/nvme0n1
sudo vgcreate nvme-storage /dev/nvme0n1
sudo lvcreate -L 500G -n data nvme-storage

# Format and mount (RHEL: XFS recommended)
sudo mkfs.xfs /dev/nvme-storage/data
sudo mkdir -p /mnt/nvme-storage
sudo mount /dev/nvme-storage/data /mnt/nvme-storage

# Add to fstab
echo '/dev/nvme-storage/data /mnt/nvme-storage xfs defaults,_netdev 0 0' | sudo tee -a /etc/fstab

# Check multipath is enabled
cat /sys/module/nvme_core/parameters/multipath  # Should show: Y

# Check all paths are live
sudo nvme list-subsys

# Check IO policy
cat /sys/class/nvme-subsystem/nvme-subsys*/iopolicy  # Should show: queue-depth

# Verify block devices are visible
lsblk | grep nvme

# Verify storage (after LVM/filesystem creation)
df -h | grep nvme

Verify Persistent Connections (recommended):

# Reboot to confirm connections re-establish automatically
sudo reboot

# After reboot, verify all paths reconnected
sudo nvme list-subsys  # All paths should show "live"