NVMe-TCP on SUSE/openSUSE - Quick Start Guide

sudo zypper install -y nvme-cli
nvme version

# Enable multipath BEFORE connecting (requires reboot to take effect)
echo 'options nvme_core multipath=Y' | sudo tee /etc/modprobe.d/nvme-tcp.conf
sudo reboot

# First storage interface
sudo tee /etc/sysconfig/network/ifcfg-<INTERFACE_NAME_1> > /dev/null <<EOF
BOOTPROTO='static'
STARTMODE='auto'
IPADDR='<HOST_IP_1>/<CIDR>'
MTU='9000'
NAME='Storage Network 1'
EOF

# Second storage interface
sudo tee /etc/sysconfig/network/ifcfg-<INTERFACE_NAME_2> > /dev/null <<EOF
BOOTPROTO='static'
STARTMODE='auto'
IPADDR='<HOST_IP_2>/<CIDR>'
MTU='9000'
NAME='Storage Network 2'
EOF

# Apply configuration
sudo wicked ifreload all
ip addr show

# Check if firewalld is running (skip if not installed, e.g., minimal/cloud images)
if systemctl is-active --quiet firewalld; then
    sudo firewall-cmd --permanent --zone=trusted --add-interface=<INTERFACE_NAME_1>
    sudo firewall-cmd --permanent --zone=trusted --add-interface=<INTERFACE_NAME_2>
    sudo firewall-cmd --reload
else
    echo "firewalld not running - skipping firewall configuration"
fi

# Replace <PORTAL_IP_1> with any storage portal IP
sudo nvme connect-all -t tcp -a <PORTAL_IP_1> -s 4420

# Verify connections
sudo nvme list-subsys

# Replace values: <PORTAL_IP_X>, <SUBSYSTEM_NQN>, <INTERFACE_NAME_X>, <HOST_IP_X>
sudo nvme connect -t tcp -a <PORTAL_IP_1> -s 4420 -n <SUBSYSTEM_NQN> \
    --host-iface=<INTERFACE_NAME_1> --host-traddr=<HOST_IP_1> \
    --ctrl-loss-tmo=1800 --reconnect-delay=10

sudo nvme connect -t tcp -a <PORTAL_IP_2> -s 4420 -n <SUBSYSTEM_NQN> \
    --host-iface=<INTERFACE_NAME_1> --host-traddr=<HOST_IP_1> \
    --ctrl-loss-tmo=1800 --reconnect-delay=10

# Repeat for all portal/interface combinations to create full mesh

# Verify connections
sudo nvme list-subsys

# Create udev rule for queue-depth IO policy
sudo tee /etc/udev/rules.d/99-nvme-iopolicy.rules > /dev/null <<'EOF'
ACTION=="add|change", SUBSYSTEM=="nvme-subsystem", ATTR{iopolicy}="queue-depth"
EOF

sudo udevadm control --reload-rules
sudo udevadm trigger

# Create discovery configuration
sudo tee /etc/nvme/discovery.conf > /dev/null <<EOF
-t tcp -a <PORTAL_IP_1> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>
-t tcp -a <PORTAL_IP_2> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>
-t tcp -a <PORTAL_IP_3> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>
-t tcp -a <PORTAL_IP_4> -s 4420 --host-traddr=<HOST_IP_1> --host-iface <INTERFACE_NAME_1>

-t tcp -a <PORTAL_IP_1> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
-t tcp -a <PORTAL_IP_2> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
-t tcp -a <PORTAL_IP_3> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
-t tcp -a <PORTAL_IP_4> -s 4420 --host-traddr=<HOST_IP_2> --host-iface <INTERFACE_NAME_2>
EOF

# Enable autoconnect service
sudo systemctl enable --now nvmf-autoconnect.service

# Find NVMe device
sudo nvme list
# Example: /dev/nvme0n1

# Create LVM
sudo pvcreate /dev/nvme0n1
sudo vgcreate nvme-storage /dev/nvme0n1
sudo lvcreate -L 500G -n data nvme-storage

# Format and mount (SUSE: XFS recommended)
sudo mkfs.xfs /dev/nvme-storage/data
sudo mkdir -p /mnt/nvme-storage
sudo mount /dev/nvme-storage/data /mnt/nvme-storage

# Add to fstab
echo '/dev/nvme-storage/data /mnt/nvme-storage xfs defaults,_netdev 0 0' | sudo tee -a /etc/fstab

# Check multipath is enabled
cat /sys/module/nvme_core/parameters/multipath  # Should show: Y

# Check all paths are live
sudo nvme list-subsys

# Check IO policy
cat /sys/class/nvme-subsystem/nvme-subsys*/iopolicy  # Should show: queue-depth

# Verify block devices are visible
lsblk | grep nvme

# Verify storage (after LVM/filesystem creation)
df -h | grep nvme

# Reboot to confirm connections re-establish automatically
sudo reboot

# After reboot, verify all paths reconnected
sudo nvme list-subsys  # All paths should show "live"