DGX GB300 Configuration

Netop Values GB300 (netop-values-gb300.yaml)

---
nfd:
  enabled: true
  deployNodeFeatureRules: true
maintenanceOperator:
  enabled: true
nvIpam:
  deploy: true
sriovNetworkOperator:
  enabled: true
sriov-network-operator:
  sriovOperatorConfig:
    configDaemonNodeSelector:
      node-role.kubernetes.io/worker: ""
    featureGates:
      parallelNicConfig: true
      mellanoxFirmwareReset: false

SRIOV Node Pools Config (sriov-node-pool-config.yaml)

apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkPoolConfig
metadata:
  name: node-pool-unavailable-config
  namespace: network-operator
spec:
  maxUnavailable: "100%"
  nodeSelector:
    matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists

Combined IP Pools GB300 (combined-ippools-gb300.yaml)

---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-a-su-1
  namespace: network-operator
spec:
  subnet: 192.168.0.0/16
  perNodeBlockSize: 32
  gateway: 192.168.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-b-su-1
  namespace: network-operator
spec:
  subnet: 192.169.0.0/16
  perNodeBlockSize: 32
  gateway: 192.169.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-c-su-1
  namespace: network-operator
spec:
  subnet: 192.170.0.0/16
  perNodeBlockSize: 32
  gateway: 192.170.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists
---
apiVersion: nv-ipam.nvidia.com/v1alpha1
kind: IPPool
metadata:
  name: sriovibnet-pool-d-su-1
  namespace: network-operator
spec:
  subnet: 192.171.0.0/16
  perNodeBlockSize: 32
  gateway: 192.171.0.1
  nodeSelector:
    nodeSelectorTerms:
    - matchExpressions:
      - key: node-role.kubernetes.io/worker
        operator: Exists

Combined SRIOV IB Network (combined-sriovibnet-gb300.yaml)

---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-a-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibp3s0" ]
  numVfs: 16
  linkType: IB
  priority: 90    # used to resolve multiple policy definitions, lower value, higher priority
  isRdma: true
  resourceName: sriovib_resource_a
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-b-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibP2p3s0" ]
  numVfs: 16
  linkType: IB
  priority: 90    # used to resolve multiple policy definitions, lower value, higher priority
  isRdma: true
  resourceName: sriovib_resource_b
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-c-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibP16p3s0" ]
  numVfs: 16
  linkType: IB
  priority: 90    # used to resolve multiple policy definitions, lower value, higher priority
  isRdma: true
  resourceName: sriovib_resource_c
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: sriovibnet-rdma-node-policy-d-su-1
  namespace: network-operator
spec:
  deviceType: netdevice
  nicSelector:
    vendor: "15b3"
    pfNames: [ "ibP18p3s0" ]
  numVfs: 16
  linkType: IB
  priority: 90    # used to resolve multiple policy definitions, lower value, higher priority
  isRdma: true
  resourceName: sriovib_resource_d
  nodeSelector:
    node-role.kubernetes.io/worker: ""
    feature.node.kubernetes.io/pci-15b3.present: "true"
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-a-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_a"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-a-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-runai-a-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "runai"
  resourceName: "sriovib_resource_a"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-a-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-b-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_b"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-b-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-runai-b-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "runai"
  resourceName: "sriovib_resource_b"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-b-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-c-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_c"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-c-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-runai-c-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "runai"
  resourceName: "sriovib_resource_c"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-c-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-default-d-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "default"
  resourceName: "sriovib_resource_d"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-d-su-1",
      "poolType": "IPPool"
    }
---
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovIBNetwork
metadata:
  name: "sriovibnet-rdma-runai-d-su-1"
  namespace: network-operator
spec:
  linkState: enable
  networkNamespace: "runai"
  resourceName: "sriovib_resource_d"
  ipam: |
    {
      "type": "nv-ipam",
      "datastore": "kubernetes",
      "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/nv-ipam.d/nv-ipam.kubeconfig"
      },
      "log_file": "/var/log/SriovIBNetwork_nv-ipam.log",
      "log_level": "debug",
      "poolName": "sriovibnet-pool-d-su-1",
      "poolType": "IPPool"
    }

NIC Cluster Policy GB300 (nic-cluster-policy-gb300.yaml)

---
apiVersion: mellanox.com/v1alpha1
kind: NicClusterPolicy
metadata:
  name: nic-cluster-policy
spec:
  secondaryNetwork:
    cniPlugins:
      image: plugins
      repository: nvcr.io/nvidia/mellanox
      version: network-operator-v25.10.0
      imagePullSecrets: []
    multus:
      image: multus-cni
      repository: nvcr.io/nvidia/mellanox
      version: network-operator-v25.10.0
      imagePullSecrets: []
      containerResources:
        - name: "kube-multus"
          limits: {memory: "100Mi"}
          requests: {memory: "100Mi"}
  nvIpam:
    image: nvidia-k8s-ipam
    repository: nvcr.io/nvidia/mellanox
    version: network-operator-v25.10.0
    imagePullSecrets: []
    enableWebhook: false