Distributed Inferences

Distributed inference enables running inference workloads across multiple pods, typically to scale model serving beyond a single container or node. This approach is useful when a single instance cannot meet resource requirements.NVIDIA Run:ai supports this model using Leader Worker Set (LWS). Each pod plays a specific role, either as a leader or worker, and together they form a coordinated service. NVIDIA Run:ai manages the orchestration and configuration of these pods to ensure efficient and scalable inference execution

Create a distributed inference. [Experimental]

post

Create a distributed inference using container related fields.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Body

application/json

and

Responses

202

Request completed successfully.

application/json

400

Bad request.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

503

unexpected error

application/json

post

/api/v1/workloads/distributed-inferences

HTTPcURLJavaScriptPython

POST /api/v1/workloads/distributed-inferences HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Content-Type: application/json
Accept: */*
Content-Length: 5707

{
  "name": "my-workload-name",
  "useGivenNameAsPrefix": true,
  "projectId": 1,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "spec": {
    "category": "text",
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "priorityClass": "text",
    "restartPolicy": "RecreateGroupOnPodRestart",
    "servingPort": {
      "port": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[e-mail]",
        "[e-mail]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "exposeExternally": true,
      "exposedUrl": "text"
    },
    "startupPolicy": "LeaderCreated",
    "workers": 4,
    "replicas": 2,
    "leader": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    },
    "worker": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    }
  }
}

{
  "name": "my-workload-name",
  "requestedName": "text",
  "workloadId": "123e4567-e89b-12d3-a456-426614174000",
  "projectId": 1,
  "departmentId": 2,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "createdBy": "[e-mail]",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "deletedAt": "2022-01-01T03:49:52.531Z",
  "desiredPhase": "Running",
  "actualPhase": "Creating",
  "spec": {
    "category": "text",
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "priorityClass": "text",
    "restartPolicy": "RecreateGroupOnPodRestart",
    "servingPort": {
      "port": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[e-mail]",
        "[e-mail]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "exposeExternally": true,
      "exposedUrl": "text"
    },
    "startupPolicy": "LeaderCreated",
    "workers": 4,
    "replicas": 2,
    "leader": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    },
    "worker": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    }
  }
}

Get a distributed inference data.

get

Retrieve a distributed inference details using a workload id.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Responses

200

Executed successfully.

application/json

and

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/distributed-inferences/{workloadId}

HTTPcURLJavaScriptPython

GET /api/v1/workloads/distributed-inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "name": "my-workload-name",
  "requestedName": "text",
  "workloadId": "123e4567-e89b-12d3-a456-426614174000",
  "projectId": 1,
  "departmentId": 2,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "createdBy": "[e-mail]",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "deletedAt": "2022-01-01T03:49:52.531Z",
  "desiredPhase": "Running",
  "actualPhase": "Creating",
  "spec": {
    "category": "text",
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "priorityClass": "text",
    "restartPolicy": "RecreateGroupOnPodRestart",
    "servingPort": {
      "port": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[e-mail]",
        "[e-mail]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "exposeExternally": true,
      "exposedUrl": "text"
    },
    "startupPolicy": "LeaderCreated",
    "workers": 4,
    "replicas": 2,
    "leader": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    },
    "worker": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    }
  }
}

Delete a distributed inference.

delete

Delete a distributed inference using a workload id.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Responses

202

Accepted.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

delete

/api/v1/workloads/distributed-inferences/{workloadId}

HTTPcURLJavaScriptPython

DELETE /api/v1/workloads/distributed-inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "code": 202,
  "message": "Request has been accepted."
}

Update distributed inference spec.

patch

Update the specification of an existing distributed inference workload.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Body

application/json

The specifications of the inference to be updated.

Responses

202

Executed successfully.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

patch

/api/v1/workloads/distributed-inferences/{workloadId}

HTTPcURLJavaScriptPython

PATCH /api/v1/workloads/distributed-inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Content-Type: application/json
Accept: */*
Content-Length: 23

{
  "spec": {
    "replicas": 2
  }
}

{
  "name": "my-workload-name",
  "requestedName": "text",
  "workloadId": "123e4567-e89b-12d3-a456-426614174000",
  "projectId": 1,
  "departmentId": 2,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "createdBy": "[e-mail]",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "deletedAt": "2022-01-01T03:49:52.531Z",
  "desiredPhase": "Running",
  "actualPhase": "Creating",
  "spec": {
    "category": "text",
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "priorityClass": "text",
    "restartPolicy": "RecreateGroupOnPodRestart",
    "servingPort": {
      "port": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[e-mail]",
        "[e-mail]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "exposeExternally": true,
      "exposedUrl": "text"
    },
    "startupPolicy": "LeaderCreated",
    "workers": 4,
    "replicas": 2,
    "leader": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    },
    "worker": {
      "annotations": [
        {
          "name": "billing",
          "value": "my-billing-unit",
          "exclude": false
        }
      ],
      "args": "-x my-script.py",
      "command": "python",
      "compute": {
        "cpuCoreLimit": 2,
        "cpuCoreRequest": 0.5,
        "cpuMemoryLimit": "30M",
        "cpuMemoryRequest": "20M",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ],
        "gpuDevicesRequest": 1,
        "gpuMemoryLimit": "10M",
        "gpuMemoryRequest": "10M",
        "gpuPortionLimit": 0.5,
        "gpuPortionRequest": 0.5,
        "gpuRequestType": "portion",
        "largeShmRequest": false
      },
      "createHomeDir": true,
      "environmentVariables": [
        {
          "name": "HOME",
          "value": "/home/my-folder",
          "secret": {
            "name": "postgress_secret",
            "key": "POSTGRES_PASSWORD"
          },
          "configMap": {
            "name": "my-config-map",
            "key": "MY_POSTGRES_SCHEMA"
          },
          "podFieldRef": {
            "path": "metadata.name"
          },
          "exclude": false,
          "description": "Home directory of the user."
        }
      ],
      "image": "python:3.8",
      "imagePullPolicy": "Always",
      "imagePullSecrets": [
        {
          "name": "text",
          "userCredential": true,
          "exclude": false
        }
      ],
      "labels": [
        {
          "name": "stage",
          "value": "initial-research",
          "exclude": false
        }
      ],
      "nodeAffinityRequired": {
        "nodeSelectorTerms": [
          {
            "matchExpressions": [
              {
                "key": "text",
                "operator": "In",
                "values": [
                  "text"
                ]
              }
            ]
          }
        ]
      },
      "nodeType": "my-node-type",
      "podAffinity": {
        "type": "Required",
        "key": "text"
      },
      "probes": {
        "readiness": {
          "initialDelaySeconds": 1,
          "periodSeconds": 1,
          "timeoutSeconds": 1,
          "successThreshold": 1,
          "failureThreshold": 1,
          "handler": {
            "httpGet": {
              "path": "/",
              "port": 1,
              "host": "example.com",
              "scheme": "HTTP"
            }
          }
        }
      },
      "security": {
        "capabilities": [
          "CHOWN",
          "KILL"
        ],
        "readOnlyRootFilesystem": false,
        "runAsGid": 30,
        "runAsNonRoot": true,
        "runAsUid": 500,
        "seccompProfileType": "RuntimeDefault",
        "supplementalGroups": "2,3,5,8",
        "uidGidSource": "fromTheImage"
      },
      "storage": {
        "configMapVolume": [
          {
            "name": "storage-instance-a",
            "configMap": "text",
            "mountPath": "text",
            "subPath": "text",
            "defaultMode": "0644",
            "exclude": false
          }
        ],
        "emptyDirVolume": [
          {
            "name": "storage-instance-a",
            "path": "/mnt/emptydir",
            "medium": "text",
            "sizeLimit": "1G",
            "exclude": false
          }
        ],
        "pvc": [
          {
            "name": "storage-instance-a",
            "path": "/container/my-claim",
            "existingPvc": false,
            "claimName": "my-claim",
            "readOnly": false,
            "ephemeral": false,
            "claimInfo": {
              "size": "1G",
              "storageClass": "my-storage-class",
              "accessModes": {
                "readWriteOnce": true,
                "readOnlyMany": false,
                "readWriteMany": false
              },
              "volumeMode": "Filesystem",
              "addedAttrValues": [
                {
                  "key": "dnsname",
                  "value": "my.dns.com"
                }
              ]
            },
            "dataSharing": false,
            "exclude": false
          }
        ],
        "secretVolume": [
          {
            "name": "storage-instance-a",
            "mountPath": "text",
            "defaultMode": "0644",
            "secret": "text",
            "exclude": false
          }
        ]
      },
      "tolerations": [
        {
          "name": "text",
          "operator": "Equal",
          "key": "text",
          "value": "text",
          "effect": "NoSchedule",
          "seconds": 1,
          "exclude": false
        }
      ],
      "workingDir": "/home/myfolder"
    }
  }
}