Inferences

Inference workloads deploy trained models into a production environment to generate predictions from live data. These workloads are prioritized over Trainings and Workspaces during scheduling. NVIDIA Run:ai Inference workloads support auto-scaling to maintain service-level agreements (SLAs) by dynamically adjusting resources as demand changes.

Create an inference.

post

Create an inference using container related fields.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Body

application/json

and

Responses

202

Request completed successfully.

application/json

400

Bad request.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

503

unexpected error

application/json

post

/api/v1/workloads/inferences

HTTPcURLJavaScriptPython

POST /api/v1/workloads/inferences HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Content-Type: application/json
Accept: */*
Content-Length: 4634

{
  "name": "my-workload-name",
  "useGivenNameAsPrefix": true,
  "projectId": 1,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "spec": {
    "annotations": [
      {
        "name": "billing",
        "value": "my-billing-unit",
        "exclude": false
      }
    ],
    "args": "-x my-script.py",
    "category": "text",
    "command": "python",
    "compute": {
      "cpuCoreLimit": 2,
      "cpuCoreRequest": 0.5,
      "cpuMemoryLimit": "30M",
      "cpuMemoryRequest": "20M",
      "extendedResources": [
        {
          "resource": "hardware-vendor.example/foo",
          "quantity": 2,
          "exclude": false
        }
      ],
      "gpuDevicesRequest": 1,
      "gpuMemoryLimit": "10M",
      "gpuMemoryRequest": "10M",
      "gpuPortionLimit": 0.5,
      "gpuPortionRequest": 0.5,
      "gpuRequestType": "portion",
      "largeShmRequest": false
    },
    "createHomeDir": true,
    "environmentVariables": [
      {
        "name": "HOME",
        "value": "/home/my-folder",
        "secret": {
          "name": "postgress_secret",
          "key": "POSTGRES_PASSWORD"
        },
        "configMap": {
          "name": "my-config-map",
          "key": "MY_POSTGRES_SCHEMA"
        },
        "podFieldRef": {
          "path": "metadata.name"
        },
        "exclude": false,
        "description": "Home directory of the user."
      }
    ],
    "exposedUrls": [
      {
        "container": 8080,
        "url": "https://my-url.com",
        "authorizedUsers": [
          "user-a",
          "user-b"
        ],
        "authorizedGroups": [
          "group-a",
          "group-b"
        ],
        "toolType": "jupyter",
        "toolName": "my-pytorch",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "image": "python:3.8",
    "imagePullPolicy": "Always",
    "imagePullSecrets": [
      {
        "name": "text",
        "userCredential": true,
        "exclude": false
      }
    ],
    "labels": [
      {
        "name": "stage",
        "value": "initial-research",
        "exclude": false
      }
    ],
    "nodeAffinityRequired": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "text",
              "operator": "In",
              "values": [
                "text"
              ]
            }
          ]
        }
      ]
    },
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "nodeType": "my-node-type",
    "podAffinity": {
      "type": "Required",
      "key": "text"
    },
    "ports": [
      {
        "container": 8080,
        "serviceType": "LoadBalancer",
        "external": 30080,
        "toolType": "pytorch",
        "toolName": "my-pytorch",
        "name": "port-instance-a",
        "exclude": false
      }
    ],
    "priorityClass": "text",
    "probes": {
      "readiness": {
        "initialDelaySeconds": 1,
        "periodSeconds": 1,
        "timeoutSeconds": 1,
        "successThreshold": 1,
        "failureThreshold": 1,
        "handler": {
          "httpGet": {
            "path": "/",
            "port": 1,
            "host": "example.com",
            "scheme": "HTTP"
          }
        }
      }
    },
    "relatedUrls": [
      {
        "url": "https://my-url.com",
        "type": "wandb",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "security": {
      "capabilities": [
        "CHOWN",
        "KILL"
      ],
      "readOnlyRootFilesystem": false,
      "runAsGid": 30,
      "runAsNonRoot": true,
      "runAsUid": 500,
      "seccompProfileType": "RuntimeDefault",
      "supplementalGroups": "2,3,5,8",
      "uidGidSource": "fromTheImage"
    },
    "servingPort": {
      "container": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[email protected]",
        "[email protected]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "clusterLocalAccessOnly": true
    },
    "storage": {
      "configMapVolume": [
        {
          "name": "storage-instance-a",
          "configMap": "text",
          "mountPath": "text",
          "subPath": "text",
          "defaultMode": "0644",
          "exclude": false
        }
      ],
      "dataVolume": [
        {
          "id": "123e4567-e89b-12d3-a456-426614174000",
          "mountPath": "/mnt/data",
          "exclude": false
        }
      ],
      "emptyDirVolume": [
        {
          "name": "storage-instance-a",
          "path": "/mnt/emptydir",
          "medium": "text",
          "sizeLimit": "1G",
          "exclude": false
        }
      ],
      "git": [
        {
          "name": "storage-instance-a",
          "repository": "https://github.com/my-git/my-repo",
          "branch": "main",
          "revision": "text",
          "path": "/container/my-repository",
          "passwordSecret": "my-password-secret",
          "secretKeyOfUser": "User",
          "secretKeyOfPassword": "Password",
          "exclude": false,
          "secretRef": {
            "name": "my-password-secret",
            "authenticationMethod": "password",
            "secretKeyOfUser": "User",
            "secretKeyOfPassword": "Password"
          }
        }
      ],
      "hostPath": [
        {
          "name": "storage-instance-a",
          "path": "/container/directory",
          "readOnly": true,
          "mountPath": "/local/directory",
          "mountPropagation": "None",
          "exclude": false
        }
      ],
      "nfs": [
        {
          "name": "storage-instance-a",
          "path": "/container/nfs",
          "readOnly": true,
          "server": "my.nfs.com",
          "mountPath": "/local/nfs",
          "exclude": false
        }
      ],
      "pvc": [
        {
          "name": "storage-instance-a",
          "path": "/container/my-claim",
          "existingPvc": false,
          "claimName": "my-claim",
          "readOnly": false,
          "ephemeral": false,
          "claimInfo": {
            "size": "1G",
            "storageClass": "my-storage-class",
            "accessModes": {
              "readWriteOnce": true,
              "readOnlyMany": false,
              "readWriteMany": false
            },
            "volumeMode": "Filesystem",
            "addedAttrValues": [
              {
                "key": "dnsname",
                "value": "my.dns.com"
              }
            ]
          },
          "dataSharing": false,
          "exclude": false
        }
      ],
      "secretVolume": [
        {
          "name": "storage-instance-a",
          "mountPath": "text",
          "defaultMode": "0644",
          "secret": "text",
          "exclude": false
        }
      ]
    },
    "tolerations": [
      {
        "name": "text",
        "operator": "Equal",
        "key": "text",
        "value": "text",
        "effect": "NoSchedule",
        "seconds": 1,
        "exclude": false
      }
    ],
    "workingDir": "/home/myfolder",
    "autoscaling": {
      "metricThresholdPercentage": 1,
      "minReplicas": 1,
      "maxReplicas": 1,
      "initialReplicas": 1,
      "activationReplicas": 1,
      "concurrencyHardLimit": 1,
      "scaleToZeroRetentionSeconds": 1,
      "scaleDownDelaySeconds": 1,
      "metric": "http_requests_total",
      "metricThreshold": 1
    },
    "servingConfiguration": {
      "initializationTimeoutSeconds": 1,
      "requestTimeoutSeconds": 1
    }
  }
}

{
  "name": "my-workload-name",
  "requestedName": "text",
  "workloadId": "123e4567-e89b-12d3-a456-426614174000",
  "projectId": 1,
  "departmentId": 2,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "createdBy": "[email protected]",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "deletedAt": "2022-01-01T03:49:52.531Z",
  "desiredPhase": "Running",
  "actualPhase": "Creating",
  "spec": {
    "annotations": [
      {
        "name": "billing",
        "value": "my-billing-unit",
        "exclude": false
      }
    ],
    "args": "-x my-script.py",
    "category": "text",
    "command": "python",
    "compute": {
      "cpuCoreLimit": 2,
      "cpuCoreRequest": 0.5,
      "cpuMemoryLimit": "30M",
      "cpuMemoryRequest": "20M",
      "extendedResources": [
        {
          "resource": "hardware-vendor.example/foo",
          "quantity": 2,
          "exclude": false
        }
      ],
      "gpuDevicesRequest": 1,
      "gpuMemoryLimit": "10M",
      "gpuMemoryRequest": "10M",
      "gpuPortionLimit": 0.5,
      "gpuPortionRequest": 0.5,
      "gpuRequestType": "portion",
      "largeShmRequest": false
    },
    "createHomeDir": true,
    "environmentVariables": [
      {
        "name": "HOME",
        "value": "/home/my-folder",
        "secret": {
          "name": "postgress_secret",
          "key": "POSTGRES_PASSWORD"
        },
        "configMap": {
          "name": "my-config-map",
          "key": "MY_POSTGRES_SCHEMA"
        },
        "podFieldRef": {
          "path": "metadata.name"
        },
        "exclude": false,
        "description": "Home directory of the user."
      }
    ],
    "exposedUrls": [
      {
        "container": 8080,
        "url": "https://my-url.com",
        "authorizedUsers": [
          "user-a",
          "user-b"
        ],
        "authorizedGroups": [
          "group-a",
          "group-b"
        ],
        "toolType": "jupyter",
        "toolName": "my-pytorch",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "image": "python:3.8",
    "imagePullPolicy": "Always",
    "imagePullSecrets": [
      {
        "name": "text",
        "userCredential": true,
        "exclude": false
      }
    ],
    "labels": [
      {
        "name": "stage",
        "value": "initial-research",
        "exclude": false
      }
    ],
    "nodeAffinityRequired": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "text",
              "operator": "In",
              "values": [
                "text"
              ]
            }
          ]
        }
      ]
    },
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "nodeType": "my-node-type",
    "podAffinity": {
      "type": "Required",
      "key": "text"
    },
    "ports": [
      {
        "container": 8080,
        "serviceType": "LoadBalancer",
        "external": 30080,
        "toolType": "pytorch",
        "toolName": "my-pytorch",
        "name": "port-instance-a",
        "exclude": false
      }
    ],
    "priorityClass": "text",
    "probes": {
      "readiness": {
        "initialDelaySeconds": 1,
        "periodSeconds": 1,
        "timeoutSeconds": 1,
        "successThreshold": 1,
        "failureThreshold": 1,
        "handler": {
          "httpGet": {
            "path": "/",
            "port": 1,
            "host": "example.com",
            "scheme": "HTTP"
          }
        }
      }
    },
    "relatedUrls": [
      {
        "url": "https://my-url.com",
        "type": "wandb",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "security": {
      "capabilities": [
        "CHOWN",
        "KILL"
      ],
      "readOnlyRootFilesystem": false,
      "runAsGid": 30,
      "runAsNonRoot": true,
      "runAsUid": 500,
      "seccompProfileType": "RuntimeDefault",
      "supplementalGroups": "2,3,5,8",
      "uidGidSource": "fromTheImage"
    },
    "servingPort": {
      "container": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[email protected]",
        "[email protected]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "clusterLocalAccessOnly": true
    },
    "storage": {
      "configMapVolume": [
        {
          "name": "storage-instance-a",
          "configMap": "text",
          "mountPath": "text",
          "subPath": "text",
          "defaultMode": "0644",
          "exclude": false
        }
      ],
      "dataVolume": [
        {
          "id": "123e4567-e89b-12d3-a456-426614174000",
          "mountPath": "/mnt/data",
          "exclude": false
        }
      ],
      "emptyDirVolume": [
        {
          "name": "storage-instance-a",
          "path": "/mnt/emptydir",
          "medium": "text",
          "sizeLimit": "1G",
          "exclude": false
        }
      ],
      "git": [
        {
          "name": "storage-instance-a",
          "repository": "https://github.com/my-git/my-repo",
          "branch": "main",
          "revision": "text",
          "path": "/container/my-repository",
          "passwordSecret": "my-password-secret",
          "secretKeyOfUser": "User",
          "secretKeyOfPassword": "Password",
          "exclude": false,
          "secretRef": {
            "name": "my-password-secret",
            "authenticationMethod": "password",
            "secretKeyOfUser": "User",
            "secretKeyOfPassword": "Password"
          }
        }
      ],
      "hostPath": [
        {
          "name": "storage-instance-a",
          "path": "/container/directory",
          "readOnly": true,
          "mountPath": "/local/directory",
          "mountPropagation": "None",
          "exclude": false
        }
      ],
      "nfs": [
        {
          "name": "storage-instance-a",
          "path": "/container/nfs",
          "readOnly": true,
          "server": "my.nfs.com",
          "mountPath": "/local/nfs",
          "exclude": false
        }
      ],
      "pvc": [
        {
          "name": "storage-instance-a",
          "path": "/container/my-claim",
          "existingPvc": false,
          "claimName": "my-claim",
          "readOnly": false,
          "ephemeral": false,
          "claimInfo": {
            "size": "1G",
            "storageClass": "my-storage-class",
            "accessModes": {
              "readWriteOnce": true,
              "readOnlyMany": false,
              "readWriteMany": false
            },
            "volumeMode": "Filesystem",
            "addedAttrValues": [
              {
                "key": "dnsname",
                "value": "my.dns.com"
              }
            ]
          },
          "dataSharing": false,
          "exclude": false
        }
      ],
      "secretVolume": [
        {
          "name": "storage-instance-a",
          "mountPath": "text",
          "defaultMode": "0644",
          "secret": "text",
          "exclude": false
        }
      ]
    },
    "tolerations": [
      {
        "name": "text",
        "operator": "Equal",
        "key": "text",
        "value": "text",
        "effect": "NoSchedule",
        "seconds": 1,
        "exclude": false
      }
    ],
    "workingDir": "/home/myfolder",
    "autoscaling": {
      "metricThresholdPercentage": 1,
      "minReplicas": 1,
      "maxReplicas": 1,
      "initialReplicas": 1,
      "activationReplicas": 1,
      "concurrencyHardLimit": 1,
      "scaleToZeroRetentionSeconds": 1,
      "scaleDownDelaySeconds": 1,
      "metric": "http_requests_total",
      "metricThreshold": 1
    },
    "servingConfiguration": {
      "initializationTimeoutSeconds": 1,
      "requestTimeoutSeconds": 1
    }
  }
}

Get inference data.

get

Retrieve inference details using a workload id.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Responses

200

Executed successfully.

application/json

and

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/inferences/{workloadId}

HTTPcURLJavaScriptPython

GET /api/v1/workloads/inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "name": "my-workload-name",
  "requestedName": "text",
  "workloadId": "123e4567-e89b-12d3-a456-426614174000",
  "projectId": 1,
  "departmentId": 2,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "createdBy": "[email protected]",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "deletedAt": "2022-01-01T03:49:52.531Z",
  "desiredPhase": "Running",
  "actualPhase": "Creating",
  "spec": {
    "annotations": [
      {
        "name": "billing",
        "value": "my-billing-unit",
        "exclude": false
      }
    ],
    "args": "-x my-script.py",
    "category": "text",
    "command": "python",
    "compute": {
      "cpuCoreLimit": 2,
      "cpuCoreRequest": 0.5,
      "cpuMemoryLimit": "30M",
      "cpuMemoryRequest": "20M",
      "extendedResources": [
        {
          "resource": "hardware-vendor.example/foo",
          "quantity": 2,
          "exclude": false
        }
      ],
      "gpuDevicesRequest": 1,
      "gpuMemoryLimit": "10M",
      "gpuMemoryRequest": "10M",
      "gpuPortionLimit": 0.5,
      "gpuPortionRequest": 0.5,
      "gpuRequestType": "portion",
      "largeShmRequest": false
    },
    "createHomeDir": true,
    "environmentVariables": [
      {
        "name": "HOME",
        "value": "/home/my-folder",
        "secret": {
          "name": "postgress_secret",
          "key": "POSTGRES_PASSWORD"
        },
        "configMap": {
          "name": "my-config-map",
          "key": "MY_POSTGRES_SCHEMA"
        },
        "podFieldRef": {
          "path": "metadata.name"
        },
        "exclude": false,
        "description": "Home directory of the user."
      }
    ],
    "exposedUrls": [
      {
        "container": 8080,
        "url": "https://my-url.com",
        "authorizedUsers": [
          "user-a",
          "user-b"
        ],
        "authorizedGroups": [
          "group-a",
          "group-b"
        ],
        "toolType": "jupyter",
        "toolName": "my-pytorch",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "image": "python:3.8",
    "imagePullPolicy": "Always",
    "imagePullSecrets": [
      {
        "name": "text",
        "userCredential": true,
        "exclude": false
      }
    ],
    "labels": [
      {
        "name": "stage",
        "value": "initial-research",
        "exclude": false
      }
    ],
    "nodeAffinityRequired": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "text",
              "operator": "In",
              "values": [
                "text"
              ]
            }
          ]
        }
      ]
    },
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "nodeType": "my-node-type",
    "podAffinity": {
      "type": "Required",
      "key": "text"
    },
    "ports": [
      {
        "container": 8080,
        "serviceType": "LoadBalancer",
        "external": 30080,
        "toolType": "pytorch",
        "toolName": "my-pytorch",
        "name": "port-instance-a",
        "exclude": false
      }
    ],
    "priorityClass": "text",
    "probes": {
      "readiness": {
        "initialDelaySeconds": 1,
        "periodSeconds": 1,
        "timeoutSeconds": 1,
        "successThreshold": 1,
        "failureThreshold": 1,
        "handler": {
          "httpGet": {
            "path": "/",
            "port": 1,
            "host": "example.com",
            "scheme": "HTTP"
          }
        }
      }
    },
    "relatedUrls": [
      {
        "url": "https://my-url.com",
        "type": "wandb",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "security": {
      "capabilities": [
        "CHOWN",
        "KILL"
      ],
      "readOnlyRootFilesystem": false,
      "runAsGid": 30,
      "runAsNonRoot": true,
      "runAsUid": 500,
      "seccompProfileType": "RuntimeDefault",
      "supplementalGroups": "2,3,5,8",
      "uidGidSource": "fromTheImage"
    },
    "servingPort": {
      "container": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[email protected]",
        "[email protected]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "clusterLocalAccessOnly": true
    },
    "storage": {
      "configMapVolume": [
        {
          "name": "storage-instance-a",
          "configMap": "text",
          "mountPath": "text",
          "subPath": "text",
          "defaultMode": "0644",
          "exclude": false
        }
      ],
      "dataVolume": [
        {
          "id": "123e4567-e89b-12d3-a456-426614174000",
          "mountPath": "/mnt/data",
          "exclude": false
        }
      ],
      "emptyDirVolume": [
        {
          "name": "storage-instance-a",
          "path": "/mnt/emptydir",
          "medium": "text",
          "sizeLimit": "1G",
          "exclude": false
        }
      ],
      "git": [
        {
          "name": "storage-instance-a",
          "repository": "https://github.com/my-git/my-repo",
          "branch": "main",
          "revision": "text",
          "path": "/container/my-repository",
          "passwordSecret": "my-password-secret",
          "secretKeyOfUser": "User",
          "secretKeyOfPassword": "Password",
          "exclude": false,
          "secretRef": {
            "name": "my-password-secret",
            "authenticationMethod": "password",
            "secretKeyOfUser": "User",
            "secretKeyOfPassword": "Password"
          }
        }
      ],
      "hostPath": [
        {
          "name": "storage-instance-a",
          "path": "/container/directory",
          "readOnly": true,
          "mountPath": "/local/directory",
          "mountPropagation": "None",
          "exclude": false
        }
      ],
      "nfs": [
        {
          "name": "storage-instance-a",
          "path": "/container/nfs",
          "readOnly": true,
          "server": "my.nfs.com",
          "mountPath": "/local/nfs",
          "exclude": false
        }
      ],
      "pvc": [
        {
          "name": "storage-instance-a",
          "path": "/container/my-claim",
          "existingPvc": false,
          "claimName": "my-claim",
          "readOnly": false,
          "ephemeral": false,
          "claimInfo": {
            "size": "1G",
            "storageClass": "my-storage-class",
            "accessModes": {
              "readWriteOnce": true,
              "readOnlyMany": false,
              "readWriteMany": false
            },
            "volumeMode": "Filesystem",
            "addedAttrValues": [
              {
                "key": "dnsname",
                "value": "my.dns.com"
              }
            ]
          },
          "dataSharing": false,
          "exclude": false
        }
      ],
      "secretVolume": [
        {
          "name": "storage-instance-a",
          "mountPath": "text",
          "defaultMode": "0644",
          "secret": "text",
          "exclude": false
        }
      ]
    },
    "tolerations": [
      {
        "name": "text",
        "operator": "Equal",
        "key": "text",
        "value": "text",
        "effect": "NoSchedule",
        "seconds": 1,
        "exclude": false
      }
    ],
    "workingDir": "/home/myfolder",
    "autoscaling": {
      "metricThresholdPercentage": 1,
      "minReplicas": 1,
      "maxReplicas": 1,
      "initialReplicas": 1,
      "activationReplicas": 1,
      "concurrencyHardLimit": 1,
      "scaleToZeroRetentionSeconds": 1,
      "scaleDownDelaySeconds": 1,
      "metric": "http_requests_total",
      "metricThreshold": 1
    },
    "servingConfiguration": {
      "initializationTimeoutSeconds": 1,
      "requestTimeoutSeconds": 1
    }
  }
}

Delete an inference.

delete

Delete an inference using a workload id.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Responses

202

Accepted.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

delete

/api/v1/workloads/inferences/{workloadId}

HTTPcURLJavaScriptPython

DELETE /api/v1/workloads/inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "code": 202,
  "message": "Request has been accepted."
}

Update inference spec. [Experimental]

patch

Update the specification of an existing inference workload.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Body

application/json

The specifications of the inference to be updated.

Responses

202

Executed successfully.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

patch

/api/v1/workloads/inferences/{workloadId}

HTTPcURLJavaScriptPython

PATCH /api/v1/workloads/inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Content-Type: application/json
Accept: */*
Content-Length: 1634

{
  "spec": {
    "args": "-x my-script.py",
    "category": "text",
    "command": "python",
    "compute": {
      "cpuCoreLimit": 2,
      "cpuCoreRequest": 0.5,
      "cpuMemoryLimit": "30M",
      "cpuMemoryRequest": "20M",
      "extendedResources": [
        {
          "resource": "hardware-vendor.example/foo",
          "quantity": 2,
          "exclude": false
        }
      ],
      "gpuDevicesRequest": 1,
      "gpuMemoryLimit": "10M",
      "gpuMemoryRequest": "10M",
      "gpuPortionLimit": 0.5,
      "gpuPortionRequest": 0.5,
      "gpuRequestType": "portion",
      "largeShmRequest": false
    },
    "createHomeDir": true,
    "environmentVariables": [
      {
        "name": "HOME",
        "value": "/home/my-folder",
        "secret": {
          "name": "postgress_secret",
          "key": "POSTGRES_PASSWORD"
        },
        "configMap": {
          "name": "my-config-map",
          "key": "MY_POSTGRES_SCHEMA"
        },
        "podFieldRef": {
          "path": "metadata.name"
        },
        "exclude": false,
        "description": "Home directory of the user."
      }
    ],
    "image": "python:3.8",
    "imagePullPolicy": "Always",
    "nodeAffinityRequired": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "text",
              "operator": "In",
              "values": [
                "text"
              ]
            }
          ]
        }
      ]
    },
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "nodeType": "my-node-type",
    "podAffinity": {
      "type": "Required",
      "key": "text"
    },
    "priorityClass": "text",
    "probes": {
      "readiness": {
        "initialDelaySeconds": 1,
        "periodSeconds": 1,
        "timeoutSeconds": 1,
        "successThreshold": 1,
        "failureThreshold": 1,
        "handler": {
          "httpGet": {
            "path": "/",
            "port": 1,
            "host": "example.com",
            "scheme": "HTTP"
          }
        }
      }
    },
    "workingDir": "/home/myfolder",
    "autoscaling": {
      "metricThresholdPercentage": 1,
      "minReplicas": 1,
      "maxReplicas": 1,
      "initialReplicas": 1,
      "activationReplicas": 1,
      "concurrencyHardLimit": 1,
      "scaleToZeroRetentionSeconds": 1,
      "scaleDownDelaySeconds": 1,
      "metric": "http_requests_total",
      "metricThreshold": 1
    },
    "servingConfiguration": {
      "initializationTimeoutSeconds": 1,
      "requestTimeoutSeconds": 1
    }
  }
}

{
  "name": "my-workload-name",
  "requestedName": "text",
  "workloadId": "123e4567-e89b-12d3-a456-426614174000",
  "projectId": 1,
  "departmentId": 2,
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "createdBy": "[email protected]",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "deletedAt": "2022-01-01T03:49:52.531Z",
  "desiredPhase": "Running",
  "actualPhase": "Creating",
  "spec": {
    "annotations": [
      {
        "name": "billing",
        "value": "my-billing-unit",
        "exclude": false
      }
    ],
    "args": "-x my-script.py",
    "category": "text",
    "command": "python",
    "compute": {
      "cpuCoreLimit": 2,
      "cpuCoreRequest": 0.5,
      "cpuMemoryLimit": "30M",
      "cpuMemoryRequest": "20M",
      "extendedResources": [
        {
          "resource": "hardware-vendor.example/foo",
          "quantity": 2,
          "exclude": false
        }
      ],
      "gpuDevicesRequest": 1,
      "gpuMemoryLimit": "10M",
      "gpuMemoryRequest": "10M",
      "gpuPortionLimit": 0.5,
      "gpuPortionRequest": 0.5,
      "gpuRequestType": "portion",
      "largeShmRequest": false
    },
    "createHomeDir": true,
    "environmentVariables": [
      {
        "name": "HOME",
        "value": "/home/my-folder",
        "secret": {
          "name": "postgress_secret",
          "key": "POSTGRES_PASSWORD"
        },
        "configMap": {
          "name": "my-config-map",
          "key": "MY_POSTGRES_SCHEMA"
        },
        "podFieldRef": {
          "path": "metadata.name"
        },
        "exclude": false,
        "description": "Home directory of the user."
      }
    ],
    "exposedUrls": [
      {
        "container": 8080,
        "url": "https://my-url.com",
        "authorizedUsers": [
          "user-a",
          "user-b"
        ],
        "authorizedGroups": [
          "group-a",
          "group-b"
        ],
        "toolType": "jupyter",
        "toolName": "my-pytorch",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "image": "python:3.8",
    "imagePullPolicy": "Always",
    "imagePullSecrets": [
      {
        "name": "text",
        "userCredential": true,
        "exclude": false
      }
    ],
    "labels": [
      {
        "name": "stage",
        "value": "initial-research",
        "exclude": false
      }
    ],
    "nodeAffinityRequired": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "text",
              "operator": "In",
              "values": [
                "text"
              ]
            }
          ]
        }
      ]
    },
    "nodePools": [
      "my-node-pool-a",
      "my-node-pool-b"
    ],
    "nodeType": "my-node-type",
    "podAffinity": {
      "type": "Required",
      "key": "text"
    },
    "ports": [
      {
        "container": 8080,
        "serviceType": "LoadBalancer",
        "external": 30080,
        "toolType": "pytorch",
        "toolName": "my-pytorch",
        "name": "port-instance-a",
        "exclude": false
      }
    ],
    "priorityClass": "text",
    "probes": {
      "readiness": {
        "initialDelaySeconds": 1,
        "periodSeconds": 1,
        "timeoutSeconds": 1,
        "successThreshold": 1,
        "failureThreshold": 1,
        "handler": {
          "httpGet": {
            "path": "/",
            "port": 1,
            "host": "example.com",
            "scheme": "HTTP"
          }
        }
      }
    },
    "relatedUrls": [
      {
        "url": "https://my-url.com",
        "type": "wandb",
        "name": "url-instance-a",
        "exclude": false
      }
    ],
    "security": {
      "capabilities": [
        "CHOWN",
        "KILL"
      ],
      "readOnlyRootFilesystem": false,
      "runAsGid": 30,
      "runAsNonRoot": true,
      "runAsUid": 500,
      "seccompProfileType": "RuntimeDefault",
      "supplementalGroups": "2,3,5,8",
      "uidGidSource": "fromTheImage"
    },
    "servingPort": {
      "container": 8080,
      "protocol": "http",
      "authorizationType": "public",
      "authorizedUsers": [
        "[email protected]",
        "[email protected]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "clusterLocalAccessOnly": true
    },
    "storage": {
      "configMapVolume": [
        {
          "name": "storage-instance-a",
          "configMap": "text",
          "mountPath": "text",
          "subPath": "text",
          "defaultMode": "0644",
          "exclude": false
        }
      ],
      "dataVolume": [
        {
          "id": "123e4567-e89b-12d3-a456-426614174000",
          "mountPath": "/mnt/data",
          "exclude": false
        }
      ],
      "emptyDirVolume": [
        {
          "name": "storage-instance-a",
          "path": "/mnt/emptydir",
          "medium": "text",
          "sizeLimit": "1G",
          "exclude": false
        }
      ],
      "git": [
        {
          "name": "storage-instance-a",
          "repository": "https://github.com/my-git/my-repo",
          "branch": "main",
          "revision": "text",
          "path": "/container/my-repository",
          "passwordSecret": "my-password-secret",
          "secretKeyOfUser": "User",
          "secretKeyOfPassword": "Password",
          "exclude": false,
          "secretRef": {
            "name": "my-password-secret",
            "authenticationMethod": "password",
            "secretKeyOfUser": "User",
            "secretKeyOfPassword": "Password"
          }
        }
      ],
      "hostPath": [
        {
          "name": "storage-instance-a",
          "path": "/container/directory",
          "readOnly": true,
          "mountPath": "/local/directory",
          "mountPropagation": "None",
          "exclude": false
        }
      ],
      "nfs": [
        {
          "name": "storage-instance-a",
          "path": "/container/nfs",
          "readOnly": true,
          "server": "my.nfs.com",
          "mountPath": "/local/nfs",
          "exclude": false
        }
      ],
      "pvc": [
        {
          "name": "storage-instance-a",
          "path": "/container/my-claim",
          "existingPvc": false,
          "claimName": "my-claim",
          "readOnly": false,
          "ephemeral": false,
          "claimInfo": {
            "size": "1G",
            "storageClass": "my-storage-class",
            "accessModes": {
              "readWriteOnce": true,
              "readOnlyMany": false,
              "readWriteMany": false
            },
            "volumeMode": "Filesystem",
            "addedAttrValues": [
              {
                "key": "dnsname",
                "value": "my.dns.com"
              }
            ]
          },
          "dataSharing": false,
          "exclude": false
        }
      ],
      "secretVolume": [
        {
          "name": "storage-instance-a",
          "mountPath": "text",
          "defaultMode": "0644",
          "secret": "text",
          "exclude": false
        }
      ]
    },
    "tolerations": [
      {
        "name": "text",
        "operator": "Equal",
        "key": "text",
        "value": "text",
        "effect": "NoSchedule",
        "seconds": 1,
        "exclude": false
      }
    ],
    "workingDir": "/home/myfolder",
    "autoscaling": {
      "metricThresholdPercentage": 1,
      "minReplicas": 1,
      "maxReplicas": 1,
      "initialReplicas": 1,
      "activationReplicas": 1,
      "concurrencyHardLimit": 1,
      "scaleToZeroRetentionSeconds": 1,
      "scaleDownDelaySeconds": 1,
      "metric": "http_requests_total",
      "metricThreshold": 1
    },
    "servingConfiguration": {
      "initializationTimeoutSeconds": 1,
      "requestTimeoutSeconds": 1
    }
  }
}

Get inference metrics data.

get

Retrieve inference metrics data by id. Supported from control-plane version 2.18 or later.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Query parameters

startstring · date-timeRequired

Start date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-06T12:09:18.211Z

endstring · date-timeRequired

End date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-07T12:09:18.211Z

numberOfSamplesinteger · max: 1000Optional

The number of samples to take in the specified time range.

Default: 20Example: 20

Responses

200

Executed successfully.

application/jsontext/csv

207

Partial success.

application/json

400

Bad request.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/inferences/{workloadId}/metrics

HTTPcURLJavaScriptPython

GET /api/v1/workloads/inferences/{workloadId}/metrics?metricType=THROUGHPUT&start=2023-06-06T12%3A09%3A18.211Z&end=2023-06-07T12%3A09%3A18.211Z HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "measurements": [
    {
      "type": "ALLOCATED_GPU",
      "labels": "{'gpu': '3'}",
      "values": [
        {
          "value": "85",
          "timestamp": "2023-06-06 12:09:18.211"
        }
      ]
    }
  ]
}

Get inference pod's metrics data.

get

Retrieve inference metrics pod's data by workload and pod id. Supported from control-plane version 2.18 or later.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

podIdstring · uuidRequired

The requested pod id.

Query parameters

startstring · date-timeRequired

Start date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-06T12:09:18.211Z

endstring · date-timeRequired

End date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-07T12:09:18.211Z

numberOfSamplesinteger · max: 1000Optional

The number of samples to take in the specified time range.

Default: 20Example: 20

Responses

200

Executed successfully.

application/jsontext/csv

207

Partial success.

application/json

400

Bad request.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/inferences/{workloadId}/pods/{podId}/metrics

HTTPcURLJavaScriptPython

GET /api/v1/workloads/inferences/{workloadId}/pods/{podId}/metrics?metricType=THROUGHPUT&start=2023-06-06T12%3A09%3A18.211Z&end=2023-06-07T12%3A09%3A18.211Z HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "measurements": [
    {
      "type": "ALLOCATED_GPU",
      "labels": "{'gpu': '3'}",
      "values": [
        {
          "value": "85",
          "timestamp": "2023-06-06 12:09:18.211"
        }
      ]
    }
  ]
}