Inferences
Inference workloads deploy trained models into a production environment to generate predictions from live data. These workloads are prioritized over Trainings and Workspaces during scheduling. NVIDIA Run:ai Inference workloads support auto-scaling to maintain service-level agreements (SLAs) by dynamically adjusting resources as demand changes.
Create an inference using container related fields.
Authorizations
AuthorizationstringRequired
Bearer authentication
Body
and
Responses
202
Request completed successfully.
application/json
and
400
Bad request.
application/json
401
Unauthorized
application/json
403
Forbidden
application/json
503
unexpected error
application/json
post/api/v1/workloads/inferences
POST /api/v1/workloads/inferences HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Content-Type: application/json
Accept: */*
Content-Length: 4458
{
"name": "my-workload-name",
"useGivenNameAsPrefix": true,
"projectId": 1,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"spec": {
"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {
"readiness": {
"initialDelaySeconds": 1,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {
"httpGet": {
"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodeAffinityRequired": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "text",
"operator": "In",
"values": [
"text"
]
}
]
}
]
},
"podAffinity": {
"type": "Required",
"key": "text"
},
"category": "text",
"priorityClass": "text",
"nodePools": [
"my-node-pool-a",
"my-node-pool-b"
],
"environmentVariables": [
{
"name": "HOME",
"value": "/home/my-folder",
"secret": {
"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {
"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {
"path": "metadata.name"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"annotations": [
{
"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"labels": [
{
"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"imagePullSecrets": [
{
"name": "text",
"userCredential": true,
"exclude": false
}
],
"tolerations": [
{
"name": "text",
"operator": "Equal",
"key": "text",
"value": "text",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"ports": [
{
"container": 8080,
"serviceType": "LoadBalancer",
"external": 30080,
"toolType": "pytorch",
"toolName": "my-pytorch",
"name": "port-instance-a",
"exclude": false
}
],
"exposedUrls": [
{
"container": 8080,
"url": "https://my-url.com",
"authorizedUsers": [
"user-a",
"user-b"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"toolType": "jupyter",
"toolName": "my-pytorch",
"name": "url-instance-a",
"exclude": false
}
],
"relatedUrls": [
{
"url": "https://my-url.com",
"type": "wandb",
"name": "url-instance-a",
"exclude": false
}
],
"compute": {
"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [
{
"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"storage": {
"dataVolume": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"mountPath": "/mnt/data",
"exclude": false
}
],
"pvc": [
{
"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {
"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {
"readWriteOnce": true,
"readOnlyMany": false,
"readWriteMany": false
},
"volumeMode": "Filesystem",
"addedAttrValues": [
{
"key": "dnsname",
"value": "my.dns.com"
}
]
},
"dataSharing": false,
"exclude": false
}
],
"hostPath": [
{
"name": "storage-instance-a",
"path": "/container/directory",
"readOnly": true,
"mountPath": "/local/directory",
"mountPropagation": "None",
"exclude": false
}
],
"nfs": [
{
"name": "storage-instance-a",
"path": "/container/nfs",
"readOnly": true,
"server": "my.nfs.com",
"mountPath": "/local/nfs",
"exclude": false
}
],
"git": [
{
"name": "storage-instance-a",
"repository": "https://github.com/my-git/my-repo",
"branch": "main",
"revision": "text",
"path": "/container/my-repository",
"passwordSecret": "my-password-secret",
"secretKeyOfUser": "User",
"secretKeyOfPassword": "Password",
"exclude": false
}
],
"configMapVolume": [
{
"name": "storage-instance-a",
"configMap": "text",
"mountPath": "text",
"subPath": "text",
"exclude": false
}
],
"secretVolume": [
{
"name": "storage-instance-a",
"mountPath": "text",
"secret": "text",
"exclude": false
}
],
"emptyDirVolume": [
{
"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "text",
"sizeLimit": "1G",
"exclude": false
}
]
},
"security": {
"uidGidSource": "fromTheImage",
"capabilities": [
"CHOWN",
"KILL"
],
"seccompProfileType": "RuntimeDefault",
"runAsNonRoot": true,
"readOnlyRootFilesystem": false,
"runAsUid": 500,
"runAsGid": 30,
"supplementalGroups": "2,3,5,8"
},
"servingPort": {
"container": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": [
"user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"clusterLocalAccessOnly": true
},
"autoscaling": {
"metricThresholdPercentage": 1,
"minReplicas": 1,
"maxReplicas": 1,
"initialReplicas": 1,
"activationReplicas": 1,
"concurrencyHardLimit": 1,
"scaleToZeroRetentionSeconds": 1,
"scaleDownDelaySeconds": 1,
"metric": "http_requests_total",
"metricThreshold": 1
},
"servingConfiguration": {
"initializationTimeoutSeconds": 1,
"requestTimeoutSeconds": 1
}
}
}{
"name": "my-workload-name",
"requestedName": "text",
"workloadId": "123e4567-e89b-12d3-a456-426614174000",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"deletedAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {
"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {
"readiness": {
"initialDelaySeconds": 1,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {
"httpGet": {
"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodeAffinityRequired": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "text",
"operator": "In",
"values": [
"text"
]
}
]
}
]
},
"podAffinity": {
"type": "Required",
"key": "text"
},
"category": "text",
"priorityClass": "text",
"nodePools": [
"my-node-pool-a",
"my-node-pool-b"
],
"environmentVariables": [
{
"name": "HOME",
"value": "/home/my-folder",
"secret": {
"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {
"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {
"path": "metadata.name"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"annotations": [
{
"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"labels": [
{
"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"imagePullSecrets": [
{
"name": "text",
"userCredential": true,
"exclude": false
}
],
"tolerations": [
{
"name": "text",
"operator": "Equal",
"key": "text",
"value": "text",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"ports": [
{
"container": 8080,
"serviceType": "LoadBalancer",
"external": 30080,
"toolType": "pytorch",
"toolName": "my-pytorch",
"name": "port-instance-a",
"exclude": false
}
],
"exposedUrls": [
{
"container": 8080,
"url": "https://my-url.com",
"authorizedUsers": [
"user-a",
"user-b"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"toolType": "jupyter",
"toolName": "my-pytorch",
"name": "url-instance-a",
"exclude": false
}
],
"relatedUrls": [
{
"url": "https://my-url.com",
"type": "wandb",
"name": "url-instance-a",
"exclude": false
}
],
"compute": {
"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [
{
"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"storage": {
"dataVolume": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"mountPath": "/mnt/data",
"exclude": false
}
],
"pvc": [
{
"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {
"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {
"readWriteOnce": true,
"readOnlyMany": false,
"readWriteMany": false
},
"volumeMode": "Filesystem",
"addedAttrValues": [
{
"key": "dnsname",
"value": "my.dns.com"
}
]
},
"dataSharing": false,
"exclude": false
}
],
"hostPath": [
{
"name": "storage-instance-a",
"path": "/container/directory",
"readOnly": true,
"mountPath": "/local/directory",
"mountPropagation": "None",
"exclude": false
}
],
"nfs": [
{
"name": "storage-instance-a",
"path": "/container/nfs",
"readOnly": true,
"server": "my.nfs.com",
"mountPath": "/local/nfs",
"exclude": false
}
],
"git": [
{
"name": "storage-instance-a",
"repository": "https://github.com/my-git/my-repo",
"branch": "main",
"revision": "text",
"path": "/container/my-repository",
"passwordSecret": "my-password-secret",
"secretKeyOfUser": "User",
"secretKeyOfPassword": "Password",
"exclude": false
}
],
"configMapVolume": [
{
"name": "storage-instance-a",
"configMap": "text",
"mountPath": "text",
"subPath": "text",
"exclude": false
}
],
"secretVolume": [
{
"name": "storage-instance-a",
"mountPath": "text",
"secret": "text",
"exclude": false
}
],
"emptyDirVolume": [
{
"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "text",
"sizeLimit": "1G",
"exclude": false
}
]
},
"security": {
"uidGidSource": "fromTheImage",
"capabilities": [
"CHOWN",
"KILL"
],
"seccompProfileType": "RuntimeDefault",
"runAsNonRoot": true,
"readOnlyRootFilesystem": false,
"runAsUid": 500,
"runAsGid": 30,
"supplementalGroups": "2,3,5,8"
},
"servingPort": {
"container": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": [
"user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"clusterLocalAccessOnly": true
},
"autoscaling": {
"metricThresholdPercentage": 1,
"minReplicas": 1,
"maxReplicas": 1,
"initialReplicas": 1,
"activationReplicas": 1,
"concurrencyHardLimit": 1,
"scaleToZeroRetentionSeconds": 1,
"scaleDownDelaySeconds": 1,
"metric": "http_requests_total",
"metricThreshold": 1
},
"servingConfiguration": {
"initializationTimeoutSeconds": 1,
"requestTimeoutSeconds": 1
}
}
}Retrieve inference details using a workload id.
Authorizations
AuthorizationstringRequired
Bearer authentication
Path parameters
workloadIdstring · uuidRequired
The Universally Unique Identifier (UUID) of the workload.
Responses
200
Executed successfully.
application/json
and
401
Unauthorized
application/json
403
Forbidden
application/json
404
The specified resource was not found
application/json
500
unexpected error
application/json
503
unexpected error
application/json
get/api/v1/workloads/inferences/{workloadId}
GET /api/v1/workloads/inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*
{
"name": "my-workload-name",
"requestedName": "text",
"workloadId": "123e4567-e89b-12d3-a456-426614174000",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"deletedAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {
"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {
"readiness": {
"initialDelaySeconds": 1,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {
"httpGet": {
"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodeAffinityRequired": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "text",
"operator": "In",
"values": [
"text"
]
}
]
}
]
},
"podAffinity": {
"type": "Required",
"key": "text"
},
"category": "text",
"priorityClass": "text",
"nodePools": [
"my-node-pool-a",
"my-node-pool-b"
],
"environmentVariables": [
{
"name": "HOME",
"value": "/home/my-folder",
"secret": {
"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {
"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {
"path": "metadata.name"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"annotations": [
{
"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"labels": [
{
"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"imagePullSecrets": [
{
"name": "text",
"userCredential": true,
"exclude": false
}
],
"tolerations": [
{
"name": "text",
"operator": "Equal",
"key": "text",
"value": "text",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"ports": [
{
"container": 8080,
"serviceType": "LoadBalancer",
"external": 30080,
"toolType": "pytorch",
"toolName": "my-pytorch",
"name": "port-instance-a",
"exclude": false
}
],
"exposedUrls": [
{
"container": 8080,
"url": "https://my-url.com",
"authorizedUsers": [
"user-a",
"user-b"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"toolType": "jupyter",
"toolName": "my-pytorch",
"name": "url-instance-a",
"exclude": false
}
],
"relatedUrls": [
{
"url": "https://my-url.com",
"type": "wandb",
"name": "url-instance-a",
"exclude": false
}
],
"compute": {
"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [
{
"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"storage": {
"dataVolume": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"mountPath": "/mnt/data",
"exclude": false
}
],
"pvc": [
{
"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {
"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {
"readWriteOnce": true,
"readOnlyMany": false,
"readWriteMany": false
},
"volumeMode": "Filesystem",
"addedAttrValues": [
{
"key": "dnsname",
"value": "my.dns.com"
}
]
},
"dataSharing": false,
"exclude": false
}
],
"hostPath": [
{
"name": "storage-instance-a",
"path": "/container/directory",
"readOnly": true,
"mountPath": "/local/directory",
"mountPropagation": "None",
"exclude": false
}
],
"nfs": [
{
"name": "storage-instance-a",
"path": "/container/nfs",
"readOnly": true,
"server": "my.nfs.com",
"mountPath": "/local/nfs",
"exclude": false
}
],
"git": [
{
"name": "storage-instance-a",
"repository": "https://github.com/my-git/my-repo",
"branch": "main",
"revision": "text",
"path": "/container/my-repository",
"passwordSecret": "my-password-secret",
"secretKeyOfUser": "User",
"secretKeyOfPassword": "Password",
"exclude": false
}
],
"configMapVolume": [
{
"name": "storage-instance-a",
"configMap": "text",
"mountPath": "text",
"subPath": "text",
"exclude": false
}
],
"secretVolume": [
{
"name": "storage-instance-a",
"mountPath": "text",
"secret": "text",
"exclude": false
}
],
"emptyDirVolume": [
{
"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "text",
"sizeLimit": "1G",
"exclude": false
}
]
},
"security": {
"uidGidSource": "fromTheImage",
"capabilities": [
"CHOWN",
"KILL"
],
"seccompProfileType": "RuntimeDefault",
"runAsNonRoot": true,
"readOnlyRootFilesystem": false,
"runAsUid": 500,
"runAsGid": 30,
"supplementalGroups": "2,3,5,8"
},
"servingPort": {
"container": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": [
"user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"clusterLocalAccessOnly": true
},
"autoscaling": {
"metricThresholdPercentage": 1,
"minReplicas": 1,
"maxReplicas": 1,
"initialReplicas": 1,
"activationReplicas": 1,
"concurrencyHardLimit": 1,
"scaleToZeroRetentionSeconds": 1,
"scaleDownDelaySeconds": 1,
"metric": "http_requests_total",
"metricThreshold": 1
},
"servingConfiguration": {
"initializationTimeoutSeconds": 1,
"requestTimeoutSeconds": 1
}
}
}Delete an inference using a workload id.
Authorizations
AuthorizationstringRequired
Bearer authentication
Path parameters
workloadIdstring · uuidRequired
The Universally Unique Identifier (UUID) of the workload.
Responses
202
Accepted.
application/json
codeinteger · min: 100 · max: 599Required
messagestringRequired
401
Unauthorized
application/json
403
Forbidden
application/json
404
The specified resource was not found
application/json
500
unexpected error
application/json
503
unexpected error
application/json
delete/api/v1/workloads/inferences/{workloadId}
DELETE /api/v1/workloads/inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*
{
"code": 202,
"message": "Request has been accepted."
}Update the specification of an existing inference workload.
Authorizations
AuthorizationstringRequired
Bearer authentication
Path parameters
workloadIdstring · uuidRequired
The Universally Unique Identifier (UUID) of the workload.
Body
The specifications of the inference to be updated.
Responses
202
Executed successfully.
application/json
and
401
Unauthorized
application/json
403
Forbidden
application/json
404
The specified resource was not found
application/json
500
unexpected error
application/json
503
unexpected error
application/json
patch/api/v1/workloads/inferences/{workloadId}
PATCH /api/v1/workloads/inferences/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Content-Type: application/json
Accept: */*
Content-Length: 1634
{
"spec": {
"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {
"readiness": {
"initialDelaySeconds": 1,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {
"httpGet": {
"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodeAffinityRequired": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "text",
"operator": "In",
"values": [
"text"
]
}
]
}
]
},
"podAffinity": {
"type": "Required",
"key": "text"
},
"category": "text",
"priorityClass": "text",
"nodePools": [
"my-node-pool-a",
"my-node-pool-b"
],
"environmentVariables": [
{
"name": "HOME",
"value": "/home/my-folder",
"secret": {
"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {
"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {
"path": "metadata.name"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"compute": {
"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [
{
"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"autoscaling": {
"metricThresholdPercentage": 1,
"minReplicas": 1,
"maxReplicas": 1,
"initialReplicas": 1,
"activationReplicas": 1,
"concurrencyHardLimit": 1,
"scaleToZeroRetentionSeconds": 1,
"scaleDownDelaySeconds": 1,
"metric": "http_requests_total",
"metricThreshold": 1
},
"servingConfiguration": {
"initializationTimeoutSeconds": 1,
"requestTimeoutSeconds": 1
}
}
}{
"name": "my-workload-name",
"requestedName": "text",
"workloadId": "123e4567-e89b-12d3-a456-426614174000",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"deletedAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {
"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {
"readiness": {
"initialDelaySeconds": 1,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {
"httpGet": {
"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodeAffinityRequired": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "text",
"operator": "In",
"values": [
"text"
]
}
]
}
]
},
"podAffinity": {
"type": "Required",
"key": "text"
},
"category": "text",
"priorityClass": "text",
"nodePools": [
"my-node-pool-a",
"my-node-pool-b"
],
"environmentVariables": [
{
"name": "HOME",
"value": "/home/my-folder",
"secret": {
"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {
"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {
"path": "metadata.name"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"annotations": [
{
"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"labels": [
{
"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"imagePullSecrets": [
{
"name": "text",
"userCredential": true,
"exclude": false
}
],
"tolerations": [
{
"name": "text",
"operator": "Equal",
"key": "text",
"value": "text",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"ports": [
{
"container": 8080,
"serviceType": "LoadBalancer",
"external": 30080,
"toolType": "pytorch",
"toolName": "my-pytorch",
"name": "port-instance-a",
"exclude": false
}
],
"exposedUrls": [
{
"container": 8080,
"url": "https://my-url.com",
"authorizedUsers": [
"user-a",
"user-b"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"toolType": "jupyter",
"toolName": "my-pytorch",
"name": "url-instance-a",
"exclude": false
}
],
"relatedUrls": [
{
"url": "https://my-url.com",
"type": "wandb",
"name": "url-instance-a",
"exclude": false
}
],
"compute": {
"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [
{
"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"storage": {
"dataVolume": [
{
"id": "123e4567-e89b-12d3-a456-426614174000",
"mountPath": "/mnt/data",
"exclude": false
}
],
"pvc": [
{
"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {
"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {
"readWriteOnce": true,
"readOnlyMany": false,
"readWriteMany": false
},
"volumeMode": "Filesystem",
"addedAttrValues": [
{
"key": "dnsname",
"value": "my.dns.com"
}
]
},
"dataSharing": false,
"exclude": false
}
],
"hostPath": [
{
"name": "storage-instance-a",
"path": "/container/directory",
"readOnly": true,
"mountPath": "/local/directory",
"mountPropagation": "None",
"exclude": false
}
],
"nfs": [
{
"name": "storage-instance-a",
"path": "/container/nfs",
"readOnly": true,
"server": "my.nfs.com",
"mountPath": "/local/nfs",
"exclude": false
}
],
"git": [
{
"name": "storage-instance-a",
"repository": "https://github.com/my-git/my-repo",
"branch": "main",
"revision": "text",
"path": "/container/my-repository",
"passwordSecret": "my-password-secret",
"secretKeyOfUser": "User",
"secretKeyOfPassword": "Password",
"exclude": false
}
],
"configMapVolume": [
{
"name": "storage-instance-a",
"configMap": "text",
"mountPath": "text",
"subPath": "text",
"exclude": false
}
],
"secretVolume": [
{
"name": "storage-instance-a",
"mountPath": "text",
"secret": "text",
"exclude": false
}
],
"emptyDirVolume": [
{
"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "text",
"sizeLimit": "1G",
"exclude": false
}
]
},
"security": {
"uidGidSource": "fromTheImage",
"capabilities": [
"CHOWN",
"KILL"
],
"seccompProfileType": "RuntimeDefault",
"runAsNonRoot": true,
"readOnlyRootFilesystem": false,
"runAsUid": 500,
"runAsGid": 30,
"supplementalGroups": "2,3,5,8"
},
"servingPort": {
"container": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": [
"user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": [
"group-a",
"group-b"
],
"clusterLocalAccessOnly": true
},
"autoscaling": {
"metricThresholdPercentage": 1,
"minReplicas": 1,
"maxReplicas": 1,
"initialReplicas": 1,
"activationReplicas": 1,
"concurrencyHardLimit": 1,
"scaleToZeroRetentionSeconds": 1,
"scaleDownDelaySeconds": 1,
"metric": "http_requests_total",
"metricThreshold": 1
},
"servingConfiguration": {
"initializationTimeoutSeconds": 1,
"requestTimeoutSeconds": 1
}
}
}Retrieve inference metrics data by id. Supported from control-plane version 2.18 or later.
Authorizations
AuthorizationstringRequired
Bearer authentication
Path parameters
workloadIdstring · uuidRequired
The Universally Unique Identifier (UUID) of the workload.
Query parameters
startstring · date-timeRequiredExample:
Start date of time range to fetch data in ISO 8601 timestamp format.
2023-06-06T12:09:18.211Zendstring · date-timeRequiredExample:
End date of time range to fetch data in ISO 8601 timestamp format.
2023-06-07T12:09:18.211ZnumberOfSamplesinteger · max: 1000OptionalDefault:
The number of samples to take in the specified time range.
20Example: 20Responses
200
Executed successfully.
207
Partial success.
application/json
400
Bad request.
application/json
401
Unauthorized
application/json
403
Forbidden
application/json
404
The specified resource was not found
application/json
500
unexpected error
application/json
503
unexpected error
application/json
get/api/v1/workloads/inferences/{workloadId}/metrics
GET /api/v1/workloads/inferences/{workloadId}/metrics?metricType=THROUGHPUT&start=2023-06-06T12%3A09%3A18.211Z&end=2023-06-07T12%3A09%3A18.211Z HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*
{
"measurements": [
{
"type": "ALLOCATED_GPU",
"labels": "{'gpu': '3'}",
"values": [
{
"value": "85",
"timestamp": "2023-06-06 12:09:18.211"
}
]
}
]
}Retrieve inference metrics pod's data by workload and pod id. Supported from control-plane version 2.18 or later.
Authorizations
AuthorizationstringRequired
Bearer authentication
Path parameters
workloadIdstring · uuidRequired
The Universally Unique Identifier (UUID) of the workload.
podIdstring · uuidRequired
The requested pod id.
Query parameters
startstring · date-timeRequiredExample:
Start date of time range to fetch data in ISO 8601 timestamp format.
2023-06-06T12:09:18.211Zendstring · date-timeRequiredExample:
End date of time range to fetch data in ISO 8601 timestamp format.
2023-06-07T12:09:18.211ZnumberOfSamplesinteger · max: 1000OptionalDefault:
The number of samples to take in the specified time range.
20Example: 20Responses
200
Executed successfully.
207
Partial success.
application/json
400
Bad request.
application/json
401
Unauthorized
application/json
403
Forbidden
application/json
404
The specified resource was not found
application/json
500
unexpected error
application/json
503
unexpected error
application/json
get/api/v1/workloads/inferences/{workloadId}/pods/{podId}/metrics
GET /api/v1/workloads/inferences/{workloadId}/pods/{podId}/metrics?metricType=THROUGHPUT&start=2023-06-06T12%3A09%3A18.211Z&end=2023-06-07T12%3A09%3A18.211Z HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*
{
"measurements": [
{
"type": "ALLOCATED_GPU",
"labels": "{'gpu': '3'}",
"values": [
{
"value": "85",
"timestamp": "2023-06-06 12:09:18.211"
}
]
}
]
}Last updated