Workloads

Workloads are both native platform workloads, Workspaces, Training and Inference, as well as workloads that originate from third-party ML frameworks, tools, or the broader Kubernetes ecosystems. For more details on the supported workloads,, see Introduction to workloads. Workloads endpoints allow you to list, retrieve, count, and view telemetry or metrics data for all workload types in your environment.

List workloads.

get

Retrieve a list of active workloads with details.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Query parameters

deletedbooleanOptional

Return only deleted resources when true.

offsetinteger · int32Optional

The offset of the first item returned in the collection.

Example: 100

limitinteger · int32 · min: 1 · max: 500Optional

The maximum number of entries to return.

Default: 50

sortOrderstring · enumOptional

Sort results in descending or ascending order.

Default: ascPossible values:

sortBystring · enumOptional

Sort results by a parameter.

Possible values:

filterBystring[]Optional

Filter results by a parameter. Use the format field-name operator value. Operators are == Equals, != Not equals, <= Less than or equal, >= Greater than or equal, =@ contains, !@ Does not contain, =^ Starts with and =$ Ends with. Dates are in ISO 8601 timestamp format and available for operators ==, !=, <= and >=.

Example: ["name!=some-workload-name","allocatedGPU>=2","createdAt>=2021-01-01T00:00:00Z"]

searchstringOptional

Filter results by a free text search.

Example: test project

Responses

200

Executed successfully.

application/json

nextintegerOptionalExample: 1

401

Unauthorized

application/json

403

Forbidden

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads

HTTPcURLJavaScriptPython

GET /api/v1/workloads HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "next": 1,
  "workloads": [
    {
      "tenantId": 1001,
      "runningPods": 1,
      "phaseUpdatedAt": "2022-06-08T11:28:24.131Z",
      "k8sPhaseUpdatedAt": "2022-06-08T11:28:24.131Z",
      "updatedAt": "2022-06-08T11:28:24.131Z",
      "source": "CLI",
      "deletedAt": "2022-08-12T19:28:24.131Z",
      "type": "runai-job",
      "name": "very-important-job",
      "id": "123e4567-e89b-12d3-a456-426614174000",
      "priority": 50,
      "priorityClassName": "high-priority",
      "submittedBy": "[email protected]",
      "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
      "projectName": "proj-1",
      "projectId": "1",
      "departmentName": "department-1",
      "departmentId": "1",
      "namespace": "runai-proj-1",
      "createdAt": "2022-01-01T03:49:52.531Z",
      "workloadRequestedResources": {
        "gpuRequestType": "portion",
        "gpu": {
          "limit": 1.5,
          "request": 1
        },
        "gpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "cpu": {
          "limit": 1.5,
          "request": 1
        },
        "cpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ]
      },
      "podsRequestedResources": {
        "gpuRequestType": "portion",
        "gpu": {
          "limit": 1.5,
          "request": 1
        },
        "gpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "cpu": {
          "limit": 1.5,
          "request": 1
        },
        "cpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ]
      },
      "allocatedResources": {
        "gpu": 1.5,
        "gpuMemory": "200Mi",
        "cpu": 0.5,
        "cpuMemory": "0B",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ]
      },
      "actionsSupport": {
        "delete": true,
        "suspend": true
      },
      "phase": "Creating",
      "conditions": [
        {
          "type": "Ready",
          "status": "False",
          "message": "Resource validation failed: ...",
          "reason": "ErrorConfig",
          "lastTransitionTime": "2022-01-01T03:49:52.531Z"
        }
      ],
      "phaseMessage": "Not enough resources in the requested nodepool",
      "k8sPhase": "Pending",
      "requestedPods": {
        "number": 1,
        "min": 2,
        "max": 5,
        "parallelism": 3,
        "completions": 5
      },
      "requestedNodePools": [
        "default"
      ],
      "currentNodePools": [
        "default"
      ],
      "completedAt": "2022-01-01T03:49:52.531Z",
      "images": [
        "alpine:latest"
      ],
      "urls": [
        "text"
      ],
      "datasources": [
        {
          "type": "pvc",
          "name": "my-pvc-datasource-1",
          "id": "123e4567-e89b-12d3-a456-426614174000"
        }
      ],
      "environments": [
        {
          "connections": [
            {
              "name": "my-pytorch-env",
              "toolType": "pytorch",
              "connectionType": "ExternalUrl",
              "url": "http://wandb.com/yourproject",
              "authorizationType": "public",
              "authorizedUsers": [
                "[email protected]",
                "[email protected]"
              ],
              "authorizedGroups": [
                "group-a",
                "group-b"
              ],
              "containerPort": 8080
            }
          ],
          "name": "pytorch",
          "id": "123e4567-e89b-12d3-a456-426614174000",
          "replicaType": "Master"
        }
      ],
      "externalConnections": [
        {
          "name": "my-pytorch-env",
          "toolType": "pytorch",
          "connectionType": "ExternalUrl",
          "url": "http://wandb.com/yourproject",
          "authorizationType": "public",
          "authorizedUsers": [
            "[email protected]",
            "[email protected]"
          ],
          "authorizedGroups": [
            "group-a",
            "group-b"
          ],
          "containerPort": 8080
        }
      ],
      "distributedFramework": "Pytorch",
      "additionalFields": {
        "ANY_ADDITIONAL_PROPERTY": "anything"
      },
      "preemptible": true,
      "environmentVariables": {
        "ANY_ADDITIONAL_PROPERTY": "text"
      },
      "command": "sleep",
      "arguments": "1000",
      "phaseReason": "text",
      "idleAllocatedGpus": 1,
      "totalPendingTimeSeconds": 60,
      "totalRunningTimeSeconds": 60,
      "category": "Train",
      "guaranteedRuntimeEndsAt": "2025-08-01T03:49:52.531Z",
      "aiApplicationId": "text",
      "aiApplicationName": "text",
      "sourceApi": "WorkloadsV2"
    }
  ]
}

Get a workload.

get

Retrieve workload data using a workloadId.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Responses

200

Executed successfully.

application/json

tenantIdinteger · int32Required

The id of the tenant.

Example: 1001

runningPodsinteger · int32RequiredExample: 1

phaseUpdatedAtstring · date-timeRequiredExample: 2022-06-08T11:28:24.131Z

k8sPhaseUpdatedAtstring · date-timeRequiredExample: 2022-06-08T11:28:24.131Z

updatedAtstring · date-timeRequiredExample: 2022-06-08T11:28:24.131Z

sourcestring · enumRequiredPossible values:

deletedAtstring · date-time · nullableRequiredExample: 2022-08-12T19:28:24.131Z

typestringRequiredExample: runai-job

namestringRequiredExample: very-important-job

idstring · uuidRequired

priorityinteger · int32 · nullableRequiredExample: 50

priorityClassNamestringRequiredExample: high-priority

submittedBystringOptionalExample: [email protected]

clusterIdstring · uuidRequired

The id of the cluster.

Example: 71f69d83-ba66-4822-adf5-55ce55efd210

projectNamestringRequiredExample: proj-1

projectIdstringRequiredExample: 1

departmentNamestringRequiredExample: department-1

departmentIdstringRequiredExample: 1

namespacestringRequiredExample: runai-proj-1

createdAtstring · date-timeRequiredExample: 2022-01-01T03:49:52.531Z

phasestring · enumRequiredPossible values:

phaseMessagestringOptionalExample: Not enough resources in the requested nodepool

k8sPhasestringRequiredExample: Pending

requestedNodePoolsstring[]OptionalExample: default

currentNodePoolsstring[]OptionalExample: default

completedAtstring · date-time · nullableOptionalExample: 2022-01-01T03:49:52.531Z

imagesstring[]OptionalExample: alpine:latest

urlsstring[]Optional

distributedFrameworkstringOptionalExample: Pytorch

preemptibleboolean · nullableOptional

commandstringOptionalExample: sleep

argumentsstringOptionalExample: 1000

phaseReasonstring · enumOptionalPossible values:

idleGpusinteger · nullableOptionalDeprecated

deprecated. use idleAllocatedGpus instead

Example: 3

idleAllocatedGpusnumber · nullableOptional

sum of idle allocated gpus in the workload

Example: 1

totalPendingTimeSecondsinteger · nullableOptional

The total cumulative time, in seconds, that the workload has spent in the Pending phase since submission.

Example: 60

totalRunningTimeSecondsinteger · nullableOptional

The total cumulative time, in seconds, that the workload has spent in the Running phase since submission.

Example: 60

categorystringRequired

Category Description

Example: Train

guaranteedRuntimeEndsAtstring · date-time · nullableOptional

A timestamp indicating when the workload will reach its minimum guaranteed runtime, as defined by minGuaranteedRuntime. Until this time, the workload is considered non-preemptible and cannot be interrupted by higher-priority workloads.

Example: 2025-08-01T03:49:52.531Z

aiApplicationIdstringOptional

aiApplicationNamestringOptional

sourceApistring · enum · nullableOptionalPossible values:

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/{workloadId}

HTTPcURLJavaScriptPython

GET /api/v1/workloads/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "tenantId": 1001,
  "runningPods": 1,
  "phaseUpdatedAt": "2022-06-08T11:28:24.131Z",
  "k8sPhaseUpdatedAt": "2022-06-08T11:28:24.131Z",
  "updatedAt": "2022-06-08T11:28:24.131Z",
  "source": "CLI",
  "deletedAt": "2022-08-12T19:28:24.131Z",
  "type": "runai-job",
  "name": "very-important-job",
  "id": "123e4567-e89b-12d3-a456-426614174000",
  "priority": 50,
  "priorityClassName": "high-priority",
  "submittedBy": "[email protected]",
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "projectName": "proj-1",
  "projectId": "1",
  "departmentName": "department-1",
  "departmentId": "1",
  "namespace": "runai-proj-1",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "workloadRequestedResources": {
    "gpuRequestType": "portion",
    "gpu": {
      "limit": 1.5,
      "request": 1
    },
    "gpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "cpu": {
      "limit": 1.5,
      "request": 1
    },
    "cpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "extendedResources": [
      {
        "resource": "hardware-vendor.example/foo",
        "quantity": 2,
        "exclude": false
      }
    ]
  },
  "podsRequestedResources": {
    "gpuRequestType": "portion",
    "gpu": {
      "limit": 1.5,
      "request": 1
    },
    "gpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "cpu": {
      "limit": 1.5,
      "request": 1
    },
    "cpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "extendedResources": [
      {
        "resource": "hardware-vendor.example/foo",
        "quantity": 2,
        "exclude": false
      }
    ]
  },
  "allocatedResources": {
    "gpu": 1.5,
    "gpuMemory": "200Mi",
    "cpu": 0.5,
    "cpuMemory": "0B",
    "extendedResources": [
      {
        "resource": "hardware-vendor.example/foo",
        "quantity": 2,
        "exclude": false
      }
    ]
  },
  "actionsSupport": {
    "delete": true,
    "suspend": true
  },
  "phase": "Creating",
  "conditions": [
    {
      "type": "Ready",
      "status": "False",
      "message": "Resource validation failed: ...",
      "reason": "ErrorConfig",
      "lastTransitionTime": "2022-01-01T03:49:52.531Z"
    }
  ],
  "phaseMessage": "Not enough resources in the requested nodepool",
  "k8sPhase": "Pending",
  "requestedPods": {
    "number": 1,
    "min": 2,
    "max": 5,
    "parallelism": 3,
    "completions": 5
  },
  "requestedNodePools": [
    "default"
  ],
  "currentNodePools": [
    "default"
  ],
  "completedAt": "2022-01-01T03:49:52.531Z",
  "images": [
    "alpine:latest"
  ],
  "urls": [
    "text"
  ],
  "datasources": [
    {
      "type": "pvc",
      "name": "my-pvc-datasource-1",
      "id": "123e4567-e89b-12d3-a456-426614174000"
    }
  ],
  "environments": [
    {
      "connections": [
        {
          "name": "my-pytorch-env",
          "toolType": "pytorch",
          "connectionType": "ExternalUrl",
          "url": "http://wandb.com/yourproject",
          "authorizationType": "public",
          "authorizedUsers": [
            "[email protected]",
            "[email protected]"
          ],
          "authorizedGroups": [
            "group-a",
            "group-b"
          ],
          "containerPort": 8080
        }
      ],
      "name": "pytorch",
      "id": "123e4567-e89b-12d3-a456-426614174000",
      "replicaType": "Master"
    }
  ],
  "externalConnections": [
    {
      "name": "my-pytorch-env",
      "toolType": "pytorch",
      "connectionType": "ExternalUrl",
      "url": "http://wandb.com/yourproject",
      "authorizationType": "public",
      "authorizedUsers": [
        "[email protected]",
        "[email protected]"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "containerPort": 8080
    }
  ],
  "distributedFramework": "Pytorch",
  "additionalFields": {
    "ANY_ADDITIONAL_PROPERTY": "anything"
  },
  "preemptible": true,
  "environmentVariables": {
    "ANY_ADDITIONAL_PROPERTY": "text"
  },
  "command": "sleep",
  "arguments": "1000",
  "phaseReason": "text",
  "idleAllocatedGpus": 1,
  "totalPendingTimeSeconds": 60,
  "totalRunningTimeSeconds": 60,
  "category": "Train",
  "guaranteedRuntimeEndsAt": "2025-08-01T03:49:52.531Z",
  "aiApplicationId": "text",
  "aiApplicationName": "text",
  "sourceApi": "WorkloadsV2",
  "pendingSchedulingMessages": [
    {
      "nodePool": "default",
      "phaseReason": "NonPreemptibleOverQuota",
      "reason": "Non-preemptible over quota",
      "orgType": "text",
      "userMessage": "You have reached the limit of non-preemptible resources"
    }
  ]
}

Count workloads.

get

Retrieve the number of workloads.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Query parameters

deletedbooleanOptional

Return only deleted resources when true.

filterBystring[]Optional

Filter results by a parameter. Use the format field-name operator value. Operators are == Equals, != Not equals, <= Less than or equal, >= Greater than or equal, =@ contains, !@ Does not contain, =^ Starts with and =$ Ends with. Dates are in ISO 8601 timestamp format and available for operators ==, !=, <= and >=.

Example: ["name!=some-workload-name","allocatedGPU>=2","createdAt>=2021-01-01T00:00:00Z"]

searchstringOptional

Filter results by a free text search.

Example: test project

Responses

200

Executed successfully.

application/json

countinteger · int64RequiredExample: 1

401

Unauthorized

application/json

403

Forbidden

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/count

HTTPcURLJavaScriptPython

GET /api/v1/workloads/count HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "count": 1
}

Get the workloads telemetry.

get

Retrieves workload data by telemetry type.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Query parameters

clusterIdstring · uuidOptional

Filter using the Universally Unique Identifier (UUID) of the cluster.

Example: d73a738f-fab3-430a-8fa3-5241493d7128

nodepoolNamestringOptional

Filter using the nodepool.

Example: default

departmentIdstringOptional

Filter using the department id.

Example: 1

telemetryTypestring · enumRequired

Select a telemetry type.

Possible values:

Responses

200

Executed successfully.

application/jsontext/csv

typestringRequired

specifies what data returned

Example: ALLOCATION_RATIO

timestampstring · date-timeRequiredExample: 2023-06-06 12:09:18.211

400

Bad request.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/telemetry

HTTPcURLJavaScriptPython

GET /api/v1/workloads/telemetry?telemetryType=WORKLOADS_COUNT HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "type": "ALLOCATION_RATIO",
  "timestamp": "2023-06-06 12:09:18.211",
  "values": [
    {
      "value": "85",
      "groups": [
        {
          "key": "department",
          "value": "1",
          "name": "department-A"
        }
      ]
    }
  ]
}

Get workload metrics data.

get

Retrieves workloads data metrics from the metrics database. Use in reporting and analysis tools.

Authorizations

bearerAuth

AuthorizationstringRequired

Bearer authentication

Path parameters

workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Query parameters

startstring · date-timeRequired

Start date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-06T12:09:18.211Z

endstring · date-timeRequired

End date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-07T12:09:18.211Z

numberOfSamplesinteger · max: 1000Optional

The number of samples to take in the specified time range.

Default: 20Example: 20

Responses

200

Executed successfully.

application/jsontext/csv

207

Partial success.

application/json

400

Bad request.

application/json

401

Unauthorized

application/json

403

Forbidden

application/json

404

The specified resource was not found

application/json

500

unexpected error

application/json

503

unexpected error

application/json

get

/api/v1/workloads/{workloadId}/metrics

HTTPcURLJavaScriptPython

GET /api/v1/workloads/{workloadId}/metrics?metricType=GPU_UTILIZATION&start=2023-06-06T12%3A09%3A18.211Z&end=2023-06-07T12%3A09%3A18.211Z HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "measurements": [
    {
      "type": "ALLOCATED_GPU",
      "labels": {
        "gpu": "0"
      },
      "values": [
        {
          "value": "85",
          "timestamp": "2024-12-20T14:30:00Z"
        },
        {
          "value": "92",
          "timestamp": "2024-12-20T14:30:30Z"
        },
        {
          "value": "78",
          "timestamp": "2024-12-20T14:37:00Z"
        }
      ]
    }
  ],
  "histogram": [
    {
      "type": "NIM_TIME_TO_FIRST_TOKEN_SECONDS",
      "values": [
        {
          "timestamp": "2024-12-20T14:30:00Z",
          "data": {
            "0.001": "45",
            "0.005": "178",
            "0.01": "312",
            "0.05": "624",
            "0.1": "812",
            "0.5": "980",
            "1.0": "1004",
            "2.0": "1015",
            "+Inf": "1024"
          }
        },
        {
          "timestamp": "2024-12-20T14:30:30Z",
          "data": {
            "0.001": "32",
            "0.005": "152",
            "0.01": "288",
            "0.05": "576",
            "0.1": "756",
            "0.5": "908",
            "1.0": "932",
            "2.0": "943",
            "+Inf": "960"
          }
        }
      ]
    },
    {
      "type": "NIM_TIME_TO_FIRST_TOKEN_SECONDS_PERCENTILES",
      "values": [
        {
          "timestamp": "2024-12-20T14:30:00Z",
          "data": {
            "p50": "0.0734",
            "p90": "0.1892",
            "p99": "0.2847"
          }
        },
        {
          "timestamp": "2024-12-20T14:30:30Z",
          "data": {
            "p50": "0.0823",
            "p90": "0.2134",
            "p99": "0.3156"
          }
        }
      ]
    }
  ]
}