Ctrlk
Contact support
For the complete documentation index, see llms.txt. This page is also available as Markdown.

Workloads

List workloads.

get

Retrieve a list of active workloads with details.

Authorizations
AuthorizationstringRequired

Bearer authentication

Query parameters
deletedbooleanOptional

Return only deleted resources when true.

offsetinteger · int32Optional

The offset of the first item returned in the collection.

Example: 100
limitinteger · int32 · min: 1 · max: 500Optional

The maximum number of entries to return.

Default: 50
sortOrderstring · enumOptional

Sort results in descending or ascending order.

Default: ascPossible values:
sortBystring · enumOptional

Sort results by a parameter.

Possible values:
filterBystring[]Optional

Filter results by a parameter. Use the format field-name operator value. Operators are == Equals, != Not equals, <= Less than or equal, >= Greater than or equal, =@ contains, !@ Does not contain, =^ Starts with and =$ Ends with. Dates are in ISO 8601 timestamp format and available for operators ==, !=, <= and >=.

Example: ["name!=some-workload-name","allocatedGPU>=2","createdAt>=2021-01-01T00:00:00Z"]
searchstringOptional

Filter results by a free text search.

Example: test project
Responses
200

Executed successfully.

application/json
nextintegerOptionalExample: 1
get/api/v1/workloads
GET /api/v1/workloads HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "next": 1,
  "workloads": [
    {
      "tenantId": 1001,
      "runningPods": 1,
      "phaseUpdatedAt": "2022-06-08T11:28:24.131Z",
      "k8sPhaseUpdatedAt": "2022-06-08T11:28:24.131Z",
      "updatedAt": "2022-06-08T11:28:24.131Z",
      "source": "CLI",
      "deletedAt": "2022-08-12T19:28:24.131Z",
      "type": "runai-job",
      "name": "very-important-job",
      "id": "123e4567-e89b-12d3-a456-426614174000",
      "priority": 50,
      "priorityClassName": "high-priority",
      "submittedBy": "researcher@run.ai",
      "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
      "projectName": "proj-1",
      "projectId": "1",
      "departmentName": "department-1",
      "departmentId": "1",
      "namespace": "runai-proj-1",
      "createdAt": "2022-01-01T03:49:52.531Z",
      "workloadRequestedResources": {
        "gpuRequestType": "portion",
        "gpu": {
          "limit": 1.5,
          "request": 1
        },
        "gpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "cpu": {
          "limit": 1.5,
          "request": 1
        },
        "cpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ]
      },
      "podsRequestedResources": {
        "gpuRequestType": "portion",
        "gpu": {
          "limit": 1.5,
          "request": 1
        },
        "gpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "cpu": {
          "limit": 1.5,
          "request": 1
        },
        "cpuMemory": {
          "limit": "2G",
          "request": "200M"
        },
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ]
      },
      "allocatedResources": {
        "gpu": 1.5,
        "gpuMemory": "200Mi",
        "cpu": 0.5,
        "cpuMemory": "0B",
        "extendedResources": [
          {
            "resource": "hardware-vendor.example/foo",
            "quantity": 2,
            "exclude": false
          }
        ]
      },
      "actionsSupport": {
        "delete": true,
        "suspend": true
      },
      "phase": "Creating",
      "conditions": [
        {
          "type": "Ready",
          "status": "False",
          "message": "Resource validation failed: ...",
          "reason": "ErrorConfig",
          "lastTransitionTime": "2022-01-01T03:49:52.531Z"
        }
      ],
      "phaseMessage": "Not enough resources in the requested nodepool",
      "k8sPhase": "Pending",
      "requestedPods": {
        "number": 1,
        "min": 2,
        "max": 5,
        "parallelism": 3,
        "completions": 5
      },
      "requestedNodePools": [
        "default"
      ],
      "currentNodePools": [
        "default"
      ],
      "completedAt": "2022-01-01T03:49:52.531Z",
      "images": [
        "alpine:latest"
      ],
      "urls": [
        "text"
      ],
      "datasources": [
        {
          "type": "pvc",
          "name": "my-pvc-datasource-1",
          "id": "123e4567-e89b-12d3-a456-426614174000"
        }
      ],
      "environments": [
        {
          "connections": [
            {
              "name": "my-pytorch-env",
              "toolType": "pytorch",
              "connectionType": "ExternalUrl",
              "url": "http://wandb.com/yourproject",
              "authorizationType": "public",
              "authorizedUsers": [
                "user@company.ai",
                "another@company.ai"
              ],
              "authorizedGroups": [
                "group-a",
                "group-b"
              ],
              "containerPort": 8080
            }
          ],
          "name": "pytorch",
          "id": "123e4567-e89b-12d3-a456-426614174000",
          "replicaType": "Master"
        }
      ],
      "externalConnections": [
        {
          "name": "my-pytorch-env",
          "toolType": "pytorch",
          "connectionType": "ExternalUrl",
          "url": "http://wandb.com/yourproject",
          "authorizationType": "public",
          "authorizedUsers": [
            "user@company.ai",
            "another@company.ai"
          ],
          "authorizedGroups": [
            "group-a",
            "group-b"
          ],
          "containerPort": 8080
        }
      ],
      "distributedFramework": "Pytorch",
      "additionalFields": {
        "ANY_ADDITIONAL_PROPERTY": "anything"
      },
      "preemptible": true,
      "environmentVariables": {
        "ANY_ADDITIONAL_PROPERTY": "text"
      },
      "command": "sleep",
      "arguments": "1000",
      "phaseReason": "text",
      "idleAllocatedGpus": 1,
      "totalPendingTimeSeconds": 60,
      "totalRunningTimeSeconds": 60,
      "category": "Train",
      "guaranteedRuntimeEndsAt": "2025-08-01T03:49:52.531Z",
      "aiApplicationId": "text",
      "aiApplicationName": "text",
      "sourceApi": "WorkloadsV2",
      "topology": {
        "name": "text",
        "preferredPlacement": "text",
        "requiredPlacement": "text"
      }
    }
  ]
}

Get a workload.

get

Retrieve workload data using a workloadId.

Authorizations
AuthorizationstringRequired

Bearer authentication

Path parameters
workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Responses
200

Executed successfully.

application/json
tenantIdinteger · int32Required

The id of the tenant.

Example: 1001
runningPodsinteger · int32RequiredExample: 1
phaseUpdatedAtstring · date-timeRequiredExample: 2022-06-08T11:28:24.131Z
k8sPhaseUpdatedAtstring · date-timeRequiredExample: 2022-06-08T11:28:24.131Z
updatedAtstring · date-timeRequiredExample: 2022-06-08T11:28:24.131Z
sourcestring · enumRequiredPossible values:
deletedAtstring · date-time · nullableRequiredExample: 2022-08-12T19:28:24.131Z
typestringRequiredExample: runai-job
namestringRequiredExample: very-important-job
idstring · uuidRequired
priorityinteger · int32 · nullableRequiredExample: 50
priorityClassNamestringRequiredExample: high-priority
submittedBystringOptionalExample: researcher@run.ai
clusterIdstring · uuidRequired

The id of the cluster.

Example: 71f69d83-ba66-4822-adf5-55ce55efd210
projectNamestringRequiredExample: proj-1
projectIdstringRequiredExample: 1
departmentNamestringRequiredExample: department-1
departmentIdstringRequiredExample: 1
namespacestringRequiredExample: runai-proj-1
createdAtstring · date-timeRequiredExample: 2022-01-01T03:49:52.531Z
phasestring · enumRequiredPossible values:
phaseMessagestringOptionalExample: Not enough resources in the requested nodepool
k8sPhasestringRequiredExample: Pending
requestedNodePoolsstring[]OptionalExample: default
currentNodePoolsstring[]OptionalExample: default
completedAtstring · date-time · nullableOptionalExample: 2022-01-01T03:49:52.531Z
imagesstring[]OptionalExample: alpine:latest
urlsstring[]Optional
distributedFrameworkstringOptionalExample: Pytorch
preemptibleboolean · nullableOptional
commandstringOptionalExample: sleep
argumentsstringOptionalExample: 1000
phaseReasonstring · enumOptionalPossible values:
idleGpusinteger · nullableOptionalDeprecated

deprecated. use idleAllocatedGpus instead

Example: 3
idleAllocatedGpusnumber · nullableOptional

sum of idle allocated gpus in the workload

Example: 1
totalPendingTimeSecondsinteger · nullableOptional

The total cumulative time, in seconds, that the workload has spent in the Pending phase since submission.

Example: 60
totalRunningTimeSecondsinteger · nullableOptional

The total cumulative time, in seconds, that the workload has spent in the Running phase since submission.

Example: 60
categorystringRequired

Category Description

Example: Train
guaranteedRuntimeEndsAtstring · date-time · nullableOptional

A timestamp indicating when the workload will reach its minimum guaranteed runtime, as defined by minGuaranteedRuntime. Until this time, the workload is considered non-preemptible and cannot be interrupted by higher-priority workloads.

Example: 2025-08-01T03:49:52.531Z
aiApplicationIdstringOptional
aiApplicationNamestringOptional
sourceApistring · enum · nullableOptionalPossible values:
get/api/v1/workloads/{workloadId}
GET /api/v1/workloads/{workloadId} HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "tenantId": 1001,
  "runningPods": 1,
  "phaseUpdatedAt": "2022-06-08T11:28:24.131Z",
  "k8sPhaseUpdatedAt": "2022-06-08T11:28:24.131Z",
  "updatedAt": "2022-06-08T11:28:24.131Z",
  "source": "CLI",
  "deletedAt": "2022-08-12T19:28:24.131Z",
  "type": "runai-job",
  "name": "very-important-job",
  "id": "123e4567-e89b-12d3-a456-426614174000",
  "priority": 50,
  "priorityClassName": "high-priority",
  "submittedBy": "researcher@run.ai",
  "clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
  "projectName": "proj-1",
  "projectId": "1",
  "departmentName": "department-1",
  "departmentId": "1",
  "namespace": "runai-proj-1",
  "createdAt": "2022-01-01T03:49:52.531Z",
  "workloadRequestedResources": {
    "gpuRequestType": "portion",
    "gpu": {
      "limit": 1.5,
      "request": 1
    },
    "gpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "cpu": {
      "limit": 1.5,
      "request": 1
    },
    "cpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "extendedResources": [
      {
        "resource": "hardware-vendor.example/foo",
        "quantity": 2,
        "exclude": false
      }
    ]
  },
  "podsRequestedResources": {
    "gpuRequestType": "portion",
    "gpu": {
      "limit": 1.5,
      "request": 1
    },
    "gpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "cpu": {
      "limit": 1.5,
      "request": 1
    },
    "cpuMemory": {
      "limit": "2G",
      "request": "200M"
    },
    "extendedResources": [
      {
        "resource": "hardware-vendor.example/foo",
        "quantity": 2,
        "exclude": false
      }
    ]
  },
  "allocatedResources": {
    "gpu": 1.5,
    "gpuMemory": "200Mi",
    "cpu": 0.5,
    "cpuMemory": "0B",
    "extendedResources": [
      {
        "resource": "hardware-vendor.example/foo",
        "quantity": 2,
        "exclude": false
      }
    ]
  },
  "actionsSupport": {
    "delete": true,
    "suspend": true
  },
  "phase": "Creating",
  "conditions": [
    {
      "type": "Ready",
      "status": "False",
      "message": "Resource validation failed: ...",
      "reason": "ErrorConfig",
      "lastTransitionTime": "2022-01-01T03:49:52.531Z"
    }
  ],
  "phaseMessage": "Not enough resources in the requested nodepool",
  "k8sPhase": "Pending",
  "requestedPods": {
    "number": 1,
    "min": 2,
    "max": 5,
    "parallelism": 3,
    "completions": 5
  },
  "requestedNodePools": [
    "default"
  ],
  "currentNodePools": [
    "default"
  ],
  "completedAt": "2022-01-01T03:49:52.531Z",
  "images": [
    "alpine:latest"
  ],
  "urls": [
    "text"
  ],
  "datasources": [
    {
      "type": "pvc",
      "name": "my-pvc-datasource-1",
      "id": "123e4567-e89b-12d3-a456-426614174000"
    }
  ],
  "environments": [
    {
      "connections": [
        {
          "name": "my-pytorch-env",
          "toolType": "pytorch",
          "connectionType": "ExternalUrl",
          "url": "http://wandb.com/yourproject",
          "authorizationType": "public",
          "authorizedUsers": [
            "user@company.ai",
            "another@company.ai"
          ],
          "authorizedGroups": [
            "group-a",
            "group-b"
          ],
          "containerPort": 8080
        }
      ],
      "name": "pytorch",
      "id": "123e4567-e89b-12d3-a456-426614174000",
      "replicaType": "Master"
    }
  ],
  "externalConnections": [
    {
      "name": "my-pytorch-env",
      "toolType": "pytorch",
      "connectionType": "ExternalUrl",
      "url": "http://wandb.com/yourproject",
      "authorizationType": "public",
      "authorizedUsers": [
        "user@company.ai",
        "another@company.ai"
      ],
      "authorizedGroups": [
        "group-a",
        "group-b"
      ],
      "containerPort": 8080
    }
  ],
  "distributedFramework": "Pytorch",
  "additionalFields": {
    "ANY_ADDITIONAL_PROPERTY": "anything"
  },
  "preemptible": true,
  "environmentVariables": {
    "ANY_ADDITIONAL_PROPERTY": "text"
  },
  "command": "sleep",
  "arguments": "1000",
  "phaseReason": "text",
  "idleAllocatedGpus": 1,
  "totalPendingTimeSeconds": 60,
  "totalRunningTimeSeconds": 60,
  "category": "Train",
  "guaranteedRuntimeEndsAt": "2025-08-01T03:49:52.531Z",
  "aiApplicationId": "text",
  "aiApplicationName": "text",
  "sourceApi": "WorkloadsV2",
  "topology": {
    "name": "text",
    "preferredPlacement": "text",
    "requiredPlacement": "text"
  },
  "pendingSchedulingMessages": [
    {
      "nodePool": "default",
      "phaseReason": "NonPreemptibleOverQuota",
      "reason": "Non-preemptible over quota",
      "orgType": "text",
      "userMessage": "You have reached the limit of non-preemptible resources"
    }
  ]
}

Count workloads.

get

Retrieve the number of workloads.

Authorizations
AuthorizationstringRequired

Bearer authentication

Query parameters
deletedbooleanOptional

Return only deleted resources when true.

filterBystring[]Optional

Filter results by a parameter. Use the format field-name operator value. Operators are == Equals, != Not equals, <= Less than or equal, >= Greater than or equal, =@ contains, !@ Does not contain, =^ Starts with and =$ Ends with. Dates are in ISO 8601 timestamp format and available for operators ==, !=, <= and >=.

Example: ["name!=some-workload-name","allocatedGPU>=2","createdAt>=2021-01-01T00:00:00Z"]
searchstringOptional

Filter results by a free text search.

Example: test project
Responses
200

Executed successfully.

application/json
countinteger · int64RequiredExample: 1
get/api/v1/workloads/count
GET /api/v1/workloads/count HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "count": 1
}

Get the workloads telemetry.

get

Retrieves workload data by telemetry type. Optionally filter by specific workload phases.

Telemetry Types:

  • WORKLOADS_COUNT: Count of workloads

  • GPU_ALLOCATION: GPU allocation metrics

  • PENDING_TIME_DISTRIBUTION: Distribution of workloads by current pending time (time since entering Pending phase).

    For this type, results are automatically grouped by 4 time buckets and any additional groupBy parameters.

Authorizations
AuthorizationstringRequired

Bearer authentication

Query parameters
clusterIdstring · uuidOptional

Filter using the Universally Unique Identifier (UUID) of the cluster.

Example: d73a738f-fab3-430a-8fa3-5241493d7128
nodepoolNamestringOptional

Filter using the nodepool.

Example: default
departmentIdstringOptional

Filter using the department id.

Example: 1
telemetryTypestring · enumRequired

Select a telemetry type.

Possible values:
Responses
200

Executed successfully.

typestringRequired

specifies what data returned

Example: ALLOCATION_RATIO
timestampstring · date-timeRequiredExample: 2023-06-06 12:09:18.211
get/api/v1/workloads/telemetry
GET /api/v1/workloads/telemetry?telemetryType=WORKLOADS_COUNT HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "type": "ALLOCATION_RATIO",
  "timestamp": "2023-06-06 12:09:18.211",
  "values": [
    {
      "value": "85",
      "groups": [
        {
          "key": "department",
          "value": "1",
          "name": "department-A"
        }
      ]
    }
  ]
}

Get workload metrics data.

get

Retrieves workloads data metrics from the metrics database. Use in reporting and analysis tools.

Authorizations
AuthorizationstringRequired

Bearer authentication

Path parameters
workloadIdstring · uuidRequired

The Universally Unique Identifier (UUID) of the workload.

Query parameters
startstring · date-timeRequired

Start date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-06T12:09:18.211Z
endstring · date-timeRequired

End date of time range to fetch data in ISO 8601 timestamp format.

Example: 2023-06-07T12:09:18.211Z
numberOfSamplesinteger · max: 1000Optional

The number of samples to take in the specified time range.

Default: 20Example: 20
Responses
200

Executed successfully.

get/api/v1/workloads/{workloadId}/metrics
GET /api/v1/workloads/{workloadId}/metrics?metricType=GPU_UTILIZATION&start=2023-06-06T12%3A09%3A18.211Z&end=2023-06-07T12%3A09%3A18.211Z HTTP/1.1
Host: app.run.ai
Authorization: Bearer YOUR_SECRET_TOKEN
Accept: */*

{
  "measurements": [
    {
      "type": "ALLOCATED_GPU",
      "labels": {
        "gpu": "0"
      },
      "values": [
        {
          "value": "85",
          "timestamp": "2024-12-20T14:30:00Z"
        },
        {
          "value": "92",
          "timestamp": "2024-12-20T14:30:30Z"
        },
        {
          "value": "78",
          "timestamp": "2024-12-20T14:37:00Z"
        }
      ]
    }
  ],
  "histogram": [
    {
      "type": "NIM_TIME_TO_FIRST_TOKEN_SECONDS",
      "values": [
        {
          "timestamp": "2024-12-20T14:30:00Z",
          "data": {
            "0.001": "45",
            "0.005": "178",
            "0.01": "312",
            "0.05": "624",
            "0.1": "812",
            "0.5": "980",
            "1.0": "1004",
            "2.0": "1015",
            "+Inf": "1024"
          }
        },
        {
          "timestamp": "2024-12-20T14:30:30Z",
          "data": {
            "0.001": "32",
            "0.005": "152",
            "0.01": "288",
            "0.05": "576",
            "0.1": "756",
            "0.5": "908",
            "1.0": "932",
            "2.0": "943",
            "+Inf": "960"
          }
        }
      ]
    },
    {
      "type": "NIM_TIME_TO_FIRST_TOKEN_SECONDS_PERCENTILES",
      "values": [
        {
          "timestamp": "2024-12-20T14:30:00Z",
          "data": {
            "p50": "0.0734",
            "p90": "0.1892",
            "p99": "0.2847"
          }
        },
        {
          "timestamp": "2024-12-20T14:30:30Z",
          "data": {
            "p50": "0.0823",
            "p90": "0.2134",
            "p99": "0.3156"
          }
        }
      ]
    }
  ]
}
PreviousPodsNextWorkloads V2

Last updated