From ce8f01a85f1feb3979621d4741e9fba53455662b Mon Sep 17 00:00:00 2001 From: psrok1 Date: Wed, 30 Sep 2026 17:28:02 +0200 Subject: [PATCH 01/21] Draft of REST API for karton.core.inspect (dashboard, MWDB status) and karton.core.main --- karton/gateway/rest/__init__.py | 0 karton/gateway/rest/models.py | 75 +++++++++ karton/gateway/rest/routes.py | 260 ++++++++++++++++++++++++++++++++ 3 files changed, 335 insertions(+) create mode 100644 karton/gateway/rest/__init__.py create mode 100644 karton/gateway/rest/models.py create mode 100644 karton/gateway/rest/routes.py diff --git a/karton/gateway/rest/__init__.py b/karton/gateway/rest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py new file mode 100644 index 00000000..36544f82 --- /dev/null +++ b/karton/gateway/rest/models.py @@ -0,0 +1,75 @@ +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from karton.core.task import TaskPriority, TaskState +from karton.gateway.models import ResourceUrl + + +class Bind(BaseModel): + model_config = ConfigDict(from_attributes=True) + + identity: str + info: str | None + version: str + persistent: bool + filters: list[dict[str, Any]] + service_version: str | None + is_async: bool + + +class ServiceInfo(BaseModel): + identity: str + karton_version: str | None + service_version: str | None + instance_id: str | None + active_replicas: int + + +class QueueView(BaseModel): + identity: str + info: str | None + version: str + persistent: bool + filters: list[dict[str, Any]] + service_version: str | None + is_async: bool + replicas: int + pending_task_uids: list[str] + crashed_task_uids: list[str] + + +class TaskView(BaseModel): + uid: str + root_uid: str + parent_uid: str | None + orig_uid: str | None + status: TaskState + priority: TaskPriority + last_update: float + headers: dict[str, Any] + headers_persistent: dict[str, Any] + payload: dict[str, Any] + payload_persistent: dict[str, Any] + error: list[str] | None + download_urls: list[ResourceUrl] = Field(default_factory=list) + + +class AnalysisView(BaseModel): + uid: str + queues: dict[str, list[TaskView]] + + +class ProducerOutput(BaseModel): + model_config = ConfigDict(from_attributes=True) + + identity: str + outputs: list[dict[str, Any]] + + +class RestartedTask(BaseModel): + uid: str + + +class APIError(BaseModel): + error: str diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py new file mode 100644 index 00000000..a5aad19a --- /dev/null +++ b/karton/gateway/rest/routes.py @@ -0,0 +1,260 @@ +import secrets + +from fastapi import APIRouter, Depends, Header, HTTPException, Response +from fastapi.responses import PlainTextResponse + +from karton.core.__version__ import __version__ +from karton.core.backend import KartonBind, KartonMetrics +from karton.core.task import Task, TaskState +from karton.gateway.backend import gateway_backend +from karton.gateway.config import gateway_config +from karton.gateway.rest.models import ( + AnalysisView, + Bind, + ProducerOutput, + QueueView, + RestartedTask, + ServiceInfo, + TaskView, +) +from karton.gateway.task import generate_resource_download_urls + + +async def verify_api_key(authorization: str | None = Header(default=None)) -> None: + # TODO: This is just a functional placeholder + # Final solution won't use Authorization: Bearer for passing the password + password = gateway_config.password + if password is None: + return + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Missing or invalid bearer token") + if not secrets.compare_digest(authorization[len("Bearer ") :], password): + raise HTTPException(status_code=401, detail="Invalid API key") + + +router = APIRouter(prefix="/api/v1", dependencies=[Depends(verify_api_key)]) +varz_router = APIRouter(dependencies=[Depends(verify_api_key)]) + + +def _allowed_buckets() -> list[str]: + return [gateway_backend.default_bucket_name, *gateway_config.allowed_extra_buckets] + + +async def _task_to_view(task: Task) -> TaskView: + download_urls = await generate_resource_download_urls(task, _allowed_buckets()) + return TaskView(**task.to_dict(), download_urls=download_urls) + + +@router.get("/binds") +async def get_binds() -> list[Bind]: + return [Bind.model_validate(bind) for bind in await gateway_backend.get_binds()] + + +@router.get("/binds/{identity}") +async def get_bind_details(identity: str) -> Bind: + for bind in await gateway_backend.get_binds(): + if bind.identity == identity: + return Bind.model_validate(bind) + raise HTTPException(status_code=404, detail="Bind doesn't exist") + + +@router.delete("/binds/{identity}") +async def delete_bind(identity: str) -> Response: + binds = {bind.identity for bind in await gateway_backend.get_binds()} + if identity not in binds: + raise HTTPException(status_code=404, detail="Bind doesn't exist") + replicas = await gateway_backend.get_online_identities() + if replicas.get(identity): + raise HTTPException( + status_code=409, + detail=( + "Bind has active replicas that need to be downscaled " + "before it can be deleted" + ), + ) + # Replace the bind with non-persistent bind with empty filters. + # This is "mark for deletion" for Karton System GC + tombstone_bind = KartonBind( + identity=identity, + info=None, + version=__version__, + persistent=False, + filters=[], + service_version=None, + is_async=False, + ) + await gateway_backend.register_bind(tombstone_bind, bind_backend=False) + return Response(status_code=204) + + +@router.get("/services") +async def get_services() -> list[ServiceInfo]: + services = await gateway_backend.get_online_services(_legacy=False) + counts: dict[tuple[str, str | None, str | None], int] = {} + for service in services: + key = (service.identity, service.karton_version, service.service_version) + counts[key] = counts.get(key, 0) + 1 + return [ + ServiceInfo( + identity=identity, + karton_version=karton_version, + service_version=service_version, + instance_id=None, + active_replicas=replicas, + ) + for (identity, karton_version, service_version), replicas in counts.items() + ] + + +async def _build_queues() -> dict[str, QueueView]: + binds = {bind.identity: bind for bind in await gateway_backend.get_binds()} + replicas = await gateway_backend.get_online_identities() + tasks = await gateway_backend.get_all_tasks(parse_resources=False) + pending = [ + task + for task in tasks + if task.status not in (TaskState.FINISHED, TaskState.CRASHED) + ] + crashed = [task for task in tasks if task.status == TaskState.CRASHED] + + queues: dict[str, QueueView] = {} + for identity, bind in binds.items(): + queue_pending = sorted( + (task for task in pending if task.headers.get("receiver") == identity), + key=lambda task: task.last_update, + reverse=True, + ) + queue_crashed = sorted( + (task for task in crashed if task.headers.get("receiver") == identity), + key=lambda task: task.last_update, + reverse=True, + ) + queues[identity] = QueueView( + identity=bind.identity, + info=bind.info, + version=bind.version, + persistent=bind.persistent, + filters=bind.filters, + service_version=bind.service_version, + is_async=bind.is_async, + replicas=len(replicas.get(identity, [])), + pending_task_uids=[task.uid for task in queue_pending], + crashed_task_uids=[task.uid for task in queue_crashed], + ) + return queues + + +@router.get("/queues") +async def get_queues() -> dict[str, QueueView]: + return await _build_queues() + + +@router.get("/queues/{identity}") +async def get_queue_details(identity: str) -> QueueView: + queues = await _build_queues() + if identity not in queues: + raise HTTPException(status_code=404, detail="Queue doesn't exist") + return queues[identity] + + +@router.get("/tasks/{uid}") +async def get_task_details(uid: str) -> TaskView: + task = await gateway_backend.get_task(uid) + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") + return await _task_to_view(task) + + +@router.get("/analyses/{uid}") +async def get_analysis_details(uid: str) -> AnalysisView: + binds = {bind.identity for bind in await gateway_backend.get_binds()} + queues: dict[str, list[TaskView]] = {} + async for task in gateway_backend.iter_task_tree(uid): + if task.status in (TaskState.FINISHED, TaskState.CRASHED): + continue + receiver = task.headers.get("receiver") + if receiver is None or receiver not in binds: + continue + queues.setdefault(receiver, []).append(await _task_to_view(task)) + return AnalysisView(uid=uid, queues=queues) + + +@router.post("/tasks/{uid}/restart") +async def restart_task(uid: str) -> RestartedTask: + task = await gateway_backend.get_task(uid) + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") + new_task = await gateway_backend.restart_task(task) + return RestartedTask(uid=new_task.uid) + + +@router.post("/tasks/{uid}/cancel") +async def cancel_task(uid: str) -> Response: + task = await gateway_backend.get_task(uid) + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") + await gateway_backend.set_task_status(task, TaskState.FINISHED) + return Response(status_code=204) + + +@router.delete("/tasks/{uid}") +async def delete_task(uid: str) -> Response: + task = await gateway_backend.get_task(uid) + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") + await gateway_backend.delete_task(task) + return Response(status_code=204) + + +@router.get("/outputs") +async def get_outputs() -> list[ProducerOutput]: + return [ + ProducerOutput.model_validate(output) + for output in await gateway_backend.get_outputs() + ] + + +@varz_router.get("/varz") +async def varz() -> PlainTextResponse: + identities = await gateway_backend.get_online_identities() + tasks = await gateway_backend.get_all_tasks(parse_resources=False) + + lines = [ + "# HELP karton_replicas Number of online replicas per identity", + "# TYPE karton_replicas gauge", + ] + for identity, services in identities.items(): + lines.append(f'karton_replicas{{identity="{identity}"}} {len(services)}') + + lines += [ + "# HELP karton_tasks Number of tasks per queue, priority and status", + "# TYPE karton_tasks gauge", + ] + task_counts: dict[tuple[str, str, str], int] = {} + for task in tasks: + receiver = task.headers.get("receiver", "") + key = (receiver, task.priority.value, task.status.value) + task_counts[key] = task_counts.get(key, 0) + 1 + for (queue, priority, status), count in task_counts.items(): + lines.append( + f'karton_tasks{{queue="{queue}",priority="{priority}",' + f'status="{status}"}} {count}' + ) + + lines += [ + "# HELP karton_metrics Task counters per identity", + "# TYPE karton_metrics gauge", + ] + for metric in KartonMetrics: + metric_label = metric.value.split(".")[-1] + values = await gateway_backend.get_metrics(metric) + for identity, count in values.items(): + lines.append( + f'karton_metrics{{metric="{metric_label}",' + f'identity="{identity}"}} {count}' + ) + + return PlainTextResponse( + "\n".join(lines) + "\n", + media_type="text/plain; version=0.0.4; charset=utf-8", + ) From 10c7ea1338030882c3545bdcf539ec9ce4ad18a8 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Fri, 9 Oct 2026 19:05:50 +0200 Subject: [PATCH 02/21] Wire gateway.rest with new authorization --- karton/core/auth/models.py | 26 ++++++++++++++ karton/gateway/app.py | 4 +++ karton/gateway/rest/auth.py | 47 +++++++++++++++++++++++++ karton/gateway/rest/models.py | 2 +- karton/gateway/rest/routes.py | 66 ++++++++++++++--------------------- 5 files changed, 104 insertions(+), 41 deletions(-) create mode 100644 karton/gateway/rest/auth.py diff --git a/karton/core/auth/models.py b/karton/core/auth/models.py index 904d9d54..0063ad14 100644 --- a/karton/core/auth/models.py +++ b/karton/core/auth/models.py @@ -85,12 +85,38 @@ class EnableDebugOperation(Operation): operation: Literal["enable_debug"] = "enable_debug" +class InspectKartonOperation(Operation): + operation: Literal["inspect_karton"] = "inspect_karton" + + +class RestartTaskOperation(Operation): + operation: Literal["restart_task"] = "restart_task" + + +class CancelTaskOperation(Operation): + operation: Literal["cancel_task"] = "cancel_task" + + +class RemoveBindOperation(Operation): + operation: Literal["remove_bind"] = "remove_bind" + + +class GetMetricsOperation(Operation): + operation: Literal["get_metrics"] = "get_metrics" + + + type AllowedOperation = ( RegisterBindOperation | ConsumeTaskOperation | ProduceTaskOperation | ConsumeLogOperation | EnableDebugOperation + | InspectKartonOperation + | RestartTaskOperation + | CancelTaskOperation + | RemoveBindOperation + | GetMetricsOperation ) diff --git a/karton/gateway/app.py b/karton/gateway/app.py index dc7f0b46..d15c1798 100644 --- a/karton/gateway/app.py +++ b/karton/gateway/app.py @@ -7,6 +7,7 @@ from .errors import InternalError, KartonGatewayError, ShutdownError from .logger import set_connection_id, setup_logger from .messages import close_websocket, send_error +from .rest.routes import rest_api_router, varz_router from .session import ClientSession @@ -23,6 +24,9 @@ async def lifespan(app: FastAPI): logger = logging.getLogger(__name__) app = FastAPI(lifespan=lifespan) +app.include_router(rest_api_router) +app.include_router(varz_router) + async def try_send_error(websocket: WebSocket, error: KartonGatewayError): try: diff --git a/karton/gateway/rest/auth.py b/karton/gateway/rest/auth.py new file mode 100644 index 00000000..3c7b9d5b --- /dev/null +++ b/karton/gateway/rest/auth.py @@ -0,0 +1,47 @@ +from fastapi import Header, HTTPException + +from karton.core.auth.models import ( + AllowedOperation, + CancelTaskOperation, + GetMetricsOperation, + InspectKartonOperation, + RemoveBindOperation, + RestartTaskOperation, +) + +from ..config import gateway_config +from ..session import parse_auth_tokens + + +class AuthorizationCheck: + def __init__(self, operation: AllowedOperation) -> None: + self.operation = operation + + def __call__(self, authorization: str | None = Header(default=None)) -> None: + if not gateway_config.auth_required: + return + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException( + status_code=401, detail="Missing or invalid bearer token" + ) + parts = authorization.split(" ") + if len(parts) > 2: + raise HTTPException( + status_code=401, detail="Missing or invalid bearer token" + ) + tokens = parts[1].split(",") + verified_grants = parse_auth_tokens(tokens) + for grant in verified_grants: + for op in grant.claims.allowed_operations: + if op.covers(self.operation): + return + raise HTTPException( + status_code=403, detail="Client is not authorized to perform this operation" + ) + + +CanInspectKarton = AuthorizationCheck(InspectKartonOperation()) +CanRestartTask = AuthorizationCheck(RestartTaskOperation()) +CanCancelTask = AuthorizationCheck(CancelTaskOperation()) +CanRemoveBind = AuthorizationCheck(RemoveBindOperation()) +CanGetMetrics = AuthorizationCheck(GetMetricsOperation()) diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py index 36544f82..c8801a07 100644 --- a/karton/gateway/rest/models.py +++ b/karton/gateway/rest/models.py @@ -2,8 +2,8 @@ from pydantic import BaseModel, ConfigDict, Field +from karton.core.gateway_protocol import ResourceUrl from karton.core.task import TaskPriority, TaskState -from karton.gateway.models import ResourceUrl class Bind(BaseModel): diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py index a5aad19a..f14c1a22 100644 --- a/karton/gateway/rest/routes.py +++ b/karton/gateway/rest/routes.py @@ -1,6 +1,4 @@ -import secrets - -from fastapi import APIRouter, Depends, Header, HTTPException, Response +from fastapi import APIRouter, Depends, HTTPException, Response from fastapi.responses import PlainTextResponse from karton.core.__version__ import __version__ @@ -8,6 +6,13 @@ from karton.core.task import Task, TaskState from karton.gateway.backend import gateway_backend from karton.gateway.config import gateway_config +from karton.gateway.rest.auth import ( + CanCancelTask, + CanGetMetrics, + CanInspectKarton, + CanRemoveBind, + CanRestartTask, +) from karton.gateway.rest.models import ( AnalysisView, Bind, @@ -19,25 +24,15 @@ ) from karton.gateway.task import generate_resource_download_urls - -async def verify_api_key(authorization: str | None = Header(default=None)) -> None: - # TODO: This is just a functional placeholder - # Final solution won't use Authorization: Bearer for passing the password - password = gateway_config.password - if password is None: - return - if not authorization or not authorization.startswith("Bearer "): - raise HTTPException(status_code=401, detail="Missing or invalid bearer token") - if not secrets.compare_digest(authorization[len("Bearer ") :], password): - raise HTTPException(status_code=401, detail="Invalid API key") - - -router = APIRouter(prefix="/api/v1", dependencies=[Depends(verify_api_key)]) -varz_router = APIRouter(dependencies=[Depends(verify_api_key)]) +rest_api_router = APIRouter(prefix="/api/v1") +varz_router = APIRouter() def _allowed_buckets() -> list[str]: - return [gateway_backend.default_bucket_name, *gateway_config.allowed_extra_buckets] + return [ + gateway_backend.default_bucket_name, + *gateway_config.allowed_foreign_buckets, + ] async def _task_to_view(task: Task) -> TaskView: @@ -45,12 +40,12 @@ async def _task_to_view(task: Task) -> TaskView: return TaskView(**task.to_dict(), download_urls=download_urls) -@router.get("/binds") +@rest_api_router.get("/binds", dependencies=[Depends(CanInspectKarton)]) async def get_binds() -> list[Bind]: return [Bind.model_validate(bind) for bind in await gateway_backend.get_binds()] -@router.get("/binds/{identity}") +@rest_api_router.get("/binds/{identity}", dependencies=[Depends(CanInspectKarton)]) async def get_bind_details(identity: str) -> Bind: for bind in await gateway_backend.get_binds(): if bind.identity == identity: @@ -58,7 +53,7 @@ async def get_bind_details(identity: str) -> Bind: raise HTTPException(status_code=404, detail="Bind doesn't exist") -@router.delete("/binds/{identity}") +@rest_api_router.delete("/binds/{identity}", dependencies=[Depends(CanRemoveBind)]) async def delete_bind(identity: str) -> Response: binds = {bind.identity for bind in await gateway_backend.get_binds()} if identity not in binds: @@ -87,7 +82,7 @@ async def delete_bind(identity: str) -> Response: return Response(status_code=204) -@router.get("/services") +@rest_api_router.get("/services", dependencies=[Depends(CanInspectKarton)]) async def get_services() -> list[ServiceInfo]: services = await gateway_backend.get_online_services(_legacy=False) counts: dict[tuple[str, str | None, str | None], int] = {} @@ -144,12 +139,12 @@ async def _build_queues() -> dict[str, QueueView]: return queues -@router.get("/queues") +@rest_api_router.get("/queues", dependencies=[Depends(CanInspectKarton)]) async def get_queues() -> dict[str, QueueView]: return await _build_queues() -@router.get("/queues/{identity}") +@rest_api_router.get("/queues/{identity}", dependencies=[Depends(CanInspectKarton)]) async def get_queue_details(identity: str) -> QueueView: queues = await _build_queues() if identity not in queues: @@ -157,7 +152,7 @@ async def get_queue_details(identity: str) -> QueueView: return queues[identity] -@router.get("/tasks/{uid}") +@rest_api_router.get("/tasks/{uid}", dependencies=[Depends(CanInspectKarton)]) async def get_task_details(uid: str) -> TaskView: task = await gateway_backend.get_task(uid) if task is None: @@ -165,7 +160,7 @@ async def get_task_details(uid: str) -> TaskView: return await _task_to_view(task) -@router.get("/analyses/{uid}") +@rest_api_router.get("/analyses/{uid}", dependencies=[Depends(CanInspectKarton)]) async def get_analysis_details(uid: str) -> AnalysisView: binds = {bind.identity for bind in await gateway_backend.get_binds()} queues: dict[str, list[TaskView]] = {} @@ -179,7 +174,7 @@ async def get_analysis_details(uid: str) -> AnalysisView: return AnalysisView(uid=uid, queues=queues) -@router.post("/tasks/{uid}/restart") +@rest_api_router.post("/tasks/{uid}/restart", dependencies=[Depends(CanRestartTask)]) async def restart_task(uid: str) -> RestartedTask: task = await gateway_backend.get_task(uid) if task is None: @@ -188,7 +183,7 @@ async def restart_task(uid: str) -> RestartedTask: return RestartedTask(uid=new_task.uid) -@router.post("/tasks/{uid}/cancel") +@rest_api_router.post("/tasks/{uid}/cancel", dependencies=[Depends(CanCancelTask)]) async def cancel_task(uid: str) -> Response: task = await gateway_backend.get_task(uid) if task is None: @@ -197,16 +192,7 @@ async def cancel_task(uid: str) -> Response: return Response(status_code=204) -@router.delete("/tasks/{uid}") -async def delete_task(uid: str) -> Response: - task = await gateway_backend.get_task(uid) - if task is None: - raise HTTPException(status_code=404, detail="Task doesn't exist") - await gateway_backend.delete_task(task) - return Response(status_code=204) - - -@router.get("/outputs") +@rest_api_router.get("/outputs", dependencies=[Depends(CanInspectKarton)]) async def get_outputs() -> list[ProducerOutput]: return [ ProducerOutput.model_validate(output) @@ -214,7 +200,7 @@ async def get_outputs() -> list[ProducerOutput]: ] -@varz_router.get("/varz") +@varz_router.get("/varz", dependencies=[Depends(CanGetMetrics)]) async def varz() -> PlainTextResponse: identities = await gateway_backend.get_online_identities() tasks = await gateway_backend.get_all_tasks(parse_resources=False) From 27fe6310950a4bd4576001946448f15c22fcdead Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 12 Oct 2026 17:39:14 +0200 Subject: [PATCH 03/21] Implement backend methods needed by Gateway REST API --- karton/core/asyncio/backend/direct.py | 311 ++++++++++++++++++++++++++ karton/core/backend/direct.py | 2 +- karton/core/utils.py | 18 +- 3 files changed, 329 insertions(+), 2 deletions(-) diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index b91ac60d..a012fe39 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -2,6 +2,7 @@ import logging import os import time +from collections import defaultdict from collections.abc import AsyncIterator from typing import IO, Any, cast @@ -16,16 +17,20 @@ from karton.core import Config, Task from karton.core.asyncio.resource import LocalResource, RemoteResource, ResourceData from karton.core.backend import KartonBind, KartonMetrics, KartonServiceInfo +from karton.core.backend.base import KartonExternalServiceInfo from karton.core.backend.direct import ( KARTON_BINDS_HSET, + KARTON_OUTPUTS_NAMESPACE, KARTON_SERVICES_NAMESPACE, KARTON_TASK_NAMESPACE, KARTON_TASKS_QUEUE, KartonBackendBase, + KartonOutputs, ) from karton.core.exceptions import BindExpiredError, HeartbeatExpiredError from karton.core.resource import LocalResource as SyncLocalResource from karton.core.task import TaskState +from karton.core.utils import achunks_iter from .base import KartonAsyncBackendProtocol @@ -178,6 +183,117 @@ def unserialize_resource(self, resource_spec: ResourceData) -> RemoteResource: """ return RemoteResource.from_dict(resource_spec, backend=self) + async def _iter_tasks( + self, + task_keys: AsyncIterator[str], + chunk_size: int = 1000, + parse_resources: bool = True, + ) -> AsyncIterator[Task]: + async for chunk in achunks_iter(task_keys, chunk_size): + for task_data in await self.redis.mget(chunk): + if task_data is None: + continue + yield Task.unserialize( + task_data, + parse_resources=parse_resources, + resource_unserializer=self.unserialize_resource, + ) + + def iter_all_tasks( + self, chunk_size: int = 1000, parse_resources: bool = True + ) -> AsyncIterator[Task]: + """ + Iterates all tasks registered in Redis + :param chunk_size: Size of chunks passed to the Redis SCAN and MGET command + :param parse_resources: If set to False, resources are not parsed. + It speeds up deserialization. Read :py:meth:`Task.unserialize` documentation + to learn more. + :return: Iterator with Task objects + """ + task_keys = self.redis.scan_iter( + match=f"{KARTON_TASK_NAMESPACE}:*", count=chunk_size + ) + return self._iter_tasks( + task_keys, chunk_size=chunk_size, parse_resources=parse_resources + ) + + async def get_all_tasks( + self, chunk_size: int = 1000, parse_resources: bool = True + ) -> list[Task]: + """ + Get all tasks registered in Redis + + .. warning:: + This method loads all tasks into memory. + It's recommended to use :py:meth:`iter_all_tasks` instead. + + :param chunk_size: Size of chunks passed to the Redis MGET command + :param parse_resources: If set to False, resources are not parsed. + It speeds up deserialization. Read :py:meth:`Task.unserialize` documentation + to learn more. + :return: List with Task objects + """ + return [ + task + async for task in self.iter_all_tasks( + chunk_size=chunk_size, parse_resources=parse_resources + ) + ] + + async def _iter_legacy_task_tree( + self, root_uid: str, chunk_size: int = 1000, parse_resources: bool = True + ) -> AsyncIterator[Task]: + """ + Processes tasks made by <5.4.0 (unrouted from <5.4.0 producers or existing + before upgrade) + + Internal function for use by iter_task_tree. + """ + # Iterate over all karton tasks that do not match the new task id format + legacy_task_keys = self.redis.scan_iter( + match=f"{KARTON_TASK_NAMESPACE}:[^{{]*", count=chunk_size + ) + async for chunk in achunks_iter(legacy_task_keys, chunk_size): + for task_data in await self.redis.mget(chunk): + if task_data is None: + continue + task = Task.unserialize( + task_data, + parse_resources=parse_resources, + resource_unserializer=self.unserialize_resource, + ) + if task.root_uid == root_uid: + yield task + + async def iter_task_tree( + self, root_uid: str, chunk_size: int = 1000, parse_resources: bool = True + ) -> AsyncIterator[Task]: + """ + Iterates all tasks that belong to the same analysis task tree + and have the same root_uid + + :param root_uid: Root identifier of task tree + :param chunk_size: Size of chunks passed to the Redis SCAN and MGET command + :param parse_resources: If set to False, resources are not parsed. + It speeds up deserialization. Read :py:meth:`Task.unserialize` documentation + to learn more. + :return: Iterator with task objects + """ + # Process <5.4.0 tasks (unrouted from <5.4.0 producers + # or existing before upgrade) + async for task in self._iter_legacy_task_tree( + root_uid, chunk_size=chunk_size, parse_resources=parse_resources + ): + yield task + # Process >=5.4.0 tasks + task_keys = self.redis.scan_iter( + match=f"{KARTON_TASK_NAMESPACE}:{{{root_uid}}}:*", count=chunk_size + ) + async for task in self._iter_tasks( + task_keys, chunk_size=chunk_size, parse_resources=parse_resources + ): + yield task + async def declare_task(self, task: Task) -> None: # Ensure all local resources have good buckets for resource in task.iterate_resources(): @@ -240,6 +356,117 @@ async def register_bind( else: return None + async def _get_online_gateway_services( + self, + ) -> AsyncIterator[KartonExternalServiceInfo]: + async for service_keys in achunks_iter( + self.redis.scan_iter( + match=f"{KARTON_SERVICES_NAMESPACE}:*", + count=100, + ), + size=100, + ): + # KartonServiceInfo is returned per connection + # so returned services can be duplicated + for client_name in cast( + list[str | None], await self.redis.mget(*service_keys) + ): + if not client_name: + continue + try: + yield KartonExternalServiceInfo.parse_client_name(client_name) + except Exception: + logger.exception( + "Fatal error while parsing client name: %s", client_name + ) + continue + + async def _get_online_direct_services( + self, + ) -> AsyncIterator[KartonExternalServiceInfo]: + for client in await self.redis.client_list(): + name = client["name"] + try: + service_info = KartonExternalServiceInfo.parse_client_name( + name, redis_client_info=client + ) + yield service_info + except Exception: + logger.exception("Fatal error while parsing client name: %s", name) + continue + + async def _get_online_services(self) -> AsyncIterator[KartonExternalServiceInfo]: + # Services having instance_id should be returned only once per instance_id + # Services without instance_id are counted by amount of Redis connections + # so they should be returned as many times as they appear in the output + services: defaultdict[KartonExternalServiceInfo, int] = defaultdict(int) + async for service in self._get_online_direct_services(): + services[service] += 1 + async for service in self._get_online_gateway_services(): + services[service] += 1 + + for service in services.keys(): + if service.instance_id is None: + for _ in range(services[service]): + yield service + else: + yield service + + async def get_online_services( + self, _legacy=True + ) -> list[KartonExternalServiceInfo]: + """ + Gets all online services providing extended service information. + + This method stays compatible with <=5.10.0 and doesn't return information + about legacy consumers (without with_service_info) flag. It also nullifies + the instance_id to aggregate KartonServiceInfo objects by equality for + counting the number of instances. New code should not rely on that behavior. + + This method will return complete information about services in >=6.0.0. + You can turn on the new behavior by enabling _legacy=True, although this + argument will be removed as well in future major version. + + .. versionadded:: 5.1.0 + + :return: List of KartonServiceInfo objects + """ + if not _legacy: + return [el async for el in self._get_online_services()] + + # Legacy users of this method doesn't expect two things: + # - that karton_version can be None in case of pre-5.10.0 consumers + # - that KartonServiceInfo are not equal per identity+version + # but per instance (e.g. karton-dashboard) + # + # That loop removes instance_id and filters out services without + # karton_version + return [ + KartonExternalServiceInfo( + identity=service.identity, + karton_version=service.karton_version, + service_version=service.service_version, + instance_id=None, + redis_client_info=service.redis_client_info, + ) + async for service in self._get_online_services() + if service.karton_version is not None + ] + + async def get_online_identities(self) -> dict[str, list[KartonExternalServiceInfo]]: + """ + Returns a dictionary for all online identities with list of + KartonServiceInfo of each instance + + :return: Dictionary {identity: [list of KartonServiceInfo objects]} + """ + identities: defaultdict[str, list[KartonExternalServiceInfo]] = defaultdict( + list + ) + async for service in self._get_online_services(): + identities[service.identity].append(service) + return dict(identities) + async def get_bind(self, identity: str) -> KartonBind | None: """ Get bind object for given identity @@ -252,9 +479,37 @@ async def get_bind(self, identity: str) -> KartonBind | None: return None return self.unserialize_bind(identity, bind_data) + async def get_binds(self) -> list[KartonBind]: + """ + Get all binds registered in Redis + + :return: List of KartonBind objects for subsequent identities + """ + return [ + self.unserialize_bind(identity, raw_bind) + for identity, raw_bind in cast( + dict[str, str], await self.redis.hgetall(KARTON_BINDS_HSET) + ).items() + ] + async def produce_unrouted_task(self, task: Task) -> None: await self.redis.rpush(KARTON_TASKS_QUEUE, task.uid) + async def produce_routed_task( + self, identity: str, task: Task, pipe: Pipeline | None = None + ) -> None: + """ + Add given task to routed task queue of given identity + + Task must be registered using :py:meth:`register_task` + + :param identity: Karton service identity + :param task: Task object + :param pipe: Optional pipeline object if operation is a part of pipeline + """ + rs = pipe or self.redis + await rs.rpush(self.get_queue_name(identity, task.priority), task.uid) + async def consume_queues( self, queues: str | list[str], timeout: int = 0 ) -> tuple[str, str] | None: @@ -322,6 +577,31 @@ async def consume_routed_task( _queue, data = item return await self.get_task(data) + async def restart_task(self, task: Task) -> Task: + """ + Requeues consumed task back to the consumer queue. + + New task is created with new uid and can be consumed by any active replica. + + Original task is marked as finished. + + :param task: Task to be restarted + :return: Restarted task object + """ + new_task = task.fork_task() + # Preserve orig_uid to point at unrouted task + new_task.orig_uid = task.orig_uid + new_task.status = TaskState.SPAWNED + + async with self.redis.pipeline(transaction=True) as pipe: + await self.register_task(new_task, pipe=pipe) + await self.produce_routed_task( + new_task.headers["receiver"], new_task, pipe=pipe + ) + await self.set_task_status(task, status=TaskState.FINISHED, pipe=pipe) + await pipe.execute() + return new_task + async def increment_metrics( self, metric: KartonMetrics, identity: str, pipe: Pipeline | None = None ) -> None: @@ -334,6 +614,19 @@ async def increment_metrics( rs = pipe or self.redis await rs.hincrby(metric.value, identity, 1) + async def get_metrics(self, metric: KartonMetrics) -> dict[str, int]: + """ + Get a {karton-identity: current-number-of-tasks} mapping for a given metric. + + :param metric: Operation metric type + """ + return { + k: int(v) + for k, v in cast( + dict[str, str], await self.redis.hgetall(metric.value) + ).items() + } + async def upload_resource( self, resource: LocalResource, @@ -414,6 +707,24 @@ async def consume_log( # other action was requested and needs to be handled yield None + async def get_outputs(self) -> list[KartonOutputs]: + """ + Get a list of the output types for each karton. + + :return: List of KartonOutputs + """ + + output_keys = cast( + list[str], await self.redis.keys(f"{KARTON_OUTPUTS_NAMESPACE}:*") + ) + return [ + self.unserialize_output( + identity.split(":")[1], + cast(set[str], await self.redis.smembers(identity)), + ) + for identity in output_keys + ] + @staticmethod def redis_heartbeat_key(service_info: KartonServiceInfo, connection_id: str) -> str: return ( diff --git a/karton/core/backend/direct.py b/karton/core/backend/direct.py index b38d1893..750c2f60 100644 --- a/karton/core/backend/direct.py +++ b/karton/core/backend/direct.py @@ -838,7 +838,7 @@ def restart_task(self, task: Task) -> Task: new_task.orig_uid = task.orig_uid new_task.status = TaskState.SPAWNED - p = self.make_pipeline() + p = self.make_pipeline(transaction=True) self.register_task(new_task, pipe=p) self.produce_routed_task(new_task.headers["receiver"], new_task, pipe=p) self.set_task_status(task, status=TaskState.FINISHED, pipe=p) diff --git a/karton/core/utils.py b/karton/core/utils.py index 16d0a1de..ed7988a2 100644 --- a/karton/core/utils.py +++ b/karton/core/utils.py @@ -1,7 +1,7 @@ import functools import itertools import signal -from collections.abc import Callable, Iterator, Sequence +from collections.abc import AsyncIterator, Callable, Iterator, Sequence from contextlib import contextmanager from typing import Any @@ -22,6 +22,22 @@ def chunks_iter[T](seq: Iterator[T], size: int) -> Iterator[Sequence[T]]: yield elements +async def achunks_iter[T]( + seq: AsyncIterator[T], size: int +) -> AsyncIterator[Sequence[T]]: + it = aiter(seq) + while True: + chunk: list[T] = [] + try: + while len(chunk) < size: + chunk.append(await anext(it)) + except StopAsyncIteration: + pass + if not chunk: + return + yield chunk + + def recursive_iter(obj: Any) -> Iterator[Any]: """ Yields all leaf values recursively from nested list/dict structures From 9e1be83c022514d97b297c0b6d02c204bd117a3c Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 12 Oct 2026 18:10:36 +0200 Subject: [PATCH 04/21] Don't use get_all_tasks for evaluation, use iter_all_tasks --- karton/core/asyncio/backend/direct.py | 23 ---------------- karton/gateway/rest/routes.py | 38 ++++++++++++++++----------- 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index a012fe39..d91fdaa4 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -217,29 +217,6 @@ def iter_all_tasks( task_keys, chunk_size=chunk_size, parse_resources=parse_resources ) - async def get_all_tasks( - self, chunk_size: int = 1000, parse_resources: bool = True - ) -> list[Task]: - """ - Get all tasks registered in Redis - - .. warning:: - This method loads all tasks into memory. - It's recommended to use :py:meth:`iter_all_tasks` instead. - - :param chunk_size: Size of chunks passed to the Redis MGET command - :param parse_resources: If set to False, resources are not parsed. - It speeds up deserialization. Read :py:meth:`Task.unserialize` documentation - to learn more. - :return: List with Task objects - """ - return [ - task - async for task in self.iter_all_tasks( - chunk_size=chunk_size, parse_resources=parse_resources - ) - ] - async def _iter_legacy_task_tree( self, root_uid: str, chunk_size: int = 1000, parse_resources: bool = True ) -> AsyncIterator[Task]: diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py index f14c1a22..4bb51688 100644 --- a/karton/gateway/rest/routes.py +++ b/karton/gateway/rest/routes.py @@ -1,3 +1,5 @@ +from collections import defaultdict + from fastapi import APIRouter, Depends, HTTPException, Response from fastapi.responses import PlainTextResponse @@ -104,24 +106,29 @@ async def get_services() -> list[ServiceInfo]: async def _build_queues() -> dict[str, QueueView]: binds = {bind.identity: bind for bind in await gateway_backend.get_binds()} replicas = await gateway_backend.get_online_identities() - tasks = await gateway_backend.get_all_tasks(parse_resources=False) - pending = [ - task - for task in tasks - if task.status not in (TaskState.FINISHED, TaskState.CRASHED) - ] - crashed = [task for task in tasks if task.status == TaskState.CRASHED] + + pending_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) + crashed_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) + + async for task in gateway_backend.iter_all_tasks(parse_resources=False): + receiver = task.headers.get("receiver") + if receiver is None or receiver not in binds: + continue + if task.status == TaskState.CRASHED: + crashed_uids[receiver].append((task.last_update, task.uid)) + elif task.status not in (TaskState.FINISHED, TaskState.CRASHED): + pending_uids[receiver].append((task.last_update, task.uid)) queues: dict[str, QueueView] = {} for identity, bind in binds.items(): queue_pending = sorted( - (task for task in pending if task.headers.get("receiver") == identity), - key=lambda task: task.last_update, + pending_uids.get(identity, []), + key=lambda item: item[0], reverse=True, ) queue_crashed = sorted( - (task for task in crashed if task.headers.get("receiver") == identity), - key=lambda task: task.last_update, + crashed_uids.get(identity, []), + key=lambda item: item[0], reverse=True, ) queues[identity] = QueueView( @@ -133,8 +140,8 @@ async def _build_queues() -> dict[str, QueueView]: service_version=bind.service_version, is_async=bind.is_async, replicas=len(replicas.get(identity, [])), - pending_task_uids=[task.uid for task in queue_pending], - crashed_task_uids=[task.uid for task in queue_crashed], + pending_task_uids=[uid for _, uid in queue_pending], + crashed_task_uids=[uid for _, uid in queue_crashed], ) return queues @@ -203,7 +210,6 @@ async def get_outputs() -> list[ProducerOutput]: @varz_router.get("/varz", dependencies=[Depends(CanGetMetrics)]) async def varz() -> PlainTextResponse: identities = await gateway_backend.get_online_identities() - tasks = await gateway_backend.get_all_tasks(parse_resources=False) lines = [ "# HELP karton_replicas Number of online replicas per identity", @@ -216,11 +222,13 @@ async def varz() -> PlainTextResponse: "# HELP karton_tasks Number of tasks per queue, priority and status", "# TYPE karton_tasks gauge", ] + task_counts: dict[tuple[str, str, str], int] = {} - for task in tasks: + async for task in gateway_backend.iter_all_tasks(parse_resources=False): receiver = task.headers.get("receiver", "") key = (receiver, task.priority.value, task.status.value) task_counts[key] = task_counts.get(key, 0) + 1 + for (queue, priority, status), count in task_counts.items(): lines.append( f'karton_tasks{{queue="{queue}",priority="{priority}",' From a17f3548fde79ee95d9f6783cd05f5a832f667ce Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 12 Oct 2026 18:18:24 +0200 Subject: [PATCH 05/21] Provide methods for cancelling and restarting whole queues --- karton/gateway/rest/models.py | 5 ++++ karton/gateway/rest/routes.py | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py index c8801a07..9b214ea2 100644 --- a/karton/gateway/rest/models.py +++ b/karton/gateway/rest/models.py @@ -71,5 +71,10 @@ class RestartedTask(BaseModel): uid: str +class BulkRestartedTask(BaseModel): + old_uid: str + new_uid: str + + class APIError(BaseModel): error: str diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py index 4bb51688..10f787b0 100644 --- a/karton/gateway/rest/routes.py +++ b/karton/gateway/rest/routes.py @@ -1,4 +1,5 @@ from collections import defaultdict +from typing import Literal from fastapi import APIRouter, Depends, HTTPException, Response from fastapi.responses import PlainTextResponse @@ -18,6 +19,7 @@ from karton.gateway.rest.models import ( AnalysisView, Bind, + BulkRestartedTask, ProducerOutput, QueueView, RestartedTask, @@ -146,6 +148,24 @@ async def _build_queues() -> dict[str, QueueView]: return queues +async def _get_queue_tasks_by_status(identity: str, status: str) -> list[Task]: + binds = {bind.identity for bind in await gateway_backend.get_binds()} + if identity not in binds: + raise HTTPException(status_code=404, detail="Queue doesn't exist") + + tasks: list[Task] = [] + async for task in gateway_backend.iter_all_tasks(parse_resources=False): + if task.headers.get("receiver") != identity: + continue + if status == "crashed": + if task.status == TaskState.CRASHED: + tasks.append(task) + elif status == "pending": + if task.status not in (TaskState.FINISHED, TaskState.CRASHED): + tasks.append(task) + return tasks + + @rest_api_router.get("/queues", dependencies=[Depends(CanInspectKarton)]) async def get_queues() -> dict[str, QueueView]: return await _build_queues() @@ -159,6 +179,34 @@ async def get_queue_details(identity: str) -> QueueView: return queues[identity] +@rest_api_router.post( + "/queues/{identity}/restart", dependencies=[Depends(CanRestartTask)] +) +async def restart_queue_tasks( + identity: str, + status: Literal["crashed", "pending"] = "crashed", +) -> list[BulkRestartedTask]: + tasks = await _get_queue_tasks_by_status(identity, status) + restarted: list[BulkRestartedTask] = [] + for task in tasks: + new_task = await gateway_backend.restart_task(task) + restarted.append(BulkRestartedTask(old_uid=task.uid, new_uid=new_task.uid)) + return restarted + + +@rest_api_router.post( + "/queues/{identity}/cancel", dependencies=[Depends(CanCancelTask)] +) +async def cancel_queue_tasks( + identity: str, + status: Literal["crashed", "pending"] = "crashed", +) -> Response: + tasks = await _get_queue_tasks_by_status(identity, status) + for task in tasks: + await gateway_backend.set_task_status(task, TaskState.FINISHED) + return Response(status_code=204) + + @rest_api_router.get("/tasks/{uid}", dependencies=[Depends(CanInspectKarton)]) async def get_task_details(uid: str) -> TaskView: task = await gateway_backend.get_task(uid) From 9eb451ba684764c1689819c9b61c06f2da1168ef Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 12 Oct 2026 18:24:02 +0200 Subject: [PATCH 06/21] Further fixes --- karton/gateway/rest/routes.py | 79 +++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py index 10f787b0..acceaa6d 100644 --- a/karton/gateway/rest/routes.py +++ b/karton/gateway/rest/routes.py @@ -1,7 +1,7 @@ from collections import defaultdict from typing import Literal -from fastapi import APIRouter, Depends, HTTPException, Response +from fastapi import APIRouter, Depends, HTTPException, Query, Response from fastapi.responses import PlainTextResponse from karton.core.__version__ import __version__ @@ -51,16 +51,16 @@ async def get_binds() -> list[Bind]: @rest_api_router.get("/binds/{identity}", dependencies=[Depends(CanInspectKarton)]) async def get_bind_details(identity: str) -> Bind: - for bind in await gateway_backend.get_binds(): - if bind.identity == identity: - return Bind.model_validate(bind) - raise HTTPException(status_code=404, detail="Bind doesn't exist") + bind = await gateway_backend.get_bind(identity) + if bind is None: + raise HTTPException(status_code=404, detail="Bind doesn't exist") + return Bind.model_validate(bind) @rest_api_router.delete("/binds/{identity}", dependencies=[Depends(CanRemoveBind)]) async def delete_bind(identity: str) -> Response: - binds = {bind.identity for bind in await gateway_backend.get_binds()} - if identity not in binds: + bind = await gateway_backend.get_bind(identity) + if bind is None: raise HTTPException(status_code=404, detail="Bind doesn't exist") replicas = await gateway_backend.get_online_identities() if replicas.get(identity): @@ -149,8 +149,8 @@ async def _build_queues() -> dict[str, QueueView]: async def _get_queue_tasks_by_status(identity: str, status: str) -> list[Task]: - binds = {bind.identity for bind in await gateway_backend.get_binds()} - if identity not in binds: + bind = await gateway_backend.get_bind(identity) + if bind is None: raise HTTPException(status_code=404, detail="Queue doesn't exist") tasks: list[Task] = [] @@ -173,10 +173,65 @@ async def get_queues() -> dict[str, QueueView]: @rest_api_router.get("/queues/{identity}", dependencies=[Depends(CanInspectKarton)]) async def get_queue_details(identity: str) -> QueueView: - queues = await _build_queues() - if identity not in queues: + bind = await gateway_backend.get_bind(identity) + if bind is None: raise HTTPException(status_code=404, detail="Queue doesn't exist") - return queues[identity] + replicas = await gateway_backend.get_online_identities() + + pending_uids: list[tuple[float, str]] = [] + crashed_uids: list[tuple[float, str]] = [] + async for task in gateway_backend.iter_all_tasks(parse_resources=False): + if task.headers.get("receiver") != identity: + continue + if task.status == TaskState.CRASHED: + crashed_uids.append((task.last_update, task.uid)) + elif task.status not in (TaskState.FINISHED, TaskState.CRASHED): + pending_uids.append((task.last_update, task.uid)) + + queue_pending = sorted(pending_uids, key=lambda item: item[0], reverse=True) + queue_crashed = sorted(crashed_uids, key=lambda item: item[0], reverse=True) + return QueueView( + identity=bind.identity, + info=bind.info, + version=bind.version, + persistent=bind.persistent, + filters=bind.filters, + service_version=bind.service_version, + is_async=bind.is_async, + replicas=len(replicas.get(identity, [])), + pending_task_uids=[uid for _, uid in queue_pending], + crashed_task_uids=[uid for _, uid in queue_crashed], + ) + + +@rest_api_router.get( + "/queues/{identity}/tasks", dependencies=[Depends(CanInspectKarton)] +) +async def get_queue_tasks( + identity: str, + status: Literal["pending", "crashed"] = "pending", + limit: int = Query(default=100, ge=1, le=1000), + offset: int = Query(default=0, ge=0), +) -> list[TaskView]: + bind = await gateway_backend.get_bind(identity) + if bind is None: + raise HTTPException(status_code=404, detail="Queue doesn't exist") + + matched: list[Task] = [] + async for task in gateway_backend.iter_all_tasks(parse_resources=False): + if task.headers.get("receiver") != identity: + continue + if status == "crashed": + if task.status != TaskState.CRASHED: + continue + else: + if task.status in (TaskState.FINISHED, TaskState.CRASHED): + continue + matched.append(task) + + matched.sort(key=lambda task: task.last_update, reverse=True) + page = matched[offset : offset + limit] + return [await _task_to_view(task) for task in page] @rest_api_router.post( From a772b0c1b314e654b8f01fcd6100e12875115625 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 12 Oct 2026 18:37:46 +0200 Subject: [PATCH 07/21] Include CRASHED tasks in get_analysis_details --- karton/gateway/rest/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py index acceaa6d..aaafd5b4 100644 --- a/karton/gateway/rest/routes.py +++ b/karton/gateway/rest/routes.py @@ -275,7 +275,7 @@ async def get_analysis_details(uid: str) -> AnalysisView: binds = {bind.identity for bind in await gateway_backend.get_binds()} queues: dict[str, list[TaskView]] = {} async for task in gateway_backend.iter_task_tree(uid): - if task.status in (TaskState.FINISHED, TaskState.CRASHED): + if task.status == TaskState.FINISHED: continue receiver = task.headers.get("receiver") if receiver is None or receiver not in binds: From f7aa018d9829af5477ae2c631a575634177edd54 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 12 Oct 2026 19:05:08 +0200 Subject: [PATCH 08/21] Further optimizations --- karton/gateway/rest/models.py | 4 ++ karton/gateway/rest/routes.py | 88 +++++++++++------------------------ 2 files changed, 32 insertions(+), 60 deletions(-) diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py index 9b214ea2..3dee0c04 100644 --- a/karton/gateway/rest/models.py +++ b/karton/gateway/rest/models.py @@ -76,5 +76,9 @@ class BulkRestartedTask(BaseModel): new_uid: str +class BatchTaskRequest(BaseModel): + uids: list[str] + + class APIError(BaseModel): error: str diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py index aaafd5b4..5d1961f3 100644 --- a/karton/gateway/rest/routes.py +++ b/karton/gateway/rest/routes.py @@ -1,11 +1,13 @@ from collections import defaultdict +from collections.abc import AsyncIterator from typing import Literal -from fastapi import APIRouter, Depends, HTTPException, Query, Response +from fastapi import APIRouter, Depends, HTTPException, Response from fastapi.responses import PlainTextResponse from karton.core.__version__ import __version__ from karton.core.backend import KartonBind, KartonMetrics +from karton.core.backend.direct import KARTON_TASK_NAMESPACE from karton.core.task import Task, TaskState from karton.gateway.backend import gateway_backend from karton.gateway.config import gateway_config @@ -18,6 +20,7 @@ ) from karton.gateway.rest.models import ( AnalysisView, + BatchTaskRequest, Bind, BulkRestartedTask, ProducerOutput, @@ -173,65 +176,10 @@ async def get_queues() -> dict[str, QueueView]: @rest_api_router.get("/queues/{identity}", dependencies=[Depends(CanInspectKarton)]) async def get_queue_details(identity: str) -> QueueView: - bind = await gateway_backend.get_bind(identity) - if bind is None: - raise HTTPException(status_code=404, detail="Queue doesn't exist") - replicas = await gateway_backend.get_online_identities() - - pending_uids: list[tuple[float, str]] = [] - crashed_uids: list[tuple[float, str]] = [] - async for task in gateway_backend.iter_all_tasks(parse_resources=False): - if task.headers.get("receiver") != identity: - continue - if task.status == TaskState.CRASHED: - crashed_uids.append((task.last_update, task.uid)) - elif task.status not in (TaskState.FINISHED, TaskState.CRASHED): - pending_uids.append((task.last_update, task.uid)) - - queue_pending = sorted(pending_uids, key=lambda item: item[0], reverse=True) - queue_crashed = sorted(crashed_uids, key=lambda item: item[0], reverse=True) - return QueueView( - identity=bind.identity, - info=bind.info, - version=bind.version, - persistent=bind.persistent, - filters=bind.filters, - service_version=bind.service_version, - is_async=bind.is_async, - replicas=len(replicas.get(identity, [])), - pending_task_uids=[uid for _, uid in queue_pending], - crashed_task_uids=[uid for _, uid in queue_crashed], - ) - - -@rest_api_router.get( - "/queues/{identity}/tasks", dependencies=[Depends(CanInspectKarton)] -) -async def get_queue_tasks( - identity: str, - status: Literal["pending", "crashed"] = "pending", - limit: int = Query(default=100, ge=1, le=1000), - offset: int = Query(default=0, ge=0), -) -> list[TaskView]: - bind = await gateway_backend.get_bind(identity) - if bind is None: + queues = await _build_queues() + if identity not in queues: raise HTTPException(status_code=404, detail="Queue doesn't exist") - - matched: list[Task] = [] - async for task in gateway_backend.iter_all_tasks(parse_resources=False): - if task.headers.get("receiver") != identity: - continue - if status == "crashed": - if task.status != TaskState.CRASHED: - continue - else: - if task.status in (TaskState.FINISHED, TaskState.CRASHED): - continue - matched.append(task) - - matched.sort(key=lambda task: task.last_update, reverse=True) - page = matched[offset : offset + limit] - return [await _task_to_view(task) for task in page] + return queues[identity] @rest_api_router.post( @@ -262,6 +210,26 @@ async def cancel_queue_tasks( return Response(status_code=204) +@rest_api_router.post("/tasks/batch", dependencies=[Depends(CanInspectKarton)]) +async def get_tasks_batch(request: BatchTaskRequest) -> list[TaskView]: + if len(request.uids) > 10000: + raise HTTPException( + status_code=400, + detail="Maximum number of task UIDs per request is 10000", + ) + + async def _key_iter() -> AsyncIterator[str]: + for uid in request.uids: + yield f"{KARTON_TASK_NAMESPACE}:{uid}" + + tasks: list[Task] = [] + async for task in gateway_backend._iter_tasks( + _key_iter(), chunk_size=1000, parse_resources=False + ): + tasks.append(task) + return [TaskView(**task.to_dict()) for task in tasks] + + @rest_api_router.get("/tasks/{uid}", dependencies=[Depends(CanInspectKarton)]) async def get_task_details(uid: str) -> TaskView: task = await gateway_backend.get_task(uid) @@ -280,7 +248,7 @@ async def get_analysis_details(uid: str) -> AnalysisView: receiver = task.headers.get("receiver") if receiver is None or receiver not in binds: continue - queues.setdefault(receiver, []).append(await _task_to_view(task)) + queues.setdefault(receiver, []).append(TaskView(**task.to_dict())) return AnalysisView(uid=uid, queues=queues) From 39d615be2ecb44e8373219a6a65dd54d3e2fad09 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 12 Oct 2026 19:26:46 +0200 Subject: [PATCH 09/21] PoC-quality LLM-generated wiring - to be redesigned --- karton/core/asyncio/backend/direct.py | 3 +- karton/core/backend/__init__.py | 12 +- karton/core/backend/base.py | 161 ++++++++++++++++++++- karton/core/backend/direct.py | 129 ++++++++++++++++- karton/core/backend/gateway.py | 198 +++++++++++++++++++++++++- karton/core/inspect.py | 177 ++++++++++++----------- karton/core/rest_models.py | 84 +++++++++++ karton/gateway/rest/models.py | 109 ++++---------- karton/gateway/rest/routes.py | 8 +- 9 files changed, 699 insertions(+), 182 deletions(-) create mode 100644 karton/core/rest_models.py diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index d91fdaa4..1703a0b9 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -17,7 +17,7 @@ from karton.core import Config, Task from karton.core.asyncio.resource import LocalResource, RemoteResource, ResourceData from karton.core.backend import KartonBind, KartonMetrics, KartonServiceInfo -from karton.core.backend.base import KartonExternalServiceInfo +from karton.core.backend.base import KartonExternalServiceInfo, KartonOutputs from karton.core.backend.direct import ( KARTON_BINDS_HSET, KARTON_OUTPUTS_NAMESPACE, @@ -25,7 +25,6 @@ KARTON_TASK_NAMESPACE, KARTON_TASKS_QUEUE, KartonBackendBase, - KartonOutputs, ) from karton.core.exceptions import BindExpiredError, HeartbeatExpiredError from karton.core.resource import LocalResource as SyncLocalResource diff --git a/karton/core/backend/__init__.py b/karton/core/backend/__init__.py index 7ac7e1a6..f16c20dc 100644 --- a/karton/core/backend/__init__.py +++ b/karton/core/backend/__init__.py @@ -1,6 +1,13 @@ from karton.core.config import Config -from .base import KartonBackendProtocol, KartonBind, KartonMetrics, KartonServiceInfo +from .base import ( + KartonBackendProtocol, + KartonBind, + KartonMetrics, + KartonOutputs, + KartonServiceInfo, + QueueSummary, +) from .direct import KartonBackend from .gateway import KartonGatewayBackend @@ -21,6 +28,9 @@ def get_backend( "KartonBind", "KartonGatewayBackend", "KartonMetrics", + "KartonOutputs", "KartonServiceInfo", + "KartonBackendProtocol", + "QueueSummary", "get_backend", ] diff --git a/karton/core/backend/base.py b/karton/core/backend/base.py index 46719f6f..58a804e0 100644 --- a/karton/core/backend/base.py +++ b/karton/core/backend/base.py @@ -2,7 +2,7 @@ import enum import uuid from collections.abc import Iterator -from typing import IO, Any, Protocol, Self +from typing import IO, Any, Literal, Protocol, Self from urllib.parse import parse_qsl, urlencode from karton.core.__version__ import __version__ @@ -169,6 +169,36 @@ def unserialize_bind(identity: str, bind: dict[str, Any]) -> KartonBind: ) +@dataclasses.dataclass(frozen=True) +class KartonOutputs: + """ + Represents the output types declared by a Karton service. + """ + + identity: str + outputs: list[dict[str, Any]] + + +@dataclasses.dataclass +class QueueSummary: + """ + Summary of a Karton queue state, returned by + :meth:`KartonBackendProtocol.get_queues`. + + For the direct backend, ``tasks`` is pre-loaded with full Task objects so that + :class:`karton.core.inspect.KartonQueue` can filter them in-memory. + + For the gateway backend, ``tasks`` is ``None`` — full task objects are fetched + lazily via :meth:`KartonBackendProtocol.get_tasks` on first access. + """ + + bind: KartonBind + replicas: int + tasks: list[Task] | None = None + pending_task_uids: list[str] = dataclasses.field(default_factory=list) + crashed_task_uids: list[str] = dataclasses.field(default_factory=list) + + class KartonBackendProtocol(Protocol): """ Protocol that defines methods that KartonBackend must implement. @@ -327,3 +357,132 @@ def consume_log( Python's :meth:`logging.Logger.setLevel`, ``level="INFO"`` matches only logs recorded at the ``INFO`` level, not ``WARNING`` or higher. """ + + def get_binds(self) -> list[KartonBind]: + """ + Get all binds registered in the backend. + + :return: List of KartonBind objects for all registered identities + """ + + def get_bind(self, identity: str) -> KartonBind | None: + """ + Get bind object for given identity. + + :param identity: Karton service identity + :return: KartonBind object or None if not found + """ + + def get_online_services( + self, _legacy: bool = True + ) -> list[KartonExternalServiceInfo]: + """ + Get all online services providing extended service information. + + :param _legacy: | + If True (default), returns services in a legacy-compatible format + (nullified instance_id, filtered to only those with karton_version). + :return: List of KartonExternalServiceInfo objects + """ + + def get_online_identities(self) -> dict[str, list[KartonExternalServiceInfo]]: + """ + Returns a dictionary for all online identities with list of + KartonExternalServiceInfo of each instance. + + :return: Dictionary {identity: [list of service info objects]} + """ + + def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: + """ + Get a summary of all queues (binds) and their task state. + + For the direct backend, full Task objects are included in each + :class:`QueueSummary`. For the gateway backend, only task UIDs are + included and full tasks are fetched lazily. + + :return: Dictionary mapping queue identity to :class:`QueueSummary` + """ + + def get_tasks(self, task_uids: list[str]) -> list[Task]: + """ + Get multiple task objects by their identifiers. + + Tasks that don't exist are silently skipped. + + :param task_uids: List of task identifiers + :return: List of Task objects (may be shorter than the input if some + tasks were not found) + """ + + def get_analysis_tasks(self, root_uid: str) -> dict[str, list[Task]]: + """ + Get all tasks belonging to the same analysis tree, grouped by + receiver queue. + + Includes both pending and crashed tasks (excludes only FINISHED). + + :param root_uid: Root identifier of the analysis task tree + :return: Dictionary mapping receiver queue identity to list of tasks + """ + + def get_task(self, task_uid: str) -> Task | None: + """ + Get task object with given identifier. + + :param task_uid: Task identifier + :return: Task object or None if not found + """ + + def restart_task(self, task: Task) -> Task: + """ + Requeue a consumed task back to the consumer queue. + + A new task is created with a new UID. The original task is marked + as finished. + + :param task: Task to be restarted + :return: The restarted (new) task object + """ + + def cancel_task(self, task_uid: str) -> None: + """ + Cancel a task by marking it as finished. + + Unlike :meth:`set_task_status`, this does not require a task token + and can be used on tasks obtained via inspection. + + :param task_uid: Task identifier to cancel + """ + + def restart_queue_tasks( + self, + identity: str, + status: Literal["crashed", "pending"] = "crashed", + ) -> list[tuple[str, str]]: + """ + Restart all tasks matching the given status in a queue. + + :param identity: Karton service identity (queue name) + :param status: Which tasks to restart (``"crashed"`` or ``"pending"``) + :return: List of ``(old_uid, new_uid)`` pairs for each restarted task + """ + + def cancel_queue_tasks( + self, + identity: str, + status: Literal["crashed", "pending"] = "crashed", + ) -> None: + """ + Cancel all tasks matching the given status in a queue. + + :param identity: Karton service identity (queue name) + :param status: Which tasks to cancel (``"crashed"`` or ``"pending"``) + """ + + def get_outputs(self) -> list[KartonOutputs]: + """ + Get a list of the output types for each Karton service. + + :return: List of KartonOutputs objects + """ diff --git a/karton/core/backend/direct.py b/karton/core/backend/direct.py index 750c2f60..a81e70f1 100644 --- a/karton/core/backend/direct.py +++ b/karton/core/backend/direct.py @@ -3,11 +3,12 @@ import logging import os import time -from collections import defaultdict, namedtuple +from collections import defaultdict from collections.abc import Iterable, Iterator from typing import ( IO, Any, + Literal, cast, ) @@ -33,7 +34,9 @@ KartonBind, KartonExternalServiceInfo, KartonMetrics, + KartonOutputs, KartonServiceInfo, + QueueSummary, unserialize_bind, ) @@ -44,7 +47,6 @@ KARTON_OUTPUTS_NAMESPACE = "karton.outputs" KARTON_SERVICES_NAMESPACE = "karton.services" -KartonOutputs = namedtuple("KartonOutputs", ["identity", "outputs"]) logger = logging.getLogger(__name__) @@ -1115,5 +1117,128 @@ def get_outputs(self) -> list[KartonOutputs]: for identity in output_keys ] + def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: + """ + Get a summary of all queues and their task state. + + Streams all tasks from Redis via :meth:`iter_all_tasks` and groups + them by receiver. Full Task objects are pre-loaded in each + :class:`QueueSummary` for in-memory filtering by + :class:`karton.core.inspect.KartonQueue`. + + :param parse_resources: If False (default), resources are not parsed + to speed up deserialization. + :return: Dictionary mapping queue identity to :class:`QueueSummary` + """ + binds = {bind.identity: bind for bind in self.get_binds()} + replicas = self.get_online_identities() + + tasks_per_queue: dict[str, list[Task]] = defaultdict(list) + pending_uids: dict[str, list[str]] = defaultdict(list) + crashed_uids: dict[str, list[str]] = defaultdict(list) + + for task in self.iter_all_tasks(parse_resources=parse_resources): + receiver = task.headers.get("receiver") + if receiver is None or receiver not in binds: + continue + if task.status == TaskState.FINISHED: + continue + tasks_per_queue[receiver].append(task) + if task.status == TaskState.CRASHED: + crashed_uids[receiver].append(task.uid) + else: + pending_uids[receiver].append(task.uid) + + queues: dict[str, QueueSummary] = {} + for identity, bind in binds.items(): + queues[identity] = QueueSummary( + bind=bind, + replicas=len(replicas.get(identity, [])), + tasks=tasks_per_queue.get(identity, []), + pending_task_uids=pending_uids.get(identity, []), + crashed_task_uids=crashed_uids.get(identity, []), + ) + return queues + + def get_analysis_tasks(self, root_uid: str) -> dict[str, list[Task]]: + """ + Get all non-finished tasks belonging to the same analysis tree, + grouped by receiver queue. + + :param root_uid: Root identifier of the analysis task tree + :return: Dictionary mapping receiver queue identity to list of tasks + """ + binds = {bind.identity for bind in self.get_binds()} + tasks_per_queue: dict[str, list[Task]] = defaultdict(list) + + for task in self.iter_task_tree(root_uid, parse_resources=False): + if task.status == TaskState.FINISHED: + continue + receiver = task.headers.get("receiver") + if receiver is None or receiver not in binds: + continue + tasks_per_queue[receiver].append(task) + + return dict(tasks_per_queue) + + def cancel_task(self, task_uid: str) -> None: + """ + Cancel a task by marking it as finished. + + :param task_uid: Task identifier to cancel + """ + task = self.get_task(task_uid) + if task is None: + raise RuntimeError(f"Task {task_uid} doesn't exist") + self.set_task_status(task, TaskState.FINISHED) + + def restart_queue_tasks( + self, + identity: str, + status: Literal["crashed", "pending"] = "crashed", + ) -> list[tuple[str, str]]: + """ + Restart all tasks matching the given status in a queue. + + :param identity: Karton service identity (queue name) + :param status: Which tasks to restart (``"crashed"`` or ``"pending"``) + :return: List of ``(old_uid, new_uid)`` pairs for each restarted task + """ + restarted: list[tuple[str, str]] = [] + for task in self.iter_all_tasks(parse_resources=False): + if task.headers.get("receiver") != identity: + continue + if status == "crashed": + if task.status != TaskState.CRASHED: + continue + else: + if task.status in (TaskState.FINISHED, TaskState.CRASHED): + continue + new_task = self.restart_task(task) + restarted.append((task.uid, new_task.uid)) + return restarted + + def cancel_queue_tasks( + self, + identity: str, + status: Literal["crashed", "pending"] = "crashed", + ) -> None: + """ + Cancel all tasks matching the given status in a queue. + + :param identity: Karton service identity (queue name) + :param status: Which tasks to cancel (``"crashed"`` or ``"pending"``) + """ + for task in self.iter_all_tasks(parse_resources=False): + if task.headers.get("receiver") != identity: + continue + if status == "crashed": + if task.status != TaskState.CRASHED: + continue + else: + if task.status in (TaskState.FINISHED, TaskState.CRASHED): + continue + self.set_task_status(task, TaskState.FINISHED) + def make_pipeline(self, transaction: bool = False) -> Pipeline: return self.redis.pipeline(transaction=transaction) diff --git a/karton/core/backend/gateway.py b/karton/core/backend/gateway.py index 2321b7a6..8afa44d7 100644 --- a/karton/core/backend/gateway.py +++ b/karton/core/backend/gateway.py @@ -2,11 +2,13 @@ import logging import time import urllib.parse +from collections import defaultdict from collections.abc import Callable, Iterator from io import BytesIO -from typing import IO, Any, cast +from typing import IO, Any, Literal, cast import httpx2 +from pydantic import TypeAdapter from karton.core.config import Config from karton.core.exceptions import BindExpiredError @@ -50,10 +52,29 @@ ResourceBase, ResourceData, ) +from karton.core.rest_models import ( + AnalysisView, + BatchTaskRequest, + BulkRestartedTask, + ProducerOutput, + QueueView, + RestartedTask, + ServiceInfo, + TaskView, +) +from karton.core.rest_models import Bind as BindView from karton.core.task import Task, TaskPriority, TaskState, root_uid_from_task_uid from karton.core.utils import recursive_map -from .base import KartonBackendProtocol, KartonBind, KartonMetrics, KartonServiceInfo +from .base import ( + KartonBackendProtocol, + KartonBind, + KartonExternalServiceInfo, + KartonMetrics, + KartonOutputs, + KartonServiceInfo, + QueueSummary, +) logger = logging.getLogger(__name__) @@ -243,7 +264,18 @@ def __init__( connection_pool_soft_limit=1, ) + auth_tokens = [t for t in self.gateway_auth_tokens if t] + headers = {} + if auth_tokens: + headers["Authorization"] = f"Bearer {','.join(auth_tokens)}" + self._http_client = httpx2.Client( + base_url=self.rest_url, + headers=headers, + timeout=self.gateway_response_timeout, + ) + def close(self) -> None: + self._http_client.close() self._gateway_client.close() def register_bind(self, bind: KartonBind) -> KartonBind | None: @@ -458,3 +490,165 @@ def consume_log( def increment_metrics(self, metric: KartonMetrics, identity: str) -> None: # This is no-op, Karton gateway manages all metrics return + + @staticmethod + def _task_from_view(view: TaskView) -> Task: + return Task( + headers=view.headers, + headers_persistent=view.headers_persistent, + payload=view.payload, + payload_persistent=view.payload_persistent, + priority=view.priority, + parent_uid=view.parent_uid, + root_uid=view.root_uid, + orig_uid=view.orig_uid, + uid=view.uid, + error=view.error, + _status=view.status, + _last_update=view.last_update, + ) + + @staticmethod + def _bind_from_view(view: BindView) -> KartonBind: + return KartonBind(**view.model_dump()) + + @staticmethod + def _service_info_from_view( + view: ServiceInfo, + ) -> list[KartonExternalServiceInfo]: + return [ + KartonExternalServiceInfo( + identity=view.identity, + karton_version=view.karton_version, + service_version=view.service_version, + ) + for _ in range(view.active_replicas) + ] + + def get_binds(self) -> list[KartonBind]: + response = self._http_client.get("/binds") + response.raise_for_status() + return [ + self._bind_from_view(b) + for b in TypeAdapter(list[BindView]).validate_json(response.content) + ] + + def get_bind(self, identity: str) -> KartonBind | None: + response = self._http_client.get(f"/binds/{identity}") + if response.status_code == 404: + return None + response.raise_for_status() + return self._bind_from_view(BindView.model_validate_json(response.content)) + + def get_online_services( + self, _legacy: bool = True + ) -> list[KartonExternalServiceInfo]: + response = self._http_client.get("/services") + response.raise_for_status() + services: list[KartonExternalServiceInfo] = [] + for s in TypeAdapter(list[ServiceInfo]).validate_json(response.content): + if _legacy and s.karton_version is None: + continue + services.extend(self._service_info_from_view(s)) + return services + + def get_online_identities(self) -> dict[str, list[KartonExternalServiceInfo]]: + response = self._http_client.get("/services") + response.raise_for_status() + identities: dict[str, list[KartonExternalServiceInfo]] = defaultdict(list) + for s in TypeAdapter(list[ServiceInfo]).validate_json(response.content): + identities[s.identity].extend(self._service_info_from_view(s)) + return dict(identities) + + def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: + response = self._http_client.get("/queues") + response.raise_for_status() + raw = TypeAdapter(dict[str, QueueView]).validate_json(response.content) + return { + identity: QueueSummary( + bind=KartonBind( + identity=qv.identity, + info=qv.info, + version=qv.version, + persistent=qv.persistent, + filters=qv.filters, + service_version=qv.service_version, + is_async=qv.is_async, + ), + replicas=qv.replicas, + tasks=None, + pending_task_uids=qv.pending_task_uids, + crashed_task_uids=qv.crashed_task_uids, + ) + for identity, qv in raw.items() + } + + def get_tasks(self, task_uids: list[str]) -> list[Task]: + if not task_uids: + return [] + response = self._http_client.post( + "/tasks/batch", + json=BatchTaskRequest(uids=task_uids).model_dump(), + ) + response.raise_for_status() + views = TypeAdapter(list[TaskView]).validate_json(response.content) + return [self._task_from_view(view) for view in views] + + def get_analysis_tasks(self, root_uid: str) -> dict[str, list[Task]]: + response = self._http_client.get(f"/analyses/{root_uid}") + response.raise_for_status() + view = AnalysisView.model_validate_json(response.content) + return { + receiver: [self._task_from_view(t) for t in tasks] + for receiver, tasks in view.queues.items() + } + + def get_task(self, task_uid: str) -> Task | None: + response = self._http_client.get(f"/tasks/{task_uid}") + if response.status_code == 404: + return None + response.raise_for_status() + return self._task_from_view(TaskView.model_validate_json(response.content)) + + def restart_task(self, task: Task) -> Task: + response = self._http_client.post(f"/tasks/{task.uid}/restart") + response.raise_for_status() + new_uid = RestartedTask.model_validate_json(response.content).uid + new_task = self.get_task(new_uid) + if new_task is None: + raise RuntimeError(f"Restarted task {new_uid} not found") + return new_task + + def cancel_task(self, task_uid: str) -> None: + response = self._http_client.post(f"/tasks/{task_uid}/cancel") + response.raise_for_status() + + def restart_queue_tasks( + self, + identity: str, + status: Literal["crashed", "pending"] = "crashed", + ) -> list[tuple[str, str]]: + response = self._http_client.post( + f"/queues/{identity}/restart", + params={"status": status}, + ) + response.raise_for_status() + restarted = TypeAdapter(list[BulkRestartedTask]).validate_json(response.content) + return [(r.old_uid, r.new_uid) for r in restarted] + + def cancel_queue_tasks( + self, + identity: str, + status: Literal["crashed", "pending"] = "crashed", + ) -> None: + response = self._http_client.post( + f"/queues/{identity}/cancel", + params={"status": status}, + ) + response.raise_for_status() + + def get_outputs(self) -> list[KartonOutputs]: + response = self._http_client.get("/outputs") + response.raise_for_status() + outputs = TypeAdapter(list[ProducerOutput]).validate_json(response.content) + return [KartonOutputs(identity=o.identity, outputs=o.outputs) for o in outputs] diff --git a/karton/core/inspect.py b/karton/core/inspect.py index 3dedd05c..70991a11 100644 --- a/karton/core/inspect.py +++ b/karton/core/inspect.py @@ -1,24 +1,26 @@ from collections import defaultdict -from .backend import KartonBackend, KartonBind +from .backend import KartonBackendProtocol, QueueSummary from .task import Task, TaskState class KartonQueue: """ - View object representing a Karton queue + View object representing a Karton queue. - :param bind: :class:`KartonBind` object representing the queue bind - :param tasks: List of tasks currently in queue - :param state: :class:`KartonState` object to be used + :param summary: :class:`QueueSummary` with queue state data + :param state: :class:`KartonState` object to be used for lazy loading """ - def __init__( - self, bind: KartonBind, tasks: list[Task], state: "KartonState" - ) -> None: - self.bind = bind - self.tasks = tasks + def __init__(self, summary: QueueSummary, state: "KartonState") -> None: + self.bind = summary.bind self.state = state + self._replicas = summary.replicas + self._tasks: list[Task] | None = summary.tasks + self.pending_task_uids = summary.pending_task_uids + self.crashed_task_uids = summary.crashed_task_uids + self._pending_tasks: list[Task] | None = None + self._crashed_tasks: list[Task] | None = None @property def last_update(self) -> float: @@ -28,33 +30,57 @@ def last_update(self) -> float: @property def online_consumers_count(self) -> int: """Get number of consumers listening on this queue""" - return ( - len(self.state.replicas[self.bind.identity]) - if self.bind.identity in self.state.replicas - else 0 - ) + return self._replicas + + @property + def tasks(self) -> list[Task]: + """ + Get all non-finished tasks in this queue (pending and crashed). + + For the direct backend, tasks are pre-loaded. For the gateway + backend, tasks are fetched lazily on first access. + """ + if self._tasks is not None: + return self._tasks + return self.pending_tasks + self.crashed_tasks @property def pending_tasks(self) -> list[Task]: """Get queue pending tasks""" - return [ - task - for task in self.tasks - if task.status not in (TaskState.FINISHED, TaskState.CRASHED) - ] + if self._tasks is not None: + return [ + task + for task in self._tasks + if task.status not in (TaskState.FINISHED, TaskState.CRASHED) + ] + if self._pending_tasks is None: + tasks = self.state.backend.get_tasks(self.pending_task_uids) + self._pending_tasks = [ + task + for task in tasks + if task.status not in (TaskState.FINISHED, TaskState.CRASHED) + ] + return self._pending_tasks @property def crashed_tasks(self) -> list[Task]: """Get queue crashed tasks""" - return [task for task in self.tasks if task.status == TaskState.CRASHED] + if self._tasks is not None: + return [task for task in self._tasks if task.status == TaskState.CRASHED] + if self._crashed_tasks is None: + tasks = self.state.backend.get_tasks(self.crashed_task_uids) + self._crashed_tasks = [ + task for task in tasks if task.status == TaskState.CRASHED + ] + return self._crashed_tasks class KartonAnalysis: """ - View object representing a Karton task analysis + View object representing a Karton task analysis. :param root_uid: Analysis root task uid - :param tasks: List of tasks + :param tasks: List of tasks (pending and crashed, excluding finished) :param state: :class:`KartonState` object to be used """ @@ -83,7 +109,7 @@ def pending_tasks(self) -> list[Task]: ] @property - def pending_queues(self) -> dict[str, KartonQueue]: + def pending_queues(self) -> dict[str, "KartonQueue"]: """Group analysis tasks by their queues""" return get_queues_for_tasks(self.tasks, self.state) @@ -97,13 +123,13 @@ def get_queues_for_tasks( tasks: list[Task], state: "KartonState" ) -> dict[str, KartonQueue]: """ - Group task objects by their queue name + Group task objects by their queue name. :param tasks: Task objects to group :param state: :class:`KartonState` object to be used - :return: A dictionary containing the queue names and lists of tasks + :return: A dictionary containing the queue names and :class:`KartonQueue` objects """ - tasks_per_queue = defaultdict(list) + tasks_per_queue: dict[str, list[Task]] = defaultdict(list) for task in tasks: if "receiver" not in task.headers: @@ -114,8 +140,16 @@ def get_queues_for_tasks( # No known bind, dangling task for non-existent queue continue tasks_per_queue[queue_name].append(task) + return { - queue_name: KartonQueue(bind=state.binds[queue_name], tasks=tasks, state=state) + queue_name: KartonQueue( + summary=QueueSummary( + bind=state.binds[queue_name], + replicas=len(state.replicas.get(queue_name, [])), + tasks=tasks, + ), + state=state, + ) for queue_name, tasks in tasks_per_queue.items() } @@ -125,75 +159,50 @@ class KartonState: Karton state inspection class. Allows to make a detailed inspection of the pipeline and analyses state. - .. versionadded: 4.0.0 + Works with both the direct backend (Redis) and the gateway backend + (REST API). When using the gateway backend, queue task lists are + fetched lazily on first access to :attr:`KartonQueue.pending_tasks` + or :attr:`KartonQueue.crashed_tasks`. - :param backend: :py:meth:`KartonBackend` object to use for data fetching + :param backend: :py:class:`KartonBackend` object to use for data fetching + :param parse_resources: If set to False (default), resources are not + parsed to speed up deserialization. """ - def __init__(self, backend: KartonBackend, parse_resources: bool = False) -> None: + def __init__( + self, backend: KartonBackendProtocol, parse_resources: bool = False + ) -> None: self.backend = backend self.binds = {bind.identity: bind for bind in backend.get_binds()} self.replicas = backend.get_online_identities() self.parse_resources = parse_resources - self._tasks: list[Task] | None = None - self._pending_tasks: list[Task] | None = None - self._analyses: dict[str, KartonAnalysis] | None = None self._queues: dict[str, KartonQueue] | None = None - @property - def tasks(self) -> list[Task]: - if self._tasks is None: - self._tasks = self.backend.get_all_tasks( - parse_resources=self.parse_resources - ) - return self._tasks - - @property - def pending_tasks(self) -> list[Task]: - if self._pending_tasks is None: - self._pending_tasks = [ - task - for task in self.tasks - if task.status not in (TaskState.FINISHED, TaskState.CRASHED) - ] - return self._pending_tasks - - @property - def analyses(self) -> dict[str, KartonAnalysis]: - if self._analyses is None: - # Tasks grouped by root_uid - tasks_per_analysis = defaultdict(list) - - for task in self.pending_tasks: - tasks_per_analysis[task.root_uid].append(task) - - self._analyses = { - root_uid: KartonAnalysis(root_uid=root_uid, tasks=tasks, state=self) - for root_uid, tasks in tasks_per_analysis.items() - } - return self._analyses - @property def queues(self) -> dict[str, KartonQueue]: + """ + Get all queues (binds) with their task state. + + Returns a dictionary mapping queue identity to + :class:`KartonQueue`. Registered queues without any tasks + are included with empty task lists. + """ if self._queues is None: - queues = get_queues_for_tasks(self.pending_tasks, self) - # Present registered queues without tasks - for bind_name in self.binds: - if bind_name not in queues: - queues[bind_name] = KartonQueue( - bind=self.binds[bind_name], tasks=[], state=self - ) - self._queues = queues + summaries = self.backend.get_queues(parse_resources=self.parse_resources) + self._queues = { + identity: KartonQueue(summary=summary, state=self) + for identity, summary in summaries.items() + } return self._queues def get_analysis(self, root_uid: str) -> KartonAnalysis: - return KartonAnalysis( - root_uid=root_uid, - tasks=list( - self.backend.iter_task_tree( - root_uid, parse_resources=self.parse_resources - ) - ), - state=self, - ) + """ + Get a detailed view of a single analysis (task tree). + + :param root_uid: Root identifier of the analysis task tree + :return: :class:`KartonAnalysis` object + """ + analysis_tasks = self.backend.get_analysis_tasks(root_uid) + tasks = [task for task_list in analysis_tasks.values() for task in task_list] + return KartonAnalysis(root_uid=root_uid, tasks=tasks, state=self) diff --git a/karton/core/rest_models.py b/karton/core/rest_models.py new file mode 100644 index 00000000..3dee0c04 --- /dev/null +++ b/karton/core/rest_models.py @@ -0,0 +1,84 @@ +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from karton.core.gateway_protocol import ResourceUrl +from karton.core.task import TaskPriority, TaskState + + +class Bind(BaseModel): + model_config = ConfigDict(from_attributes=True) + + identity: str + info: str | None + version: str + persistent: bool + filters: list[dict[str, Any]] + service_version: str | None + is_async: bool + + +class ServiceInfo(BaseModel): + identity: str + karton_version: str | None + service_version: str | None + instance_id: str | None + active_replicas: int + + +class QueueView(BaseModel): + identity: str + info: str | None + version: str + persistent: bool + filters: list[dict[str, Any]] + service_version: str | None + is_async: bool + replicas: int + pending_task_uids: list[str] + crashed_task_uids: list[str] + + +class TaskView(BaseModel): + uid: str + root_uid: str + parent_uid: str | None + orig_uid: str | None + status: TaskState + priority: TaskPriority + last_update: float + headers: dict[str, Any] + headers_persistent: dict[str, Any] + payload: dict[str, Any] + payload_persistent: dict[str, Any] + error: list[str] | None + download_urls: list[ResourceUrl] = Field(default_factory=list) + + +class AnalysisView(BaseModel): + uid: str + queues: dict[str, list[TaskView]] + + +class ProducerOutput(BaseModel): + model_config = ConfigDict(from_attributes=True) + + identity: str + outputs: list[dict[str, Any]] + + +class RestartedTask(BaseModel): + uid: str + + +class BulkRestartedTask(BaseModel): + old_uid: str + new_uid: str + + +class BatchTaskRequest(BaseModel): + uids: list[str] + + +class APIError(BaseModel): + error: str diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py index 3dee0c04..996f8d9f 100644 --- a/karton/gateway/rest/models.py +++ b/karton/gateway/rest/models.py @@ -1,84 +1,25 @@ -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field - -from karton.core.gateway_protocol import ResourceUrl -from karton.core.task import TaskPriority, TaskState - - -class Bind(BaseModel): - model_config = ConfigDict(from_attributes=True) - - identity: str - info: str | None - version: str - persistent: bool - filters: list[dict[str, Any]] - service_version: str | None - is_async: bool - - -class ServiceInfo(BaseModel): - identity: str - karton_version: str | None - service_version: str | None - instance_id: str | None - active_replicas: int - - -class QueueView(BaseModel): - identity: str - info: str | None - version: str - persistent: bool - filters: list[dict[str, Any]] - service_version: str | None - is_async: bool - replicas: int - pending_task_uids: list[str] - crashed_task_uids: list[str] - - -class TaskView(BaseModel): - uid: str - root_uid: str - parent_uid: str | None - orig_uid: str | None - status: TaskState - priority: TaskPriority - last_update: float - headers: dict[str, Any] - headers_persistent: dict[str, Any] - payload: dict[str, Any] - payload_persistent: dict[str, Any] - error: list[str] | None - download_urls: list[ResourceUrl] = Field(default_factory=list) - - -class AnalysisView(BaseModel): - uid: str - queues: dict[str, list[TaskView]] - - -class ProducerOutput(BaseModel): - model_config = ConfigDict(from_attributes=True) - - identity: str - outputs: list[dict[str, Any]] - - -class RestartedTask(BaseModel): - uid: str - - -class BulkRestartedTask(BaseModel): - old_uid: str - new_uid: str - - -class BatchTaskRequest(BaseModel): - uids: list[str] - - -class APIError(BaseModel): - error: str +from karton.core.rest_models import ( + AnalysisView, + APIError, + BatchTaskRequest, + Bind, + BulkRestartedTask, + ProducerOutput, + QueueView, + RestartedTask, + ServiceInfo, + TaskView, +) + +__all__ = [ + "APIError", + "AnalysisView", + "BatchTaskRequest", + "Bind", + "BulkRestartedTask", + "ProducerOutput", + "QueueView", + "RestartedTask", + "ServiceInfo", + "TaskView", +] diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py index 5d1961f3..6f9505c5 100644 --- a/karton/gateway/rest/routes.py +++ b/karton/gateway/rest/routes.py @@ -42,11 +42,6 @@ def _allowed_buckets() -> list[str]: ] -async def _task_to_view(task: Task) -> TaskView: - download_urls = await generate_resource_download_urls(task, _allowed_buckets()) - return TaskView(**task.to_dict(), download_urls=download_urls) - - @rest_api_router.get("/binds", dependencies=[Depends(CanInspectKarton)]) async def get_binds() -> list[Bind]: return [Bind.model_validate(bind) for bind in await gateway_backend.get_binds()] @@ -235,7 +230,8 @@ async def get_task_details(uid: str) -> TaskView: task = await gateway_backend.get_task(uid) if task is None: raise HTTPException(status_code=404, detail="Task doesn't exist") - return await _task_to_view(task) + download_urls = await generate_resource_download_urls(task, _allowed_buckets()) + return TaskView(**task.to_dict(), download_urls=download_urls) @rest_api_router.get("/analyses/{uid}", dependencies=[Depends(CanInspectKarton)]) From 0d51506b5166296e9b6d41c622726702d9e2a9c3 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Thu, 15 Oct 2026 18:13:46 +0200 Subject: [PATCH 10/21] Lint --- karton/core/asyncio/backend/direct.py | 2 +- karton/core/backend/__init__.py | 1 - pyproject.toml | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index 1703a0b9..b42b0b6d 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -381,7 +381,7 @@ async def _get_online_services(self) -> AsyncIterator[KartonExternalServiceInfo] async for service in self._get_online_gateway_services(): services[service] += 1 - for service in services.keys(): + for service in services: if service.instance_id is None: for _ in range(services[service]): yield service diff --git a/karton/core/backend/__init__.py b/karton/core/backend/__init__.py index f16c20dc..e955f460 100644 --- a/karton/core/backend/__init__.py +++ b/karton/core/backend/__init__.py @@ -30,7 +30,6 @@ def get_backend( "KartonMetrics", "KartonOutputs", "KartonServiceInfo", - "KartonBackendProtocol", "QueueSummary", "get_backend", ] diff --git a/pyproject.toml b/pyproject.toml index 80033cfa..b9604d58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ extend-ignore = [ "ASYNC230", "BLE001", "RUF012", + "SIM102", "SIM103", "SIM115", "TRY004", From a98e9803f096d38bd099cfd9ecf8de7e28bb3e21 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Thu, 15 Oct 2026 19:01:14 +0200 Subject: [PATCH 11/21] Split routes --- karton/gateway/rest/routes.py | 321 ------------------------- karton/gateway/rest/routes/__init__.py | 11 + karton/gateway/rest/routes/_routers.py | 4 + karton/gateway/rest/routes/binds.py | 56 +++++ karton/gateway/rest/routes/outputs.py | 15 ++ karton/gateway/rest/routes/queues.py | 120 +++++++++ karton/gateway/rest/routes/services.py | 26 ++ karton/gateway/rest/routes/tasks.py | 90 +++++++ karton/gateway/rest/routes/varz.py | 57 +++++ pyproject.toml | 2 +- 10 files changed, 380 insertions(+), 322 deletions(-) delete mode 100644 karton/gateway/rest/routes.py create mode 100644 karton/gateway/rest/routes/__init__.py create mode 100644 karton/gateway/rest/routes/_routers.py create mode 100644 karton/gateway/rest/routes/binds.py create mode 100644 karton/gateway/rest/routes/outputs.py create mode 100644 karton/gateway/rest/routes/queues.py create mode 100644 karton/gateway/rest/routes/services.py create mode 100644 karton/gateway/rest/routes/tasks.py create mode 100644 karton/gateway/rest/routes/varz.py diff --git a/karton/gateway/rest/routes.py b/karton/gateway/rest/routes.py deleted file mode 100644 index 6f9505c5..00000000 --- a/karton/gateway/rest/routes.py +++ /dev/null @@ -1,321 +0,0 @@ -from collections import defaultdict -from collections.abc import AsyncIterator -from typing import Literal - -from fastapi import APIRouter, Depends, HTTPException, Response -from fastapi.responses import PlainTextResponse - -from karton.core.__version__ import __version__ -from karton.core.backend import KartonBind, KartonMetrics -from karton.core.backend.direct import KARTON_TASK_NAMESPACE -from karton.core.task import Task, TaskState -from karton.gateway.backend import gateway_backend -from karton.gateway.config import gateway_config -from karton.gateway.rest.auth import ( - CanCancelTask, - CanGetMetrics, - CanInspectKarton, - CanRemoveBind, - CanRestartTask, -) -from karton.gateway.rest.models import ( - AnalysisView, - BatchTaskRequest, - Bind, - BulkRestartedTask, - ProducerOutput, - QueueView, - RestartedTask, - ServiceInfo, - TaskView, -) -from karton.gateway.task import generate_resource_download_urls - -rest_api_router = APIRouter(prefix="/api/v1") -varz_router = APIRouter() - - -def _allowed_buckets() -> list[str]: - return [ - gateway_backend.default_bucket_name, - *gateway_config.allowed_foreign_buckets, - ] - - -@rest_api_router.get("/binds", dependencies=[Depends(CanInspectKarton)]) -async def get_binds() -> list[Bind]: - return [Bind.model_validate(bind) for bind in await gateway_backend.get_binds()] - - -@rest_api_router.get("/binds/{identity}", dependencies=[Depends(CanInspectKarton)]) -async def get_bind_details(identity: str) -> Bind: - bind = await gateway_backend.get_bind(identity) - if bind is None: - raise HTTPException(status_code=404, detail="Bind doesn't exist") - return Bind.model_validate(bind) - - -@rest_api_router.delete("/binds/{identity}", dependencies=[Depends(CanRemoveBind)]) -async def delete_bind(identity: str) -> Response: - bind = await gateway_backend.get_bind(identity) - if bind is None: - raise HTTPException(status_code=404, detail="Bind doesn't exist") - replicas = await gateway_backend.get_online_identities() - if replicas.get(identity): - raise HTTPException( - status_code=409, - detail=( - "Bind has active replicas that need to be downscaled " - "before it can be deleted" - ), - ) - # Replace the bind with non-persistent bind with empty filters. - # This is "mark for deletion" for Karton System GC - tombstone_bind = KartonBind( - identity=identity, - info=None, - version=__version__, - persistent=False, - filters=[], - service_version=None, - is_async=False, - ) - await gateway_backend.register_bind(tombstone_bind, bind_backend=False) - return Response(status_code=204) - - -@rest_api_router.get("/services", dependencies=[Depends(CanInspectKarton)]) -async def get_services() -> list[ServiceInfo]: - services = await gateway_backend.get_online_services(_legacy=False) - counts: dict[tuple[str, str | None, str | None], int] = {} - for service in services: - key = (service.identity, service.karton_version, service.service_version) - counts[key] = counts.get(key, 0) + 1 - return [ - ServiceInfo( - identity=identity, - karton_version=karton_version, - service_version=service_version, - instance_id=None, - active_replicas=replicas, - ) - for (identity, karton_version, service_version), replicas in counts.items() - ] - - -async def _build_queues() -> dict[str, QueueView]: - binds = {bind.identity: bind for bind in await gateway_backend.get_binds()} - replicas = await gateway_backend.get_online_identities() - - pending_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) - crashed_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) - - async for task in gateway_backend.iter_all_tasks(parse_resources=False): - receiver = task.headers.get("receiver") - if receiver is None or receiver not in binds: - continue - if task.status == TaskState.CRASHED: - crashed_uids[receiver].append((task.last_update, task.uid)) - elif task.status not in (TaskState.FINISHED, TaskState.CRASHED): - pending_uids[receiver].append((task.last_update, task.uid)) - - queues: dict[str, QueueView] = {} - for identity, bind in binds.items(): - queue_pending = sorted( - pending_uids.get(identity, []), - key=lambda item: item[0], - reverse=True, - ) - queue_crashed = sorted( - crashed_uids.get(identity, []), - key=lambda item: item[0], - reverse=True, - ) - queues[identity] = QueueView( - identity=bind.identity, - info=bind.info, - version=bind.version, - persistent=bind.persistent, - filters=bind.filters, - service_version=bind.service_version, - is_async=bind.is_async, - replicas=len(replicas.get(identity, [])), - pending_task_uids=[uid for _, uid in queue_pending], - crashed_task_uids=[uid for _, uid in queue_crashed], - ) - return queues - - -async def _get_queue_tasks_by_status(identity: str, status: str) -> list[Task]: - bind = await gateway_backend.get_bind(identity) - if bind is None: - raise HTTPException(status_code=404, detail="Queue doesn't exist") - - tasks: list[Task] = [] - async for task in gateway_backend.iter_all_tasks(parse_resources=False): - if task.headers.get("receiver") != identity: - continue - if status == "crashed": - if task.status == TaskState.CRASHED: - tasks.append(task) - elif status == "pending": - if task.status not in (TaskState.FINISHED, TaskState.CRASHED): - tasks.append(task) - return tasks - - -@rest_api_router.get("/queues", dependencies=[Depends(CanInspectKarton)]) -async def get_queues() -> dict[str, QueueView]: - return await _build_queues() - - -@rest_api_router.get("/queues/{identity}", dependencies=[Depends(CanInspectKarton)]) -async def get_queue_details(identity: str) -> QueueView: - queues = await _build_queues() - if identity not in queues: - raise HTTPException(status_code=404, detail="Queue doesn't exist") - return queues[identity] - - -@rest_api_router.post( - "/queues/{identity}/restart", dependencies=[Depends(CanRestartTask)] -) -async def restart_queue_tasks( - identity: str, - status: Literal["crashed", "pending"] = "crashed", -) -> list[BulkRestartedTask]: - tasks = await _get_queue_tasks_by_status(identity, status) - restarted: list[BulkRestartedTask] = [] - for task in tasks: - new_task = await gateway_backend.restart_task(task) - restarted.append(BulkRestartedTask(old_uid=task.uid, new_uid=new_task.uid)) - return restarted - - -@rest_api_router.post( - "/queues/{identity}/cancel", dependencies=[Depends(CanCancelTask)] -) -async def cancel_queue_tasks( - identity: str, - status: Literal["crashed", "pending"] = "crashed", -) -> Response: - tasks = await _get_queue_tasks_by_status(identity, status) - for task in tasks: - await gateway_backend.set_task_status(task, TaskState.FINISHED) - return Response(status_code=204) - - -@rest_api_router.post("/tasks/batch", dependencies=[Depends(CanInspectKarton)]) -async def get_tasks_batch(request: BatchTaskRequest) -> list[TaskView]: - if len(request.uids) > 10000: - raise HTTPException( - status_code=400, - detail="Maximum number of task UIDs per request is 10000", - ) - - async def _key_iter() -> AsyncIterator[str]: - for uid in request.uids: - yield f"{KARTON_TASK_NAMESPACE}:{uid}" - - tasks: list[Task] = [] - async for task in gateway_backend._iter_tasks( - _key_iter(), chunk_size=1000, parse_resources=False - ): - tasks.append(task) - return [TaskView(**task.to_dict()) for task in tasks] - - -@rest_api_router.get("/tasks/{uid}", dependencies=[Depends(CanInspectKarton)]) -async def get_task_details(uid: str) -> TaskView: - task = await gateway_backend.get_task(uid) - if task is None: - raise HTTPException(status_code=404, detail="Task doesn't exist") - download_urls = await generate_resource_download_urls(task, _allowed_buckets()) - return TaskView(**task.to_dict(), download_urls=download_urls) - - -@rest_api_router.get("/analyses/{uid}", dependencies=[Depends(CanInspectKarton)]) -async def get_analysis_details(uid: str) -> AnalysisView: - binds = {bind.identity for bind in await gateway_backend.get_binds()} - queues: dict[str, list[TaskView]] = {} - async for task in gateway_backend.iter_task_tree(uid): - if task.status == TaskState.FINISHED: - continue - receiver = task.headers.get("receiver") - if receiver is None or receiver not in binds: - continue - queues.setdefault(receiver, []).append(TaskView(**task.to_dict())) - return AnalysisView(uid=uid, queues=queues) - - -@rest_api_router.post("/tasks/{uid}/restart", dependencies=[Depends(CanRestartTask)]) -async def restart_task(uid: str) -> RestartedTask: - task = await gateway_backend.get_task(uid) - if task is None: - raise HTTPException(status_code=404, detail="Task doesn't exist") - new_task = await gateway_backend.restart_task(task) - return RestartedTask(uid=new_task.uid) - - -@rest_api_router.post("/tasks/{uid}/cancel", dependencies=[Depends(CanCancelTask)]) -async def cancel_task(uid: str) -> Response: - task = await gateway_backend.get_task(uid) - if task is None: - raise HTTPException(status_code=404, detail="Task doesn't exist") - await gateway_backend.set_task_status(task, TaskState.FINISHED) - return Response(status_code=204) - - -@rest_api_router.get("/outputs", dependencies=[Depends(CanInspectKarton)]) -async def get_outputs() -> list[ProducerOutput]: - return [ - ProducerOutput.model_validate(output) - for output in await gateway_backend.get_outputs() - ] - - -@varz_router.get("/varz", dependencies=[Depends(CanGetMetrics)]) -async def varz() -> PlainTextResponse: - identities = await gateway_backend.get_online_identities() - - lines = [ - "# HELP karton_replicas Number of online replicas per identity", - "# TYPE karton_replicas gauge", - ] - for identity, services in identities.items(): - lines.append(f'karton_replicas{{identity="{identity}"}} {len(services)}') - - lines += [ - "# HELP karton_tasks Number of tasks per queue, priority and status", - "# TYPE karton_tasks gauge", - ] - - task_counts: dict[tuple[str, str, str], int] = {} - async for task in gateway_backend.iter_all_tasks(parse_resources=False): - receiver = task.headers.get("receiver", "") - key = (receiver, task.priority.value, task.status.value) - task_counts[key] = task_counts.get(key, 0) + 1 - - for (queue, priority, status), count in task_counts.items(): - lines.append( - f'karton_tasks{{queue="{queue}",priority="{priority}",' - f'status="{status}"}} {count}' - ) - - lines += [ - "# HELP karton_metrics Task counters per identity", - "# TYPE karton_metrics gauge", - ] - for metric in KartonMetrics: - metric_label = metric.value.split(".")[-1] - values = await gateway_backend.get_metrics(metric) - for identity, count in values.items(): - lines.append( - f'karton_metrics{{metric="{metric_label}",' - f'identity="{identity}"}} {count}' - ) - - return PlainTextResponse( - "\n".join(lines) + "\n", - media_type="text/plain; version=0.0.4; charset=utf-8", - ) diff --git a/karton/gateway/rest/routes/__init__.py b/karton/gateway/rest/routes/__init__.py new file mode 100644 index 00000000..7c98db6c --- /dev/null +++ b/karton/gateway/rest/routes/__init__.py @@ -0,0 +1,11 @@ +import importlib +import pkgutil + +from ._routers import rest_api_router, varz_router + +# Import every submodule so route decorators register on the routers above. +for _module_info in pkgutil.iter_modules(__path__): + if not _module_info.name.startswith("_"): + importlib.import_module(f"{__name__}.{_module_info.name}") + +__all__ = ["rest_api_router", "varz_router"] diff --git a/karton/gateway/rest/routes/_routers.py b/karton/gateway/rest/routes/_routers.py new file mode 100644 index 00000000..c340fc43 --- /dev/null +++ b/karton/gateway/rest/routes/_routers.py @@ -0,0 +1,4 @@ +from fastapi import APIRouter + +rest_api_router = APIRouter(prefix="/api/v1") +varz_router = APIRouter() diff --git a/karton/gateway/rest/routes/binds.py b/karton/gateway/rest/routes/binds.py new file mode 100644 index 00000000..7bdb6336 --- /dev/null +++ b/karton/gateway/rest/routes/binds.py @@ -0,0 +1,56 @@ +from fastapi import Depends, HTTPException, Response + +from karton.core.__version__ import __version__ +from karton.core.backend import KartonBind +from karton.gateway.backend import gateway_backend +from karton.gateway.rest.auth import ( + CanInspectKarton, + CanRemoveBind, +) +from karton.gateway.rest.models import ( + Bind, +) + +from ._routers import rest_api_router + + +@rest_api_router.get("/binds", dependencies=[Depends(CanInspectKarton)]) +async def get_binds() -> list[Bind]: + return [Bind.model_validate(bind) for bind in await gateway_backend.get_binds()] + + +@rest_api_router.get("/binds/{identity}", dependencies=[Depends(CanInspectKarton)]) +async def get_bind_details(identity: str) -> Bind: + bind = await gateway_backend.get_bind(identity) + if bind is None: + raise HTTPException(status_code=404, detail="Bind doesn't exist") + return Bind.model_validate(bind) + + +@rest_api_router.delete("/binds/{identity}", dependencies=[Depends(CanRemoveBind)]) +async def delete_bind(identity: str) -> Response: + bind = await gateway_backend.get_bind(identity) + if bind is None: + raise HTTPException(status_code=404, detail="Bind doesn't exist") + replicas = await gateway_backend.get_online_identities() + if replicas.get(identity): + raise HTTPException( + status_code=409, + detail=( + "Bind has active replicas that need to be downscaled " + "before it can be deleted" + ), + ) + # Replace the bind with non-persistent bind with empty filters. + # This is "mark for deletion" for Karton System GC + tombstone_bind = KartonBind( + identity=identity, + info=None, + version=__version__, + persistent=False, + filters=[], + service_version=None, + is_async=False, + ) + await gateway_backend.register_bind(tombstone_bind, bind_backend=False) + return Response(status_code=204) diff --git a/karton/gateway/rest/routes/outputs.py b/karton/gateway/rest/routes/outputs.py new file mode 100644 index 00000000..337b3859 --- /dev/null +++ b/karton/gateway/rest/routes/outputs.py @@ -0,0 +1,15 @@ +from fastapi import Depends + +from karton.gateway.backend import gateway_backend +from karton.gateway.rest.auth import CanInspectKarton +from karton.gateway.rest.models import ProducerOutput + +from ._routers import rest_api_router + + +@rest_api_router.get("/outputs", dependencies=[Depends(CanInspectKarton)]) +async def get_outputs() -> list[ProducerOutput]: + return [ + ProducerOutput.model_validate(output) + for output in await gateway_backend.get_outputs() + ] diff --git a/karton/gateway/rest/routes/queues.py b/karton/gateway/rest/routes/queues.py new file mode 100644 index 00000000..bfa78648 --- /dev/null +++ b/karton/gateway/rest/routes/queues.py @@ -0,0 +1,120 @@ +from collections import defaultdict +from typing import Literal + +from fastapi import Depends, HTTPException, Response + +from karton.core.task import Task, TaskState +from karton.gateway.backend import gateway_backend +from karton.gateway.rest.auth import ( + CanCancelTask, + CanInspectKarton, + CanRestartTask, +) +from karton.gateway.rest.models import ( + BulkRestartedTask, + QueueView, +) + +from ._routers import rest_api_router + + +async def _build_queues() -> dict[str, QueueView]: + binds = {bind.identity: bind for bind in await gateway_backend.get_binds()} + replicas = await gateway_backend.get_online_identities() + + pending_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) + crashed_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) + + async for task in gateway_backend.iter_all_tasks(parse_resources=False): + receiver = task.headers.get("receiver") + if receiver is None or receiver not in binds: + continue + if task.status == TaskState.CRASHED: + crashed_uids[receiver].append((task.last_update, task.uid)) + elif task.status not in (TaskState.FINISHED, TaskState.CRASHED): + pending_uids[receiver].append((task.last_update, task.uid)) + + queues: dict[str, QueueView] = {} + for identity, bind in binds.items(): + queue_pending = sorted( + pending_uids.get(identity, []), + key=lambda item: item[0], + reverse=True, + ) + queue_crashed = sorted( + crashed_uids.get(identity, []), + key=lambda item: item[0], + reverse=True, + ) + queues[identity] = QueueView( + identity=bind.identity, + info=bind.info, + version=bind.version, + persistent=bind.persistent, + filters=bind.filters, + service_version=bind.service_version, + is_async=bind.is_async, + replicas=len(replicas.get(identity, [])), + pending_task_uids=[uid for _, uid in queue_pending], + crashed_task_uids=[uid for _, uid in queue_crashed], + ) + return queues + + +async def _get_queue_tasks_by_status(identity: str, status: str) -> list[Task]: + bind = await gateway_backend.get_bind(identity) + if bind is None: + raise HTTPException(status_code=404, detail="Queue doesn't exist") + + tasks: list[Task] = [] + async for task in gateway_backend.iter_all_tasks(parse_resources=False): + if task.headers.get("receiver") != identity: + continue + if status == "crashed": + if task.status == TaskState.CRASHED: + tasks.append(task) + elif status == "pending": + if task.status not in (TaskState.FINISHED, TaskState.CRASHED): + tasks.append(task) + return tasks + + +@rest_api_router.get("/queues", dependencies=[Depends(CanInspectKarton)]) +async def get_queues() -> dict[str, QueueView]: + return await _build_queues() + + +@rest_api_router.get("/queues/{identity}", dependencies=[Depends(CanInspectKarton)]) +async def get_queue_details(identity: str) -> QueueView: + queues = await _build_queues() + if identity not in queues: + raise HTTPException(status_code=404, detail="Queue doesn't exist") + return queues[identity] + + +@rest_api_router.post( + "/queues/{identity}/restart", dependencies=[Depends(CanRestartTask)] +) +async def restart_queue_tasks( + identity: str, + status: Literal["crashed", "pending"] = "crashed", +) -> list[BulkRestartedTask]: + tasks = await _get_queue_tasks_by_status(identity, status) + restarted: list[BulkRestartedTask] = [] + for task in tasks: + new_task = await gateway_backend.restart_task(task) + restarted.append(BulkRestartedTask(old_uid=task.uid, new_uid=new_task.uid)) + return restarted + + +@rest_api_router.post( + "/queues/{identity}/cancel", dependencies=[Depends(CanCancelTask)] +) +async def cancel_queue_tasks( + identity: str, + status: Literal["crashed", "pending"] = "crashed", +) -> Response: + tasks = await _get_queue_tasks_by_status(identity, status) + for task in tasks: + await gateway_backend.set_task_status(task, TaskState.FINISHED) + return Response(status_code=204) diff --git a/karton/gateway/rest/routes/services.py b/karton/gateway/rest/routes/services.py new file mode 100644 index 00000000..0a228fdc --- /dev/null +++ b/karton/gateway/rest/routes/services.py @@ -0,0 +1,26 @@ +from fastapi import Depends + +from karton.gateway.backend import gateway_backend +from karton.gateway.rest.auth import CanInspectKarton +from karton.gateway.rest.models import ServiceInfo + +from ._routers import rest_api_router + + +@rest_api_router.get("/services", dependencies=[Depends(CanInspectKarton)]) +async def get_services() -> list[ServiceInfo]: + services = await gateway_backend.get_online_services(_legacy=False) + counts: dict[tuple[str, str | None, str | None], int] = {} + for service in services: + key = (service.identity, service.karton_version, service.service_version) + counts[key] = counts.get(key, 0) + 1 + return [ + ServiceInfo( + identity=identity, + karton_version=karton_version, + service_version=service_version, + instance_id=None, + active_replicas=replicas, + ) + for (identity, karton_version, service_version), replicas in counts.items() + ] diff --git a/karton/gateway/rest/routes/tasks.py b/karton/gateway/rest/routes/tasks.py new file mode 100644 index 00000000..8cb1b7b7 --- /dev/null +++ b/karton/gateway/rest/routes/tasks.py @@ -0,0 +1,90 @@ +from collections.abc import AsyncIterator + +from fastapi import Depends, HTTPException, Response + +from karton.core.backend.direct import KARTON_TASK_NAMESPACE +from karton.core.task import Task, TaskState +from karton.gateway.backend import gateway_backend +from karton.gateway.config import gateway_config +from karton.gateway.rest.auth import ( + CanCancelTask, + CanInspectKarton, + CanRestartTask, +) +from karton.gateway.rest.models import ( + AnalysisView, + BatchTaskRequest, + RestartedTask, + TaskView, +) +from karton.gateway.task import generate_resource_download_urls + +from ._routers import rest_api_router + + +def _allowed_buckets() -> list[str]: + return [ + gateway_backend.default_bucket_name, + *gateway_config.allowed_foreign_buckets, + ] + + +@rest_api_router.post("/tasks/batch", dependencies=[Depends(CanInspectKarton)]) +async def get_tasks_batch(request: BatchTaskRequest) -> list[TaskView]: + if len(request.uids) > 10000: + raise HTTPException( + status_code=400, + detail="Maximum number of task UIDs per request is 10000", + ) + + async def _key_iter() -> AsyncIterator[str]: + for uid in request.uids: + yield f"{KARTON_TASK_NAMESPACE}:{uid}" + + tasks: list[Task] = [] + async for task in gateway_backend._iter_tasks( + _key_iter(), chunk_size=1000, parse_resources=False + ): + tasks.append(task) + return [TaskView(**task.to_dict()) for task in tasks] + + +@rest_api_router.get("/tasks/{uid}", dependencies=[Depends(CanInspectKarton)]) +async def get_task_details(uid: str) -> TaskView: + task = await gateway_backend.get_task(uid) + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") + download_urls = await generate_resource_download_urls(task, _allowed_buckets()) + return TaskView(**task.to_dict(), download_urls=download_urls) + + +@rest_api_router.get("/analyses/{uid}", dependencies=[Depends(CanInspectKarton)]) +async def get_analysis_details(uid: str) -> AnalysisView: + binds = {bind.identity for bind in await gateway_backend.get_binds()} + queues: dict[str, list[TaskView]] = {} + async for task in gateway_backend.iter_task_tree(uid): + if task.status == TaskState.FINISHED: + continue + receiver = task.headers.get("receiver") + if receiver is None or receiver not in binds: + continue + queues.setdefault(receiver, []).append(TaskView(**task.to_dict())) + return AnalysisView(uid=uid, queues=queues) + + +@rest_api_router.post("/tasks/{uid}/restart", dependencies=[Depends(CanRestartTask)]) +async def restart_task(uid: str) -> RestartedTask: + task = await gateway_backend.get_task(uid) + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") + new_task = await gateway_backend.restart_task(task) + return RestartedTask(uid=new_task.uid) + + +@rest_api_router.post("/tasks/{uid}/cancel", dependencies=[Depends(CanCancelTask)]) +async def cancel_task(uid: str) -> Response: + task = await gateway_backend.get_task(uid) + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") + await gateway_backend.set_task_status(task, TaskState.FINISHED) + return Response(status_code=204) diff --git a/karton/gateway/rest/routes/varz.py b/karton/gateway/rest/routes/varz.py new file mode 100644 index 00000000..67199a5c --- /dev/null +++ b/karton/gateway/rest/routes/varz.py @@ -0,0 +1,57 @@ +from fastapi import Depends +from fastapi.responses import PlainTextResponse + +from karton.core.backend import KartonMetrics +from karton.gateway.backend import gateway_backend +from karton.gateway.rest.auth import ( + CanGetMetrics, +) + +from ._routers import varz_router + + +@varz_router.get("/varz", dependencies=[Depends(CanGetMetrics)]) +async def varz() -> PlainTextResponse: + identities = await gateway_backend.get_online_identities() + + lines = [ + "# HELP karton_replicas Number of online replicas per identity", + "# TYPE karton_replicas gauge", + ] + for identity, services in identities.items(): + lines.append(f'karton_replicas{{identity="{identity}"}} {len(services)}') + + lines += [ + "# HELP karton_tasks Number of tasks per queue, priority and status", + "# TYPE karton_tasks gauge", + ] + + task_counts: dict[tuple[str, str, str], int] = {} + async for task in gateway_backend.iter_all_tasks(parse_resources=False): + receiver = task.headers.get("receiver", "") + key = (receiver, task.priority.value, task.status.value) + task_counts[key] = task_counts.get(key, 0) + 1 + + for (queue, priority, status), count in task_counts.items(): + lines.append( + f'karton_tasks{{queue="{queue}",priority="{priority}",' + f'status="{status}"}} {count}' + ) + + lines += [ + "# HELP karton_metrics Task counters per identity", + "# TYPE karton_metrics gauge", + ] + for metric in KartonMetrics: + metric_label = metric.value.split(".")[-1] + values = await gateway_backend.get_metrics(metric) + for identity, count in values.items(): + lines.append( + f'karton_metrics{{metric="{metric_label}",' + f'identity="{identity}"}} {count}' + ) + + return PlainTextResponse( + "\n".join(lines) + "\n", + media_type="text/plain; version=0.0.4; charset=utf-8", + ) diff --git a/pyproject.toml b/pyproject.toml index b9604d58..07615fdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ karton = "karton.core.main:main" karton-auth = "karton.core.auth.cli:main" [tool.setuptools] -packages = ["karton.core", "karton.core.auth", "karton.core.asyncio", "karton.core.backend", "karton.core.asyncio.backend", "karton.system", "karton.gateway"] +packages = ["karton.core", "karton.core.auth", "karton.core.asyncio", "karton.core.backend", "karton.core.asyncio.backend", "karton.system", "karton.gateway", "karton.gateway.rest", "karton.gateway.rest.routes"] [tool.setuptools.dynamic] version = {attr = "karton.core.__version__.__version__"} From 412784522155f34dc04ec5cadc357f0574cbde52 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Fri, 16 Oct 2026 17:39:55 +0200 Subject: [PATCH 12/21] Refactoring, optimizations (analysis status without all queue fetching), bugfixes (chunking) --- karton/core/asyncio/backend/direct.py | 41 ++------- karton/core/backend/__init__.py | 4 +- karton/core/backend/base.py | 23 ++--- karton/core/backend/direct.py | 20 +---- karton/core/backend/gateway.py | 60 ++++++------- ..._models.py => gateway_inspect_protocol.py} | 4 +- karton/core/inspect.py | 89 ++++++++++++------- karton/gateway/rest/models.py | 10 +-- karton/gateway/rest/routes/analyses.py | 27 ++++++ karton/gateway/rest/routes/binds.py | 10 +-- karton/gateway/rest/routes/queues.py | 1 + karton/gateway/rest/routes/services.py | 8 +- karton/gateway/rest/routes/tasks.py | 15 ---- 13 files changed, 145 insertions(+), 167 deletions(-) rename karton/core/{rest_models.py => gateway_inspect_protocol.py} (96%) create mode 100644 karton/gateway/rest/routes/analyses.py diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index b42b0b6d..fd155573 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -388,53 +388,22 @@ async def _get_online_services(self) -> AsyncIterator[KartonExternalServiceInfo] else: yield service - async def get_online_services( - self, _legacy=True - ) -> list[KartonExternalServiceInfo]: + async def get_online_services(self) -> list[KartonExternalServiceInfo]: """ Gets all online services providing extended service information. - This method stays compatible with <=5.10.0 and doesn't return information - about legacy consumers (without with_service_info) flag. It also nullifies - the instance_id to aggregate KartonServiceInfo objects by equality for - counting the number of instances. New code should not rely on that behavior. - - This method will return complete information about services in >=6.0.0. - You can turn on the new behavior by enabling _legacy=True, although this - argument will be removed as well in future major version. - .. versionadded:: 5.1.0 - :return: List of KartonServiceInfo objects + :return: List of KartonExternalServiceInfo objects """ - if not _legacy: - return [el async for el in self._get_online_services()] - - # Legacy users of this method doesn't expect two things: - # - that karton_version can be None in case of pre-5.10.0 consumers - # - that KartonServiceInfo are not equal per identity+version - # but per instance (e.g. karton-dashboard) - # - # That loop removes instance_id and filters out services without - # karton_version - return [ - KartonExternalServiceInfo( - identity=service.identity, - karton_version=service.karton_version, - service_version=service.service_version, - instance_id=None, - redis_client_info=service.redis_client_info, - ) - async for service in self._get_online_services() - if service.karton_version is not None - ] + return [el async for el in self._get_online_services()] async def get_online_identities(self) -> dict[str, list[KartonExternalServiceInfo]]: """ Returns a dictionary for all online identities with list of - KartonServiceInfo of each instance + KartonExternalServiceInfo of each instance - :return: Dictionary {identity: [list of KartonServiceInfo objects]} + :return: Dictionary {identity: [list of KartonExternalServiceInfo objects]} """ identities: defaultdict[str, list[KartonExternalServiceInfo]] = defaultdict( list diff --git a/karton/core/backend/__init__.py b/karton/core/backend/__init__.py index e955f460..34afdd77 100644 --- a/karton/core/backend/__init__.py +++ b/karton/core/backend/__init__.py @@ -5,8 +5,8 @@ KartonBind, KartonMetrics, KartonOutputs, + KartonQueueState, KartonServiceInfo, - QueueSummary, ) from .direct import KartonBackend from .gateway import KartonGatewayBackend @@ -29,7 +29,7 @@ def get_backend( "KartonGatewayBackend", "KartonMetrics", "KartonOutputs", + "KartonQueueState", "KartonServiceInfo", - "QueueSummary", "get_backend", ] diff --git a/karton/core/backend/base.py b/karton/core/backend/base.py index 58a804e0..ce5a4738 100644 --- a/karton/core/backend/base.py +++ b/karton/core/backend/base.py @@ -180,21 +180,13 @@ class KartonOutputs: @dataclasses.dataclass -class QueueSummary: +class KartonQueueState: """ - Summary of a Karton queue state, returned by - :meth:`KartonBackendProtocol.get_queues`. - - For the direct backend, ``tasks`` is pre-loaded with full Task objects so that - :class:`karton.core.inspect.KartonQueue` can filter them in-memory. - - For the gateway backend, ``tasks`` is ``None`` — full task objects are fetched - lazily via :meth:`KartonBackendProtocol.get_tasks` on first access. + Karton queue state, returned by :meth:`KartonBackendProtocol.get_queues`. """ bind: KartonBind replicas: int - tasks: list[Task] | None = None pending_task_uids: list[str] = dataclasses.field(default_factory=list) crashed_task_uids: list[str] = dataclasses.field(default_factory=list) @@ -373,15 +365,12 @@ def get_bind(self, identity: str) -> KartonBind | None: :return: KartonBind object or None if not found """ - def get_online_services( - self, _legacy: bool = True - ) -> list[KartonExternalServiceInfo]: + def get_online_services(self) -> list[KartonExternalServiceInfo]: """ Get all online services providing extended service information. - :param _legacy: | - If True (default), returns services in a legacy-compatible format - (nullified instance_id, filtered to only those with karton_version). + .. versionadded:: 5.1.0 + :return: List of KartonExternalServiceInfo objects """ @@ -393,7 +382,7 @@ def get_online_identities(self) -> dict[str, list[KartonExternalServiceInfo]]: :return: Dictionary {identity: [list of service info objects]} """ - def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: + def get_queues(self, parse_resources: bool = False) -> dict[str, KartonQueueState]: """ Get a summary of all queues (binds) and their task state. diff --git a/karton/core/backend/direct.py b/karton/core/backend/direct.py index a81e70f1..cabcd3c5 100644 --- a/karton/core/backend/direct.py +++ b/karton/core/backend/direct.py @@ -35,8 +35,8 @@ KartonExternalServiceInfo, KartonMetrics, KartonOutputs, + KartonQueueState, KartonServiceInfo, - QueueSummary, unserialize_bind, ) @@ -433,20 +433,9 @@ def _get_online_services(self) -> Iterator[KartonExternalServiceInfo]: yield service def get_online_services(self) -> list[KartonExternalServiceInfo]: - """ - Gets all online services providing extended service information. - - :return: List of KartonServiceInfo objects - """ return list(self._get_online_services()) def get_online_identities(self) -> dict[str, list[KartonExternalServiceInfo]]: - """ - Returns a dictionary for all online identities with list of - KartonServiceInfo of each instance - - :return: Dictionary {identity: [list of KartonServiceInfo objects]} - """ identities: defaultdict[str, list[KartonExternalServiceInfo]] = defaultdict( list ) @@ -1117,7 +1106,7 @@ def get_outputs(self) -> list[KartonOutputs]: for identity in output_keys ] - def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: + def get_queues(self, parse_resources: bool = False) -> dict[str, KartonQueueState]: """ Get a summary of all queues and their task state. @@ -1149,12 +1138,11 @@ def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: else: pending_uids[receiver].append(task.uid) - queues: dict[str, QueueSummary] = {} + queues: dict[str, KartonQueueState] = {} for identity, bind in binds.items(): - queues[identity] = QueueSummary( + queues[identity] = KartonQueueState( bind=bind, replicas=len(replicas.get(identity, [])), - tasks=tasks_per_queue.get(identity, []), pending_task_uids=pending_uids.get(identity, []), crashed_task_uids=crashed_uids.get(identity, []), ) diff --git a/karton/core/backend/gateway.py b/karton/core/backend/gateway.py index 8afa44d7..78cba39f 100644 --- a/karton/core/backend/gateway.py +++ b/karton/core/backend/gateway.py @@ -19,6 +19,17 @@ OperationTimeoutError, SyncGatewayClient, ) +from karton.core.gateway_inspect_protocol import ( + AnalysisView, + BatchTaskRequest, + BindView, + BulkRestartedTask, + ProducerOutput, + QueueView, + RestartedTask, + ServiceInfoView, + TaskView, +) from karton.core.gateway_protocol import ( BindRequest, BindRequestMessage, @@ -52,19 +63,8 @@ ResourceBase, ResourceData, ) -from karton.core.rest_models import ( - AnalysisView, - BatchTaskRequest, - BulkRestartedTask, - ProducerOutput, - QueueView, - RestartedTask, - ServiceInfo, - TaskView, -) -from karton.core.rest_models import Bind as BindView from karton.core.task import Task, TaskPriority, TaskState, root_uid_from_task_uid -from karton.core.utils import recursive_map +from karton.core.utils import chunks, recursive_map from .base import ( KartonBackendProtocol, @@ -72,8 +72,8 @@ KartonExternalServiceInfo, KartonMetrics, KartonOutputs, + KartonQueueState, KartonServiceInfo, - QueueSummary, ) logger = logging.getLogger(__name__) @@ -514,7 +514,7 @@ def _bind_from_view(view: BindView) -> KartonBind: @staticmethod def _service_info_from_view( - view: ServiceInfo, + view: ServiceInfoView, ) -> list[KartonExternalServiceInfo]: return [ KartonExternalServiceInfo( @@ -540,15 +540,11 @@ def get_bind(self, identity: str) -> KartonBind | None: response.raise_for_status() return self._bind_from_view(BindView.model_validate_json(response.content)) - def get_online_services( - self, _legacy: bool = True - ) -> list[KartonExternalServiceInfo]: + def get_online_services(self) -> list[KartonExternalServiceInfo]: response = self._http_client.get("/services") response.raise_for_status() services: list[KartonExternalServiceInfo] = [] - for s in TypeAdapter(list[ServiceInfo]).validate_json(response.content): - if _legacy and s.karton_version is None: - continue + for s in TypeAdapter(list[ServiceInfoView]).validate_json(response.content): services.extend(self._service_info_from_view(s)) return services @@ -556,16 +552,16 @@ def get_online_identities(self) -> dict[str, list[KartonExternalServiceInfo]]: response = self._http_client.get("/services") response.raise_for_status() identities: dict[str, list[KartonExternalServiceInfo]] = defaultdict(list) - for s in TypeAdapter(list[ServiceInfo]).validate_json(response.content): + for s in TypeAdapter(list[ServiceInfoView]).validate_json(response.content): identities[s.identity].extend(self._service_info_from_view(s)) return dict(identities) - def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: + def get_queues(self, parse_resources: bool = False) -> dict[str, KartonQueueState]: response = self._http_client.get("/queues") response.raise_for_status() raw = TypeAdapter(dict[str, QueueView]).validate_json(response.content) return { - identity: QueueSummary( + identity: KartonQueueState( bind=KartonBind( identity=qv.identity, info=qv.info, @@ -576,7 +572,6 @@ def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: is_async=qv.is_async, ), replicas=qv.replicas, - tasks=None, pending_task_uids=qv.pending_task_uids, crashed_task_uids=qv.crashed_task_uids, ) @@ -586,13 +581,16 @@ def get_queues(self, parse_resources: bool = False) -> dict[str, QueueSummary]: def get_tasks(self, task_uids: list[str]) -> list[Task]: if not task_uids: return [] - response = self._http_client.post( - "/tasks/batch", - json=BatchTaskRequest(uids=task_uids).model_dump(), - ) - response.raise_for_status() - views = TypeAdapter(list[TaskView]).validate_json(response.content) - return [self._task_from_view(view) for view in views] + tasks: list[Task] = [] + for chunk in chunks(task_uids, 10000): + response = self._http_client.post( + "/tasks/batch", + json=BatchTaskRequest(uids=list(chunk)).model_dump(), + ) + response.raise_for_status() + views = TypeAdapter(list[TaskView]).validate_json(response.content) + tasks.extend(self._task_from_view(view) for view in views) + return tasks def get_analysis_tasks(self, root_uid: str) -> dict[str, list[Task]]: response = self._http_client.get(f"/analyses/{root_uid}") diff --git a/karton/core/rest_models.py b/karton/core/gateway_inspect_protocol.py similarity index 96% rename from karton/core/rest_models.py rename to karton/core/gateway_inspect_protocol.py index 3dee0c04..18d0b771 100644 --- a/karton/core/rest_models.py +++ b/karton/core/gateway_inspect_protocol.py @@ -6,7 +6,7 @@ from karton.core.task import TaskPriority, TaskState -class Bind(BaseModel): +class BindView(BaseModel): model_config = ConfigDict(from_attributes=True) identity: str @@ -18,7 +18,7 @@ class Bind(BaseModel): is_async: bool -class ServiceInfo(BaseModel): +class ServiceInfoView(BaseModel): identity: str karton_version: str | None service_version: str | None diff --git a/karton/core/inspect.py b/karton/core/inspect.py index 70991a11..7c9efbf7 100644 --- a/karton/core/inspect.py +++ b/karton/core/inspect.py @@ -1,6 +1,6 @@ from collections import defaultdict -from .backend import KartonBackendProtocol, QueueSummary +from .backend import KartonBackendProtocol, KartonBind from .task import Task, TaskState @@ -12,16 +12,41 @@ class KartonQueue: :param state: :class:`KartonState` object to be used for lazy loading """ - def __init__(self, summary: QueueSummary, state: "KartonState") -> None: - self.bind = summary.bind + def __init__( + self, + bind: KartonBind, + replicas: int, + state: "KartonState", + tasks: list[Task] | None = None, + pending_task_uids: list[str] | None = None, + crashed_task_uids: list[str] | None = None, + ) -> None: + self.bind = bind + self.replicas = replicas self.state = state - self._replicas = summary.replicas - self._tasks: list[Task] | None = summary.tasks - self.pending_task_uids = summary.pending_task_uids - self.crashed_task_uids = summary.crashed_task_uids + self.pending_task_uids: list[str] = pending_task_uids or [] + self.crashed_task_uids: list[str] = crashed_task_uids or [] self._pending_tasks: list[Task] | None = None self._crashed_tasks: list[Task] | None = None + if tasks is not None: + self._pending_tasks = self._filter_pending_tasks(tasks) + self._crashed_tasks = self._filter_crashed_tasks(tasks) + self.pending_task_uids = [task.uid for task in self._pending_tasks] + self.crashed_task_uids = [task.uid for task in self._crashed_tasks] + + @staticmethod + def _filter_pending_tasks(tasks: list[Task]) -> list[Task]: + return [ + task + for task in tasks + if task.status not in (TaskState.FINISHED, TaskState.CRASHED) + ] + + @staticmethod + def _filter_crashed_tasks(tasks: list[Task]) -> list[Task]: + return [task for task in tasks if task.status == TaskState.CRASHED] + @property def last_update(self) -> float: """Get the last task update from this queue""" @@ -30,7 +55,7 @@ def last_update(self) -> float: @property def online_consumers_count(self) -> int: """Get number of consumers listening on this queue""" - return self._replicas + return self.replicas @property def tasks(self) -> list[Task]: @@ -40,38 +65,30 @@ def tasks(self) -> list[Task]: For the direct backend, tasks are pre-loaded. For the gateway backend, tasks are fetched lazily on first access. """ - if self._tasks is not None: - return self._tasks + if self._pending_tasks is None and self._crashed_tasks is None: + # Load everything in one request + tasks = self.state.backend.get_tasks( + self.pending_task_uids + self.crashed_task_uids + ) + self._pending_tasks = self._filter_pending_tasks(tasks) + self._crashed_tasks = self._filter_crashed_tasks(tasks) + return self.pending_tasks + self.crashed_tasks @property def pending_tasks(self) -> list[Task]: """Get queue pending tasks""" - if self._tasks is not None: - return [ - task - for task in self._tasks - if task.status not in (TaskState.FINISHED, TaskState.CRASHED) - ] if self._pending_tasks is None: tasks = self.state.backend.get_tasks(self.pending_task_uids) - self._pending_tasks = [ - task - for task in tasks - if task.status not in (TaskState.FINISHED, TaskState.CRASHED) - ] + self._pending_tasks = self._filter_pending_tasks(tasks) return self._pending_tasks @property def crashed_tasks(self) -> list[Task]: """Get queue crashed tasks""" - if self._tasks is not None: - return [task for task in self._tasks if task.status == TaskState.CRASHED] if self._crashed_tasks is None: tasks = self.state.backend.get_tasks(self.crashed_task_uids) - self._crashed_tasks = [ - task for task in tasks if task.status == TaskState.CRASHED - ] + self._crashed_tasks = self._filter_crashed_tasks(tasks) return self._crashed_tasks @@ -143,12 +160,10 @@ def get_queues_for_tasks( return { queue_name: KartonQueue( - summary=QueueSummary( - bind=state.binds[queue_name], - replicas=len(state.replicas.get(queue_name, [])), - tasks=tasks, - ), + bind=state.binds[queue_name], + replicas=len(state.replicas.get(queue_name, [])), state=state, + tasks=tasks, ) for queue_name, tasks in tasks_per_queue.items() } @@ -189,10 +204,16 @@ def queues(self) -> dict[str, KartonQueue]: are included with empty task lists. """ if self._queues is None: - summaries = self.backend.get_queues(parse_resources=self.parse_resources) + queues_state = self.backend.get_queues(parse_resources=self.parse_resources) self._queues = { - identity: KartonQueue(summary=summary, state=self) - for identity, summary in summaries.items() + identity: KartonQueue( + bind=queue_state.bind, + replicas=queue_state.replicas, + state=self, + pending_task_uids=queue_state.pending_task_uids, + crashed_task_uids=queue_state.crashed_task_uids, + ) + for identity, queue_state in queues_state.items() } return self._queues diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py index 996f8d9f..fb401a18 100644 --- a/karton/gateway/rest/models.py +++ b/karton/gateway/rest/models.py @@ -1,13 +1,13 @@ -from karton.core.rest_models import ( +from karton.core.gateway_inspect_protocol import ( AnalysisView, APIError, BatchTaskRequest, - Bind, + BindView, BulkRestartedTask, ProducerOutput, QueueView, RestartedTask, - ServiceInfo, + ServiceInfoView, TaskView, ) @@ -15,11 +15,11 @@ "APIError", "AnalysisView", "BatchTaskRequest", - "Bind", + "BindView", "BulkRestartedTask", "ProducerOutput", "QueueView", "RestartedTask", - "ServiceInfo", + "ServiceInfoView", "TaskView", ] diff --git a/karton/gateway/rest/routes/analyses.py b/karton/gateway/rest/routes/analyses.py new file mode 100644 index 00000000..6926379c --- /dev/null +++ b/karton/gateway/rest/routes/analyses.py @@ -0,0 +1,27 @@ +from fastapi import Depends + +from karton.core.task import TaskState +from karton.gateway.backend import gateway_backend +from karton.gateway.rest.auth import ( + CanInspectKarton, +) +from karton.gateway.rest.models import ( + AnalysisView, + TaskView, +) + +from ._routers import rest_api_router + + +@rest_api_router.get("/analyses/{uid}", dependencies=[Depends(CanInspectKarton)]) +async def get_analysis_details(uid: str) -> AnalysisView: + binds = {bind.identity for bind in await gateway_backend.get_binds()} + queues: dict[str, list[TaskView]] = {} + async for task in gateway_backend.iter_task_tree(uid): + if task.status == TaskState.FINISHED: + continue + receiver = task.headers.get("receiver") + if receiver is None or receiver not in binds: + continue + queues.setdefault(receiver, []).append(TaskView(**task.to_dict())) + return AnalysisView(uid=uid, queues=queues) diff --git a/karton/gateway/rest/routes/binds.py b/karton/gateway/rest/routes/binds.py index 7bdb6336..1b563f5f 100644 --- a/karton/gateway/rest/routes/binds.py +++ b/karton/gateway/rest/routes/binds.py @@ -8,23 +8,23 @@ CanRemoveBind, ) from karton.gateway.rest.models import ( - Bind, + BindView, ) from ._routers import rest_api_router @rest_api_router.get("/binds", dependencies=[Depends(CanInspectKarton)]) -async def get_binds() -> list[Bind]: - return [Bind.model_validate(bind) for bind in await gateway_backend.get_binds()] +async def get_binds() -> list[BindView]: + return [BindView.model_validate(bind) for bind in await gateway_backend.get_binds()] @rest_api_router.get("/binds/{identity}", dependencies=[Depends(CanInspectKarton)]) -async def get_bind_details(identity: str) -> Bind: +async def get_bind_details(identity: str) -> BindView: bind = await gateway_backend.get_bind(identity) if bind is None: raise HTTPException(status_code=404, detail="Bind doesn't exist") - return Bind.model_validate(bind) + return BindView.model_validate(bind) @rest_api_router.delete("/binds/{identity}", dependencies=[Depends(CanRemoveBind)]) diff --git a/karton/gateway/rest/routes/queues.py b/karton/gateway/rest/routes/queues.py index bfa78648..9e0eb6cc 100644 --- a/karton/gateway/rest/routes/queues.py +++ b/karton/gateway/rest/routes/queues.py @@ -25,6 +25,7 @@ async def _build_queues() -> dict[str, QueueView]: pending_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) crashed_uids: dict[str, list[tuple[float, str]]] = defaultdict(list) + # The future planned optimization here is to avoid task deserialization at all async for task in gateway_backend.iter_all_tasks(parse_resources=False): receiver = task.headers.get("receiver") if receiver is None or receiver not in binds: diff --git a/karton/gateway/rest/routes/services.py b/karton/gateway/rest/routes/services.py index 0a228fdc..a8c573d9 100644 --- a/karton/gateway/rest/routes/services.py +++ b/karton/gateway/rest/routes/services.py @@ -2,20 +2,20 @@ from karton.gateway.backend import gateway_backend from karton.gateway.rest.auth import CanInspectKarton -from karton.gateway.rest.models import ServiceInfo +from karton.gateway.rest.models import ServiceInfoView from ._routers import rest_api_router @rest_api_router.get("/services", dependencies=[Depends(CanInspectKarton)]) -async def get_services() -> list[ServiceInfo]: - services = await gateway_backend.get_online_services(_legacy=False) +async def get_services() -> list[ServiceInfoView]: + services = await gateway_backend.get_online_services() counts: dict[tuple[str, str | None, str | None], int] = {} for service in services: key = (service.identity, service.karton_version, service.service_version) counts[key] = counts.get(key, 0) + 1 return [ - ServiceInfo( + ServiceInfoView( identity=identity, karton_version=karton_version, service_version=service_version, diff --git a/karton/gateway/rest/routes/tasks.py b/karton/gateway/rest/routes/tasks.py index 8cb1b7b7..3a0e9005 100644 --- a/karton/gateway/rest/routes/tasks.py +++ b/karton/gateway/rest/routes/tasks.py @@ -12,7 +12,6 @@ CanRestartTask, ) from karton.gateway.rest.models import ( - AnalysisView, BatchTaskRequest, RestartedTask, TaskView, @@ -58,20 +57,6 @@ async def get_task_details(uid: str) -> TaskView: return TaskView(**task.to_dict(), download_urls=download_urls) -@rest_api_router.get("/analyses/{uid}", dependencies=[Depends(CanInspectKarton)]) -async def get_analysis_details(uid: str) -> AnalysisView: - binds = {bind.identity for bind in await gateway_backend.get_binds()} - queues: dict[str, list[TaskView]] = {} - async for task in gateway_backend.iter_task_tree(uid): - if task.status == TaskState.FINISHED: - continue - receiver = task.headers.get("receiver") - if receiver is None or receiver not in binds: - continue - queues.setdefault(receiver, []).append(TaskView(**task.to_dict())) - return AnalysisView(uid=uid, queues=queues) - - @rest_api_router.post("/tasks/{uid}/restart", dependencies=[Depends(CanRestartTask)]) async def restart_task(uid: str) -> RestartedTask: task = await gateway_backend.get_task(uid) From ddb6edf363d1685e538caa933d64a4c9329ec56c Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 19 Oct 2026 15:49:16 +0200 Subject: [PATCH 13/21] Add iter_download_resource method --- karton/core/backend/base.py | 18 ++++++++++++----- karton/core/backend/direct.py | 33 ++++++++++-------------------- karton/core/backend/gateway.py | 18 +++++++++++++---- karton/core/inspect.py | 37 ++++++++++++++++------------------ 4 files changed, 55 insertions(+), 51 deletions(-) diff --git a/karton/core/backend/base.py b/karton/core/backend/base.py index ce5a4738..98334688 100644 --- a/karton/core/backend/base.py +++ b/karton/core/backend/base.py @@ -277,6 +277,17 @@ def upload_resource_from_file(self, resource: LocalResource, path: str) -> None: :param path: Path to the object content """ + def iter_download_resource( + self, resource: RemoteResource, chunk_size: int = 64 * 1024 + ) -> Iterator[bytes]: + """ + Download resource object, yielding its content as chunks of bytes. + + :param resource: Resource to download + :param chunk_size: Chunk size in bytes + :return: Iterator of content chunks + """ + def download_resource(self, resource: RemoteResource) -> bytes: """ Download resource object from object storage. @@ -434,14 +445,11 @@ def restart_task(self, task: Task) -> Task: :return: The restarted (new) task object """ - def cancel_task(self, task_uid: str) -> None: + def cancel_task(self, task: Task) -> None: """ Cancel a task by marking it as finished. - Unlike :meth:`set_task_status`, this does not require a task token - and can be used on tasks obtained via inspection. - - :param task_uid: Task identifier to cancel + :param task: Task to be cancelled """ def restart_queue_tasks( diff --git a/karton/core/backend/direct.py b/karton/core/backend/direct.py index cabcd3c5..aa18d191 100644 --- a/karton/core/backend/direct.py +++ b/karton/core/backend/direct.py @@ -21,7 +21,6 @@ from botocore.session import get_session from redis import AuthenticationError, Redis from redis.client import Pipeline -from urllib3.response import HTTPResponse from karton.core.config import Config from karton.core.exceptions import BindExpiredError @@ -946,19 +945,17 @@ def upload_resource_from_file(self, resource: LocalResource, path: str) -> None: with open(path, "rb") as f: self.upload_resource(resource, f) - def get_object(self, bucket: str, object_uid: str) -> HTTPResponse: - """ - Get resource object stream with the content. - - Returned response should be closed after use to release network resources. - To reuse the connection, it's required to call `response.release_conn()` - explicitly. + def iter_download_resource( + self, resource: RemoteResource, chunk_size: int = 64 * 1024 + ) -> Iterator[bytes]: + if resource.bucket is None: + raise RuntimeError( + "Resource object can't be downloaded because its bucket is not set" + ) - :param bucket: Bucket name - :param object_uid: Object identifier - :return: Response object with content - """ - return self.s3.get_object(Bucket=bucket, Key=object_uid)["Body"] + with self.s3.get_object(Bucket=resource.bucket, Key=resource.uid)["Body"] as f: + while chunk := f.read(chunk_size): + yield chunk def download_resource(self, resource: RemoteResource) -> bytes: if resource.bucket is None: @@ -1169,15 +1166,7 @@ def get_analysis_tasks(self, root_uid: str) -> dict[str, list[Task]]: return dict(tasks_per_queue) - def cancel_task(self, task_uid: str) -> None: - """ - Cancel a task by marking it as finished. - - :param task_uid: Task identifier to cancel - """ - task = self.get_task(task_uid) - if task is None: - raise RuntimeError(f"Task {task_uid} doesn't exist") + def cancel_task(self, task: Task) -> None: self.set_task_status(task, TaskState.FINISHED) def restart_queue_tasks( diff --git a/karton/core/backend/gateway.py b/karton/core/backend/gateway.py index 78cba39f..c7234abf 100644 --- a/karton/core/backend/gateway.py +++ b/karton/core/backend/gateway.py @@ -431,13 +431,23 @@ def upload_resource_from_file(self, resource: LocalResource, path: str) -> None: def _get_download_stream( self, resource: RemoteResource ) -> contextlib.AbstractContextManager[httpx2.Response]: + timeout = self.gateway_response_timeout if self.gateway_s3_hostname_override is not None: host, download_url = override_presigned_url( resource.download_url, self.gateway_s3_hostname_override ) - return httpx2.stream("GET", download_url, headers={"Host": host}) + return httpx2.stream( + "GET", download_url, headers={"Host": host}, timeout=timeout + ) else: - return httpx2.stream("GET", resource.download_url) + return httpx2.stream("GET", resource.download_url, timeout=timeout) + + def iter_download_resource( + self, resource: RemoteResource, chunk_size: int = 64 * 1024 + ) -> Iterator[bytes]: + with self._get_download_stream(resource) as response: + response.raise_for_status() + yield from response.iter_bytes(chunk_size) def download_resource(self, resource: RemoteResource) -> bytes: with self._get_download_stream(resource) as response: @@ -617,8 +627,8 @@ def restart_task(self, task: Task) -> Task: raise RuntimeError(f"Restarted task {new_uid} not found") return new_task - def cancel_task(self, task_uid: str) -> None: - response = self._http_client.post(f"/tasks/{task_uid}/cancel") + def cancel_task(self, task: Task) -> None: + response = self._http_client.post(f"/tasks/{task.uid}/cancel") response.raise_for_status() def restart_queue_tasks( diff --git a/karton/core/inspect.py b/karton/core/inspect.py index 7c9efbf7..236df9b7 100644 --- a/karton/core/inspect.py +++ b/karton/core/inspect.py @@ -8,8 +8,12 @@ class KartonQueue: """ View object representing a Karton queue. - :param summary: :class:`QueueSummary` with queue state data + :param bind: :class:`KartonBind` object representing the queue bind + :param replicas: Amount of active consumer replicas :param state: :class:`KartonState` object to be used for lazy loading + :param tasks: List of tasks currently in queue (optional) + :param pending_task_uids: List of pending task UIDs (optional) + :param crashed_task_uids: List of crashed task UIDs (optional) """ def __init__( @@ -26,14 +30,15 @@ def __init__( self.state = state self.pending_task_uids: list[str] = pending_task_uids or [] self.crashed_task_uids: list[str] = crashed_task_uids or [] - self._pending_tasks: list[Task] | None = None - self._crashed_tasks: list[Task] | None = None + self._tasks: list[Task] | None = tasks if tasks is not None: - self._pending_tasks = self._filter_pending_tasks(tasks) - self._crashed_tasks = self._filter_crashed_tasks(tasks) - self.pending_task_uids = [task.uid for task in self._pending_tasks] - self.crashed_task_uids = [task.uid for task in self._crashed_tasks] + self.pending_task_uids = [ + task.uid for task in self._filter_pending_tasks(tasks) + ] + self.crashed_task_uids = [ + task.uid for task in self._filter_crashed_tasks(tasks) + ] @staticmethod def _filter_pending_tasks(tasks: list[Task]) -> list[Task]: @@ -65,31 +70,23 @@ def tasks(self) -> list[Task]: For the direct backend, tasks are pre-loaded. For the gateway backend, tasks are fetched lazily on first access. """ - if self._pending_tasks is None and self._crashed_tasks is None: + if self._tasks is None: # Load everything in one request - tasks = self.state.backend.get_tasks( + self._tasks = self.state.backend.get_tasks( self.pending_task_uids + self.crashed_task_uids ) - self._pending_tasks = self._filter_pending_tasks(tasks) - self._crashed_tasks = self._filter_crashed_tasks(tasks) - return self.pending_tasks + self.crashed_tasks + return self._tasks @property def pending_tasks(self) -> list[Task]: """Get queue pending tasks""" - if self._pending_tasks is None: - tasks = self.state.backend.get_tasks(self.pending_task_uids) - self._pending_tasks = self._filter_pending_tasks(tasks) - return self._pending_tasks + return self._filter_pending_tasks(self.tasks) @property def crashed_tasks(self) -> list[Task]: """Get queue crashed tasks""" - if self._crashed_tasks is None: - tasks = self.state.backend.get_tasks(self.crashed_task_uids) - self._crashed_tasks = self._filter_crashed_tasks(tasks) - return self._crashed_tasks + return self._filter_crashed_tasks(self.tasks) class KartonAnalysis: From 0176677df6a55c47a6c67e81e380783d07dd68cb Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 19 Oct 2026 17:49:10 +0200 Subject: [PATCH 14/21] Fix resource deserialization --- karton/core/asyncio/backend/direct.py | 10 +++-- karton/core/backend/gateway.py | 60 ++++++++++++++++--------- karton/core/gateway_inspect_protocol.py | 5 +-- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index fd155573..f03a3ade 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -194,8 +194,9 @@ async def _iter_tasks( continue yield Task.unserialize( task_data, - parse_resources=parse_resources, - resource_unserializer=self.unserialize_resource, + resource_unserializer=self.unserialize_resource + if parse_resources + else None, ) def iter_all_tasks( @@ -235,8 +236,9 @@ async def _iter_legacy_task_tree( continue task = Task.unserialize( task_data, - parse_resources=parse_resources, - resource_unserializer=self.unserialize_resource, + resource_unserializer=self.unserialize_resource + if parse_resources + else None, ) if task.root_uid == root_uid: yield task diff --git a/karton/core/backend/gateway.py b/karton/core/backend/gateway.py index c7234abf..6fb51c62 100644 --- a/karton/core/backend/gateway.py +++ b/karton/core/backend/gateway.py @@ -44,6 +44,7 @@ LogResponse, LogSentResponse, NewTaskParameters, + ResourceUrl, SendLogRequest, SendLogRequestMessage, SendTaskRequest, @@ -63,7 +64,13 @@ ResourceBase, ResourceData, ) -from karton.core.task import Task, TaskPriority, TaskState, root_uid_from_task_uid +from karton.core.task import ( + Task, + TaskPriority, + TaskState, + map_resource_data, + root_uid_from_task_uid, +) from karton.core.utils import chunks, recursive_map from .base import ( @@ -349,6 +356,24 @@ def produce_unrouted_task(self, task: Task) -> None: expected_response=SuccessResponse, ) + def _deserialize_resources[T]( + self, payload: T, download_urls: list[ResourceUrl] + ) -> T: + grouped_download_urls: dict[ResourceIdentifier, str] = { + (spec.bucket, spec.uid): spec.url for spec in download_urls + } + + def deserialize_resource(resource_data: ResourceData) -> RemoteResource: + return RemoteResource.from_dict( + resource_data, + backend=self, + download_url=grouped_download_urls[ + (resource_data["bucket"], resource_data["uid"]) + ], + ) + + return map_resource_data(payload, mapper=deserialize_resource) + def consume_routed_task(self, identity: str, timeout: int = 5) -> Task | None: if self._bind_id is None: raise RuntimeError("Bug: Tried to consume task without registering bind") @@ -365,23 +390,11 @@ def consume_routed_task(self, identity: str, timeout: int = 5) -> Task | None: return None except GatewayBindExpiredError as e: raise BindExpiredError(e.message) from e - download_urls: dict[ResourceIdentifier, str] = { - (spec.bucket, spec.uid): spec.url for spec in response.message.download_urls - } - - def deserialize_resource(resource_data: ResourceData) -> RemoteResource: - return RemoteResource.from_dict( - resource_data, - backend=self, - download_url=download_urls[ - (resource_data["bucket"], resource_data["uid"]) - ], - ) task_data = response.message.task - payload = deserialize_resources(task_data.payload, deserialize_resource) - payload_persistent = deserialize_resources( - task_data.payload_persistent, deserialize_resource + payload, payload_persistent = self._deserialize_resources( + (task_data.payload, task_data.payload_persistent), + response.message.download_urls, ) return Task( @@ -501,20 +514,23 @@ def increment_metrics(self, metric: KartonMetrics, identity: str) -> None: # This is no-op, Karton gateway manages all metrics return - @staticmethod - def _task_from_view(view: TaskView) -> Task: + def _task_from_view(self, view: TaskView) -> Task: + payload, payload_persistent = self._deserialize_resources( + (view.payload, view.payload_persistent), view.download_urls + ) + return Task( headers=view.headers, headers_persistent=view.headers_persistent, - payload=view.payload, - payload_persistent=view.payload_persistent, - priority=view.priority, + payload=payload, + payload_persistent=payload_persistent, + priority=TaskPriority(view.priority), parent_uid=view.parent_uid, root_uid=view.root_uid, orig_uid=view.orig_uid, uid=view.uid, error=view.error, - _status=view.status, + _status=TaskState(view.status), _last_update=view.last_update, ) diff --git a/karton/core/gateway_inspect_protocol.py b/karton/core/gateway_inspect_protocol.py index 18d0b771..c520601b 100644 --- a/karton/core/gateway_inspect_protocol.py +++ b/karton/core/gateway_inspect_protocol.py @@ -3,7 +3,6 @@ from pydantic import BaseModel, ConfigDict, Field from karton.core.gateway_protocol import ResourceUrl -from karton.core.task import TaskPriority, TaskState class BindView(BaseModel): @@ -44,8 +43,8 @@ class TaskView(BaseModel): root_uid: str parent_uid: str | None orig_uid: str | None - status: TaskState - priority: TaskPriority + status: str + priority: str last_update: float headers: dict[str, Any] headers_persistent: dict[str, Any] From 93cb24fea261f14d26771f66b7ff9bbfc4a4879e Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 19 Oct 2026 19:10:41 +0200 Subject: [PATCH 15/21] Fix bugs --- karton/core/backend/base.py | 8 +++----- karton/core/backend/direct.py | 11 +++-------- karton/core/backend/gateway.py | 10 ++++------ karton/gateway/rest/routes/queues.py | 3 +-- karton/gateway/rest/routes/tasks.py | 18 +++++++++++------- karton/gateway/task.py | 4 ++++ 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/karton/core/backend/base.py b/karton/core/backend/base.py index 98334688..f0d46f21 100644 --- a/karton/core/backend/base.py +++ b/karton/core/backend/base.py @@ -452,16 +452,14 @@ def cancel_task(self, task: Task) -> None: :param task: Task to be cancelled """ - def restart_queue_tasks( + def restart_queue_crashed_tasks( self, identity: str, - status: Literal["crashed", "pending"] = "crashed", ) -> list[tuple[str, str]]: """ - Restart all tasks matching the given status in a queue. + Bulk restart all crashed tasks in a queue. :param identity: Karton service identity (queue name) - :param status: Which tasks to restart (``"crashed"`` or ``"pending"``) :return: List of ``(old_uid, new_uid)`` pairs for each restarted task """ @@ -471,7 +469,7 @@ def cancel_queue_tasks( status: Literal["crashed", "pending"] = "crashed", ) -> None: """ - Cancel all tasks matching the given status in a queue. + Bulk cancel all tasks matching the given status in a queue. :param identity: Karton service identity (queue name) :param status: Which tasks to cancel (``"crashed"`` or ``"pending"``) diff --git a/karton/core/backend/direct.py b/karton/core/backend/direct.py index aa18d191..f392e990 100644 --- a/karton/core/backend/direct.py +++ b/karton/core/backend/direct.py @@ -1169,10 +1169,9 @@ def get_analysis_tasks(self, root_uid: str) -> dict[str, list[Task]]: def cancel_task(self, task: Task) -> None: self.set_task_status(task, TaskState.FINISHED) - def restart_queue_tasks( + def restart_queue_crashed_tasks( self, identity: str, - status: Literal["crashed", "pending"] = "crashed", ) -> list[tuple[str, str]]: """ Restart all tasks matching the given status in a queue. @@ -1185,12 +1184,8 @@ def restart_queue_tasks( for task in self.iter_all_tasks(parse_resources=False): if task.headers.get("receiver") != identity: continue - if status == "crashed": - if task.status != TaskState.CRASHED: - continue - else: - if task.status in (TaskState.FINISHED, TaskState.CRASHED): - continue + if task.status != TaskState.CRASHED: + continue new_task = self.restart_task(task) restarted.append((task.uid, new_task.uid)) return restarted diff --git a/karton/core/backend/gateway.py b/karton/core/backend/gateway.py index 6fb51c62..80637128 100644 --- a/karton/core/backend/gateway.py +++ b/karton/core/backend/gateway.py @@ -367,9 +367,9 @@ def deserialize_resource(resource_data: ResourceData) -> RemoteResource: return RemoteResource.from_dict( resource_data, backend=self, - download_url=grouped_download_urls[ - (resource_data["bucket"], resource_data["uid"]) - ], + download_url=grouped_download_urls.get( + (resource_data["bucket"], resource_data["uid"]), None + ), ) return map_resource_data(payload, mapper=deserialize_resource) @@ -647,14 +647,12 @@ def cancel_task(self, task: Task) -> None: response = self._http_client.post(f"/tasks/{task.uid}/cancel") response.raise_for_status() - def restart_queue_tasks( + def restart_queue_crashed_tasks( self, identity: str, - status: Literal["crashed", "pending"] = "crashed", ) -> list[tuple[str, str]]: response = self._http_client.post( f"/queues/{identity}/restart", - params={"status": status}, ) response.raise_for_status() restarted = TypeAdapter(list[BulkRestartedTask]).validate_json(response.content) diff --git a/karton/gateway/rest/routes/queues.py b/karton/gateway/rest/routes/queues.py index 9e0eb6cc..6739df2a 100644 --- a/karton/gateway/rest/routes/queues.py +++ b/karton/gateway/rest/routes/queues.py @@ -98,9 +98,8 @@ async def get_queue_details(identity: str) -> QueueView: ) async def restart_queue_tasks( identity: str, - status: Literal["crashed", "pending"] = "crashed", ) -> list[BulkRestartedTask]: - tasks = await _get_queue_tasks_by_status(identity, status) + tasks = await _get_queue_tasks_by_status(identity, "crashed") restarted: list[BulkRestartedTask] = [] for task in tasks: new_task = await gateway_backend.restart_task(task) diff --git a/karton/gateway/rest/routes/tasks.py b/karton/gateway/rest/routes/tasks.py index 3a0e9005..a7e80e30 100644 --- a/karton/gateway/rest/routes/tasks.py +++ b/karton/gateway/rest/routes/tasks.py @@ -44,6 +44,8 @@ async def _key_iter() -> AsyncIterator[str]: async for task in gateway_backend._iter_tasks( _key_iter(), chunk_size=1000, parse_resources=False ): + if task.status == TaskState.FINISHED: + continue tasks.append(task) return [TaskView(**task.to_dict()) for task in tasks] @@ -51,17 +53,19 @@ async def _key_iter() -> AsyncIterator[str]: @rest_api_router.get("/tasks/{uid}", dependencies=[Depends(CanInspectKarton)]) async def get_task_details(uid: str) -> TaskView: task = await gateway_backend.get_task(uid) - if task is None: - raise HTTPException(status_code=404, detail="Task doesn't exist") - download_urls = await generate_resource_download_urls(task, _allowed_buckets()) + if task is None or task.status == TaskState.FINISHED: + raise HTTPException(status_code=404, detail="Task doesn't exist or is finished") + download_urls = await generate_resource_download_urls( + task, _allowed_buckets(), omit_disallowed_buckets=True + ) return TaskView(**task.to_dict(), download_urls=download_urls) @rest_api_router.post("/tasks/{uid}/restart", dependencies=[Depends(CanRestartTask)]) async def restart_task(uid: str) -> RestartedTask: task = await gateway_backend.get_task(uid) - if task is None: - raise HTTPException(status_code=404, detail="Task doesn't exist") + if task is None or task.status == TaskState.FINISHED: + raise HTTPException(status_code=404, detail="Task doesn't exist or is finished") new_task = await gateway_backend.restart_task(task) return RestartedTask(uid=new_task.uid) @@ -69,7 +73,7 @@ async def restart_task(uid: str) -> RestartedTask: @rest_api_router.post("/tasks/{uid}/cancel", dependencies=[Depends(CanCancelTask)]) async def cancel_task(uid: str) -> Response: task = await gateway_backend.get_task(uid) - if task is None: - raise HTTPException(status_code=404, detail="Task doesn't exist") + if task is None or task.status == TaskState.FINISHED: + raise HTTPException(status_code=404, detail="Task doesn't exist or is finished") await gateway_backend.set_task_status(task, TaskState.FINISHED) return Response(status_code=204) diff --git a/karton/gateway/task.py b/karton/gateway/task.py index 412c6e41..a834a40c 100644 --- a/karton/gateway/task.py +++ b/karton/gateway/task.py @@ -228,12 +228,16 @@ async def generate_resource_upload_urls( async def generate_resource_download_urls( task: Task, allowed_buckets: list[str], + omit_disallowed_buckets: bool = False, ) -> list[ResourceUrl]: """ Generates a list of presigned download URLs for an incoming task :param task: Task containing resources :param allowed_buckets: Allowed target buckets to use for resource download + :param omit_disallowed_buckets: | + If True (default is False), function just doesn't generate download URLs + for resources referencing disallowed buckets instead of raising InvalidTaskError. :return: List of ResourceUrl objects with presigned download URLs """ resources = {} From 7fa9f615b57a5dc66566c6c35f2c35bb462981f7 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Mon, 19 Oct 2026 19:17:14 +0200 Subject: [PATCH 16/21] Remove usage of private method from KartonAsyncBackend, fix unnecessary resource parsing --- karton/core/asyncio/backend/direct.py | 29 +++++++++++++++++++++++++- karton/gateway/rest/routes/analyses.py | 2 +- karton/gateway/rest/routes/tasks.py | 11 ++-------- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index f03a3ade..4ae0456b 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -3,7 +3,7 @@ import os import time from collections import defaultdict -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterable from typing import IO, Any, cast import aioboto3 @@ -199,6 +199,33 @@ async def _iter_tasks( else None, ) + def iter_tasks( + self, + task_uid_list: Iterable[str], + chunk_size: int = 1000, + parse_resources: bool = True, + ) -> AsyncIterator[Task]: + """ + Get multiple tasks for given identifier list as an iterator + + :param task_uid_list: List of task fully-qualified identifiers + :param chunk_size: Size of chunks passed to the Redis MGET command + :param parse_resources: If set to False, resources are not parsed. + It speeds up deserialization. Read :py:meth:`Task.unserialize` documentation + to learn more. + :return: Iterator with task objects + """ + + async def _key_iter(uid_list: Iterable[str]) -> AsyncIterator[str]: + for uid in uid_list: + yield f"{KARTON_TASK_NAMESPACE}:{uid}" + + return self._iter_tasks( + _key_iter(task_uid_list), + chunk_size=chunk_size, + parse_resources=parse_resources, + ) + def iter_all_tasks( self, chunk_size: int = 1000, parse_resources: bool = True ) -> AsyncIterator[Task]: diff --git a/karton/gateway/rest/routes/analyses.py b/karton/gateway/rest/routes/analyses.py index 6926379c..26749696 100644 --- a/karton/gateway/rest/routes/analyses.py +++ b/karton/gateway/rest/routes/analyses.py @@ -17,7 +17,7 @@ async def get_analysis_details(uid: str) -> AnalysisView: binds = {bind.identity for bind in await gateway_backend.get_binds()} queues: dict[str, list[TaskView]] = {} - async for task in gateway_backend.iter_task_tree(uid): + async for task in gateway_backend.iter_task_tree(uid, parse_resources=False): if task.status == TaskState.FINISHED: continue receiver = task.headers.get("receiver") diff --git a/karton/gateway/rest/routes/tasks.py b/karton/gateway/rest/routes/tasks.py index a7e80e30..fcd4652b 100644 --- a/karton/gateway/rest/routes/tasks.py +++ b/karton/gateway/rest/routes/tasks.py @@ -1,8 +1,5 @@ -from collections.abc import AsyncIterator - from fastapi import Depends, HTTPException, Response -from karton.core.backend.direct import KARTON_TASK_NAMESPACE from karton.core.task import Task, TaskState from karton.gateway.backend import gateway_backend from karton.gateway.config import gateway_config @@ -36,13 +33,9 @@ async def get_tasks_batch(request: BatchTaskRequest) -> list[TaskView]: detail="Maximum number of task UIDs per request is 10000", ) - async def _key_iter() -> AsyncIterator[str]: - for uid in request.uids: - yield f"{KARTON_TASK_NAMESPACE}:{uid}" - tasks: list[Task] = [] - async for task in gateway_backend._iter_tasks( - _key_iter(), chunk_size=1000, parse_resources=False + async for task in gateway_backend.iter_tasks( + request.uids, chunk_size=1000, parse_resources=False ): if task.status == TaskState.FINISHED: continue From 692de31670f7898165c8fda64f49e476adb34013 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Tue, 20 Oct 2026 13:18:13 +0200 Subject: [PATCH 17/21] Implement interface for metrics instead of exposing /varz --- karton/core/asyncio/backend/direct.py | 3 +++ karton/core/backend/base.py | 5 +++++ karton/core/backend/direct.py | 8 +++----- karton/core/backend/gateway.py | 6 ++++++ karton/core/gateway_inspect_protocol.py | 5 +++++ karton/gateway/rest/models.py | 2 ++ karton/gateway/rest/routes/metrics.py | 14 ++++++++++++++ 7 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 karton/gateway/rest/routes/metrics.py diff --git a/karton/core/asyncio/backend/direct.py b/karton/core/asyncio/backend/direct.py index 4ae0456b..a7c804cb 100644 --- a/karton/core/asyncio/backend/direct.py +++ b/karton/core/asyncio/backend/direct.py @@ -601,6 +601,9 @@ async def get_metrics(self, metric: KartonMetrics) -> dict[str, int]: ).items() } + async def get_all_metrics(self) -> dict[KartonMetrics, dict[str, int]]: + return {metric: await self.get_metrics(metric) for metric in KartonMetrics} + async def upload_resource( self, resource: LocalResource, diff --git a/karton/core/backend/base.py b/karton/core/backend/base.py index f0d46f21..73b18f27 100644 --- a/karton/core/backend/base.py +++ b/karton/core/backend/base.py @@ -257,6 +257,11 @@ def increment_metrics(self, metric: KartonMetrics, identity: str) -> None: :param identity: Related Karton service identity """ + def get_all_metrics(self) -> dict[KartonMetrics, dict[str, int]]: + """ + Get a {karton-identity: current-number-of-tasks} mapping for each metric. + """ + def upload_resource( self, resource: LocalResource, diff --git a/karton/core/backend/direct.py b/karton/core/backend/direct.py index f392e990..cdc87cdd 100644 --- a/karton/core/backend/direct.py +++ b/karton/core/backend/direct.py @@ -916,16 +916,14 @@ def increment_metrics_list( p.execute() def get_metrics(self, metric: KartonMetrics) -> dict[str, int]: - """ - Get a {karton-identity: current-number-of-tasks} mapping for a given metric. - - :param metric: Operation metric type - """ return { k: int(v) for k, v in cast(dict[str, str], self.redis.hgetall(metric.value)).items() } + def get_all_metrics(self) -> dict[KartonMetrics, dict[str, int]]: + return {metric: self.get_metrics(metric) for metric in KartonMetrics} + def upload_resource( self, resource: LocalResource, diff --git a/karton/core/backend/gateway.py b/karton/core/backend/gateway.py index 80637128..692593f6 100644 --- a/karton/core/backend/gateway.py +++ b/karton/core/backend/gateway.py @@ -24,6 +24,7 @@ BatchTaskRequest, BindView, BulkRestartedTask, + MetricsView, ProducerOutput, QueueView, RestartedTask, @@ -514,6 +515,11 @@ def increment_metrics(self, metric: KartonMetrics, identity: str) -> None: # This is no-op, Karton gateway manages all metrics return + def get_all_metrics(self) -> dict[KartonMetrics, dict[str, int]]: + response = self._http_client.get("/metrics") + response.raise_for_status() + return MetricsView.model_validate_json(response.content).metrics + def _task_from_view(self, view: TaskView) -> Task: payload, payload_persistent = self._deserialize_resources( (view.payload, view.payload_persistent), view.download_urls diff --git a/karton/core/gateway_inspect_protocol.py b/karton/core/gateway_inspect_protocol.py index c520601b..bce476d7 100644 --- a/karton/core/gateway_inspect_protocol.py +++ b/karton/core/gateway_inspect_protocol.py @@ -2,6 +2,7 @@ from pydantic import BaseModel, ConfigDict, Field +from karton.core.backend import KartonMetrics from karton.core.gateway_protocol import ResourceUrl @@ -59,6 +60,10 @@ class AnalysisView(BaseModel): queues: dict[str, list[TaskView]] +class MetricsView(BaseModel): + metrics: dict[KartonMetrics, dict[str, int]] + + class ProducerOutput(BaseModel): model_config = ConfigDict(from_attributes=True) diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py index fb401a18..c6b55e1c 100644 --- a/karton/gateway/rest/models.py +++ b/karton/gateway/rest/models.py @@ -4,6 +4,7 @@ BatchTaskRequest, BindView, BulkRestartedTask, + MetricsView, ProducerOutput, QueueView, RestartedTask, @@ -17,6 +18,7 @@ "BatchTaskRequest", "BindView", "BulkRestartedTask", + "MetricsView", "ProducerOutput", "QueueView", "RestartedTask", diff --git a/karton/gateway/rest/routes/metrics.py b/karton/gateway/rest/routes/metrics.py new file mode 100644 index 00000000..680fe1c1 --- /dev/null +++ b/karton/gateway/rest/routes/metrics.py @@ -0,0 +1,14 @@ +from fastapi import Depends + +from karton.gateway.backend import gateway_backend +from karton.gateway.rest.auth import ( + CanGetMetrics, +) +from karton.gateway.rest.models import MetricsView + +from ._routers import rest_api_router + + +@rest_api_router.get("/metrics", dependencies=[Depends(CanGetMetrics)]) +async def get_metrics() -> MetricsView: + return MetricsView(metrics=await gateway_backend.get_all_metrics()) From d4b6eb383bc8fd685c503b069805ed6793164841 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Tue, 20 Oct 2026 14:21:48 +0200 Subject: [PATCH 18/21] Fixes from AI-assisted code review --- karton/core/backend/base.py | 6 +-- karton/core/backend/direct.py | 14 ------ karton/core/gateway_inspect_protocol.py | 1 - karton/gateway/app.py | 3 +- karton/gateway/rest/routes/__init__.py | 4 +- karton/gateway/rest/routes/_routers.py | 1 - karton/gateway/rest/routes/services.py | 1 - karton/gateway/rest/routes/tasks.py | 8 ++-- karton/gateway/rest/routes/varz.py | 57 ------------------------- karton/gateway/task.py | 2 + 10 files changed, 11 insertions(+), 86 deletions(-) delete mode 100644 karton/gateway/rest/routes/varz.py diff --git a/karton/core/backend/base.py b/karton/core/backend/base.py index 73b18f27..479b5047 100644 --- a/karton/core/backend/base.py +++ b/karton/core/backend/base.py @@ -402,10 +402,8 @@ def get_queues(self, parse_resources: bool = False) -> dict[str, KartonQueueStat """ Get a summary of all queues (binds) and their task state. - For the direct backend, full Task objects are included in each - :class:`QueueSummary`. For the gateway backend, only task UIDs are - included and full tasks are fetched lazily. - + :param parse_resources: If False (default), resources are not parsed + to speed up deserialization. :return: Dictionary mapping queue identity to :class:`QueueSummary` """ diff --git a/karton/core/backend/direct.py b/karton/core/backend/direct.py index cdc87cdd..626a2bfe 100644 --- a/karton/core/backend/direct.py +++ b/karton/core/backend/direct.py @@ -1102,22 +1102,9 @@ def get_outputs(self) -> list[KartonOutputs]: ] def get_queues(self, parse_resources: bool = False) -> dict[str, KartonQueueState]: - """ - Get a summary of all queues and their task state. - - Streams all tasks from Redis via :meth:`iter_all_tasks` and groups - them by receiver. Full Task objects are pre-loaded in each - :class:`QueueSummary` for in-memory filtering by - :class:`karton.core.inspect.KartonQueue`. - - :param parse_resources: If False (default), resources are not parsed - to speed up deserialization. - :return: Dictionary mapping queue identity to :class:`QueueSummary` - """ binds = {bind.identity: bind for bind in self.get_binds()} replicas = self.get_online_identities() - tasks_per_queue: dict[str, list[Task]] = defaultdict(list) pending_uids: dict[str, list[str]] = defaultdict(list) crashed_uids: dict[str, list[str]] = defaultdict(list) @@ -1127,7 +1114,6 @@ def get_queues(self, parse_resources: bool = False) -> dict[str, KartonQueueStat continue if task.status == TaskState.FINISHED: continue - tasks_per_queue[receiver].append(task) if task.status == TaskState.CRASHED: crashed_uids[receiver].append(task.uid) else: diff --git a/karton/core/gateway_inspect_protocol.py b/karton/core/gateway_inspect_protocol.py index bce476d7..20a518da 100644 --- a/karton/core/gateway_inspect_protocol.py +++ b/karton/core/gateway_inspect_protocol.py @@ -22,7 +22,6 @@ class ServiceInfoView(BaseModel): identity: str karton_version: str | None service_version: str | None - instance_id: str | None active_replicas: int diff --git a/karton/gateway/app.py b/karton/gateway/app.py index d15c1798..493d159e 100644 --- a/karton/gateway/app.py +++ b/karton/gateway/app.py @@ -7,7 +7,7 @@ from .errors import InternalError, KartonGatewayError, ShutdownError from .logger import set_connection_id, setup_logger from .messages import close_websocket, send_error -from .rest.routes import rest_api_router, varz_router +from .rest.routes import rest_api_router from .session import ClientSession @@ -25,7 +25,6 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan) app.include_router(rest_api_router) -app.include_router(varz_router) async def try_send_error(websocket: WebSocket, error: KartonGatewayError): diff --git a/karton/gateway/rest/routes/__init__.py b/karton/gateway/rest/routes/__init__.py index 7c98db6c..53c0edfa 100644 --- a/karton/gateway/rest/routes/__init__.py +++ b/karton/gateway/rest/routes/__init__.py @@ -1,11 +1,11 @@ import importlib import pkgutil -from ._routers import rest_api_router, varz_router +from ._routers import rest_api_router # Import every submodule so route decorators register on the routers above. for _module_info in pkgutil.iter_modules(__path__): if not _module_info.name.startswith("_"): importlib.import_module(f"{__name__}.{_module_info.name}") -__all__ = ["rest_api_router", "varz_router"] +__all__ = ["rest_api_router"] diff --git a/karton/gateway/rest/routes/_routers.py b/karton/gateway/rest/routes/_routers.py index c340fc43..6c89a49b 100644 --- a/karton/gateway/rest/routes/_routers.py +++ b/karton/gateway/rest/routes/_routers.py @@ -1,4 +1,3 @@ from fastapi import APIRouter rest_api_router = APIRouter(prefix="/api/v1") -varz_router = APIRouter() diff --git a/karton/gateway/rest/routes/services.py b/karton/gateway/rest/routes/services.py index a8c573d9..239c49e3 100644 --- a/karton/gateway/rest/routes/services.py +++ b/karton/gateway/rest/routes/services.py @@ -19,7 +19,6 @@ async def get_services() -> list[ServiceInfoView]: identity=identity, karton_version=karton_version, service_version=service_version, - instance_id=None, active_replicas=replicas, ) for (identity, karton_version, service_version), replicas in counts.items() diff --git a/karton/gateway/rest/routes/tasks.py b/karton/gateway/rest/routes/tasks.py index fcd4652b..737a73f6 100644 --- a/karton/gateway/rest/routes/tasks.py +++ b/karton/gateway/rest/routes/tasks.py @@ -37,8 +37,6 @@ async def get_tasks_batch(request: BatchTaskRequest) -> list[TaskView]: async for task in gateway_backend.iter_tasks( request.uids, chunk_size=1000, parse_resources=False ): - if task.status == TaskState.FINISHED: - continue tasks.append(task) return [TaskView(**task.to_dict()) for task in tasks] @@ -46,8 +44,8 @@ async def get_tasks_batch(request: BatchTaskRequest) -> list[TaskView]: @rest_api_router.get("/tasks/{uid}", dependencies=[Depends(CanInspectKarton)]) async def get_task_details(uid: str) -> TaskView: task = await gateway_backend.get_task(uid) - if task is None or task.status == TaskState.FINISHED: - raise HTTPException(status_code=404, detail="Task doesn't exist or is finished") + if task is None: + raise HTTPException(status_code=404, detail="Task doesn't exist") download_urls = await generate_resource_download_urls( task, _allowed_buckets(), omit_disallowed_buckets=True ) @@ -59,6 +57,8 @@ async def restart_task(uid: str) -> RestartedTask: task = await gateway_backend.get_task(uid) if task is None or task.status == TaskState.FINISHED: raise HTTPException(status_code=404, detail="Task doesn't exist or is finished") + if task.status != TaskState.CRASHED: + raise HTTPException(status_code=409, detail="Task is not in CRASHED state") new_task = await gateway_backend.restart_task(task) return RestartedTask(uid=new_task.uid) diff --git a/karton/gateway/rest/routes/varz.py b/karton/gateway/rest/routes/varz.py deleted file mode 100644 index 67199a5c..00000000 --- a/karton/gateway/rest/routes/varz.py +++ /dev/null @@ -1,57 +0,0 @@ -from fastapi import Depends -from fastapi.responses import PlainTextResponse - -from karton.core.backend import KartonMetrics -from karton.gateway.backend import gateway_backend -from karton.gateway.rest.auth import ( - CanGetMetrics, -) - -from ._routers import varz_router - - -@varz_router.get("/varz", dependencies=[Depends(CanGetMetrics)]) -async def varz() -> PlainTextResponse: - identities = await gateway_backend.get_online_identities() - - lines = [ - "# HELP karton_replicas Number of online replicas per identity", - "# TYPE karton_replicas gauge", - ] - for identity, services in identities.items(): - lines.append(f'karton_replicas{{identity="{identity}"}} {len(services)}') - - lines += [ - "# HELP karton_tasks Number of tasks per queue, priority and status", - "# TYPE karton_tasks gauge", - ] - - task_counts: dict[tuple[str, str, str], int] = {} - async for task in gateway_backend.iter_all_tasks(parse_resources=False): - receiver = task.headers.get("receiver", "") - key = (receiver, task.priority.value, task.status.value) - task_counts[key] = task_counts.get(key, 0) + 1 - - for (queue, priority, status), count in task_counts.items(): - lines.append( - f'karton_tasks{{queue="{queue}",priority="{priority}",' - f'status="{status}"}} {count}' - ) - - lines += [ - "# HELP karton_metrics Task counters per identity", - "# TYPE karton_metrics gauge", - ] - for metric in KartonMetrics: - metric_label = metric.value.split(".")[-1] - values = await gateway_backend.get_metrics(metric) - for identity, count in values.items(): - lines.append( - f'karton_metrics{{metric="{metric_label}",' - f'identity="{identity}"}} {count}' - ) - - return PlainTextResponse( - "\n".join(lines) + "\n", - media_type="text/plain; version=0.0.4; charset=utf-8", - ) diff --git a/karton/gateway/task.py b/karton/gateway/task.py index a834a40c..3a5deaa7 100644 --- a/karton/gateway/task.py +++ b/karton/gateway/task.py @@ -252,6 +252,8 @@ async def generate_resource_download_urls( # Already processed continue if bucket not in allowed_buckets: + if omit_disallowed_buckets: + continue raise InvalidTaskError( f"Task is referencing a bucket '{bucket}' " f"but gateway server is not allowed to access it." From 885338971d4a3b09d57107d7812f912a1c723770 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Tue, 20 Oct 2026 16:33:25 +0200 Subject: [PATCH 19/21] Change gateway URL convention: ws URI is derived from base HTTP URI --- karton/core/gateway_inspect_protocol.py | 4 ---- karton/gateway/rest/models.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/karton/core/gateway_inspect_protocol.py b/karton/core/gateway_inspect_protocol.py index 20a518da..1a3b1f73 100644 --- a/karton/core/gateway_inspect_protocol.py +++ b/karton/core/gateway_inspect_protocol.py @@ -81,7 +81,3 @@ class BulkRestartedTask(BaseModel): class BatchTaskRequest(BaseModel): uids: list[str] - - -class APIError(BaseModel): - error: str diff --git a/karton/gateway/rest/models.py b/karton/gateway/rest/models.py index c6b55e1c..c1724591 100644 --- a/karton/gateway/rest/models.py +++ b/karton/gateway/rest/models.py @@ -1,6 +1,5 @@ from karton.core.gateway_inspect_protocol import ( AnalysisView, - APIError, BatchTaskRequest, BindView, BulkRestartedTask, @@ -13,7 +12,6 @@ ) __all__ = [ - "APIError", "AnalysisView", "BatchTaskRequest", "BindView", From 34f42be19bf0f5254cb895ab91d32484201008c3 Mon Sep 17 00:00:00 2001 From: psrok1 Date: Tue, 20 Oct 2026 17:30:13 +0200 Subject: [PATCH 20/21] Improve CLI --- karton/core/auth/cli.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/karton/core/auth/cli.py b/karton/core/auth/cli.py index 5faa0d4a..27a9e264 100644 --- a/karton/core/auth/cli.py +++ b/karton/core/auth/cli.py @@ -21,7 +21,8 @@ ConsumeTaskOperation, EnableDebugOperation, ProduceTaskOperation, - RegisterBindOperation, + RegisterBindOperation, RemoveBindOperation, GetMetricsOperation, + RestartTaskOperation, InspectKartonOperation, CancelTaskOperation, ) from karton.core.auth.signer import generate_new_keypair, issue_auth_token from karton.core.auth.verifier import decode_auth_token @@ -353,6 +354,16 @@ def claims_from_args(args: argparse.Namespace) -> AuthClaims: operations.append(ConsumeLogOperation()) if args.debug: operations.append(EnableDebugOperation()) + if args.inspect_karton: + operations.append(InspectKartonOperation()) + if args.restart_task: + operations.append(RestartTaskOperation()) + if args.cancel_task: + operations.append(CancelTaskOperation()) + if args.remove_bind: + operations.append(RemoveBindOperation()) + if args.get_metrics: + operations.append(GetMetricsOperation()) return AuthClaims( identity=args.identity, @@ -597,6 +608,31 @@ def main() -> None: action="store_true", help="Allow the consume_log operation", ) + claims_parser.add_argument( + "--inspect-karton", + action="store_true", + help="Allow the inspect_karton operation", + ) + claims_parser.add_argument( + "--restart-task", + action="store_true", + help="Allow the restart_task operation", + ) + claims_parser.add_argument( + "--cancel-task", + action="store_true", + help="Allow the cancel_task operation", + ) + claims_parser.add_argument( + "--remove-bind", + action="store_true", + help="Allow the remove_bind operation", + ) + claims_parser.add_argument( + "--get-metrics", + action="store_true", + help="Allow the get_metrics operation", + ) claims_parser.add_argument( "--register-bind", action="append", From ec31868460ab602fb0277a66bdbbed617504327f Mon Sep 17 00:00:00 2001 From: psrok1 Date: Tue, 20 Oct 2026 17:30:34 +0200 Subject: [PATCH 21/21] Fix CLI and gateway configuration --- karton/core/auth/cli.py | 8 ++++++-- karton/core/auth/models.py | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/karton/core/auth/cli.py b/karton/core/auth/cli.py index 27a9e264..cd591b08 100644 --- a/karton/core/auth/cli.py +++ b/karton/core/auth/cli.py @@ -17,12 +17,16 @@ from karton.core.auth.models import ( AllowedOperation, AuthClaims, + CancelTaskOperation, ConsumeLogOperation, ConsumeTaskOperation, EnableDebugOperation, + GetMetricsOperation, + InspectKartonOperation, ProduceTaskOperation, - RegisterBindOperation, RemoveBindOperation, GetMetricsOperation, - RestartTaskOperation, InspectKartonOperation, CancelTaskOperation, + RegisterBindOperation, + RemoveBindOperation, + RestartTaskOperation, ) from karton.core.auth.signer import generate_new_keypair, issue_auth_token from karton.core.auth.verifier import decode_auth_token diff --git a/karton/core/auth/models.py b/karton/core/auth/models.py index 0063ad14..f03397a2 100644 --- a/karton/core/auth/models.py +++ b/karton/core/auth/models.py @@ -105,7 +105,6 @@ class GetMetricsOperation(Operation): operation: Literal["get_metrics"] = "get_metrics" - type AllowedOperation = ( RegisterBindOperation | ConsumeTaskOperation