diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/HealthController.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/HealthController.kt index bf9d71f4..bec30c00 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/HealthController.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/HealthController.kt @@ -1,7 +1,9 @@ package no.iktdev.mediaprocessing.coordinator.controller -import no.iktdev.mediaprocessing.coordinator.dto.CoordinatorHealth +import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth +import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate import no.iktdev.mediaprocessing.coordinator.services.CoordinatorHealthService +import no.iktdev.mediaprocessing.coordinator.util.DiskInfo import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RestController @@ -16,4 +18,10 @@ class HealthController( fun getHealth(): CoordinatorHealth { return healthService.getHealth() } + + @GetMapping("/health/events") + fun getEventRate(): EventRate = healthService.getEventRate() + + @GetMapping("/health/storage") + fun getDiskStatus(): List = healthService.getDiskHealth() } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/TaskController.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/TaskController.kt index 7b42fb68..f5c21c2a 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/TaskController.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/TaskController.kt @@ -3,8 +3,8 @@ package no.iktdev.mediaprocessing.coordinator.controller import no.iktdev.mediaprocessing.coordinator.services.EventService import no.iktdev.mediaprocessing.coordinator.services.TaskService -import no.iktdev.mediaprocessing.coordinator.translateDto.CoordinatorTaskTransferDto -import no.iktdev.mediaprocessing.coordinator.translateDto.toCoordinatorTransferDto +import no.iktdev.mediaprocessing.coordinator.dto.translate.CoordinatorTaskTransferDto +import no.iktdev.mediaprocessing.coordinator.dto.translate.toCoordinatorTransferDto import no.iktdev.mediaprocessing.ffmpeg.util.UtcNow import no.iktdev.mediaprocessing.shared.common.dto.Paginated import no.iktdev.mediaprocessing.shared.common.dto.ResetTaskResponse diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/CoordinatorHealth.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/CoordinatorHealth.kt similarity index 85% rename from apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/CoordinatorHealth.kt rename to apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/CoordinatorHealth.kt index 41a9473c..08423ca0 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/CoordinatorHealth.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/CoordinatorHealth.kt @@ -1,4 +1,4 @@ -package no.iktdev.mediaprocessing.coordinator.dto +package no.iktdev.mediaprocessing.coordinator.dto.health import java.time.Instant @@ -8,6 +8,7 @@ data class CoordinatorHealth( val stalledTasks: Int, val activeTasks: Int, val queuedTasks: Int, + val failedTasks: Int, val lastActivity: Instant?, // IDs for UI linking diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/CoordinatorHealthStatus.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/CoordinatorHealthStatus.kt similarity index 58% rename from apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/CoordinatorHealthStatus.kt rename to apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/CoordinatorHealthStatus.kt index 9291b43f..4f40d705 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/CoordinatorHealthStatus.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/CoordinatorHealthStatus.kt @@ -1,4 +1,4 @@ -package no.iktdev.mediaprocessing.coordinator.dto +package no.iktdev.mediaprocessing.coordinator.dto.health enum class CoordinatorHealthStatus { HEALTHY, diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/SequenceHealth.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/SequenceHealth.kt similarity index 86% rename from apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/SequenceHealth.kt rename to apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/SequenceHealth.kt index dd2a6f7d..13ba1ce8 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/SequenceHealth.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/health/SequenceHealth.kt @@ -1,4 +1,4 @@ -package no.iktdev.mediaprocessing.coordinator.dto +package no.iktdev.mediaprocessing.coordinator.dto.health import java.time.Duration import java.time.Instant diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/rate/EventRate.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/rate/EventRate.kt new file mode 100644 index 00000000..fad65116 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/rate/EventRate.kt @@ -0,0 +1,6 @@ +package no.iktdev.mediaprocessing.coordinator.dto.rate + +data class EventRate( + val lastMinute: Long, + val lastFiveMinutes: Long +) diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/translateDto/CoordinatorTaskTransferDto.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/translate/CoordinatorTaskTransferDto.kt similarity index 94% rename from apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/translateDto/CoordinatorTaskTransferDto.kt rename to apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/translate/CoordinatorTaskTransferDto.kt index bdf3d4ce..62c6a649 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/translateDto/CoordinatorTaskTransferDto.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/dto/translate/CoordinatorTaskTransferDto.kt @@ -1,4 +1,4 @@ -package no.iktdev.mediaprocessing.coordinator.translateDto +package no.iktdev.mediaprocessing.coordinator.dto.translate import no.iktdev.eventi.models.store.PersistedTask import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/CoordinatorHealthService.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/CoordinatorHealthService.kt index 6831f594..84d1a79e 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/CoordinatorHealthService.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/CoordinatorHealthService.kt @@ -1,8 +1,12 @@ package no.iktdev.mediaprocessing.coordinator.services -import no.iktdev.mediaprocessing.coordinator.dto.CoordinatorHealth -import no.iktdev.mediaprocessing.coordinator.dto.CoordinatorHealthStatus -import no.iktdev.mediaprocessing.coordinator.dto.SequenceHealth +import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv +import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth +import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealthStatus +import no.iktdev.mediaprocessing.coordinator.dto.health.SequenceHealth +import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate +import no.iktdev.mediaprocessing.coordinator.util.DiskInfo +import no.iktdev.mediaprocessing.coordinator.util.getDiskInfoFor import no.iktdev.mediaprocessing.shared.common.rules.EventLifecycleRules import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules import no.iktdev.mediaprocessing.shared.database.stores.TaskStore @@ -13,7 +17,8 @@ import java.time.Instant @Service class CoordinatorHealthService( private val taskService: TaskService, - private val eventService: EventService + private val eventService: EventService, + private val coordinatorEnv: CoordinatorEnv ) { fun getHealth(): CoordinatorHealth { @@ -31,6 +36,8 @@ class CoordinatorHealthService( .filter { TaskLifecycleRules.isStalled(it) } .map { it.taskId } + val failedTasks = taskService.getFailedTasks() + // --- SEQUENCE HEALTH --- val overdueSequences = incompleteSequences .filter { EventLifecycleRules.isOverdue(it) } @@ -75,8 +82,6 @@ class CoordinatorHealthService( else -> CoordinatorHealthStatus.HEALTHY } - val eventsLastMinute = eventService.getEventsLast(1) - val eventsLastFive = eventService.getEventsLast(5) val lastActivityCandidates = listOfNotNull( tasks.maxOfOrNull { it.persistedAt }, @@ -88,6 +93,7 @@ class CoordinatorHealthService( abandonedTasks = abandonedTaskIds.size, stalledTasks = stalledTaskIds.size, activeTasks = tasks.count { !it.consumed }, + failedTasks = failedTasks.count(), queuedTasks = TaskStore.getPendingTasks().size, lastActivity = lastActivityCandidates.maxOrNull(), @@ -99,10 +105,23 @@ class CoordinatorHealthService( details = mapOf( "oldestActiveTaskAgeMinutes" to tasks.minOfOrNull { Duration.between(it.persistedAt, Instant.now()).toMinutes() - }, - "eventsLastMinute" to eventsLastMinute, - "eventsLastFiveMinutes" to eventsLastFive, + } ) ) } + + fun getEventRate(): EventRate { + return EventRate( + lastMinute = eventService.getEventsLast(1), + lastFiveMinutes = eventService.getEventsLast(5) + ) + } + + fun getDiskHealth(): List { + val paths = listOf(coordinatorEnv.incomingContent, + coordinatorEnv.cachedContent, + coordinatorEnv.outgoingContent) + .map { it -> it.absolutePath } + return getDiskInfoFor(paths) + } } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/TaskService.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/TaskService.kt index f12fb2eb..6d07276b 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/TaskService.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/TaskService.kt @@ -28,4 +28,8 @@ class TaskService { val resetSuccess = TaskStore.resetTaskById(taskId).isSuccess return resetSuccess } + + fun getFailedTasks(): List { + return TaskStore.getFailedTasks() + } } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/util/DiskInfo.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/util/DiskInfo.kt index c44136db..b58c3f4a 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/util/DiskInfo.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/util/DiskInfo.kt @@ -1,35 +1,33 @@ package no.iktdev.mediaprocessing.coordinator.util -import java.nio.file.FileSystems import java.nio.file.Files -import java.nio.file.Paths.get +import java.nio.file.Paths data class DiskInfo( val mount: String, val device: String, val totalBytes: Long, - val freeBytes: Long + val freeBytes: Long, + val usedBytes: Long, + val usedPercent: Double ) -fun getDiskInfoFor(mounts: List): List { - val fileStores = FileSystems.getDefault().fileStores - return mounts.mapNotNull { mount -> - val path = get(mount) +fun getDiskInfoFor(mounts: List): List = + mounts.mapNotNull { mount -> + val path = Paths.get(mount) - val store = fileStores.find { fs -> - try { - Files.getFileStore(path) == fs - } catch (e: Exception) { - false - } - } ?: return@mapNotNull null + val store = runCatching { Files.getFileStore(path) }.getOrNull() + ?: return@mapNotNull null DiskInfo( mount = mount, device = store.name(), totalBytes = store.totalSpace, - freeBytes = store.usableSpace + freeBytes = store.usableSpace, + usedBytes = store.totalSpace - store.usableSpace, + usedPercent = if (store.totalSpace > 0) + ((store.totalSpace - store.usableSpace).toDouble() / store.totalSpace.toDouble()) * 100 + else 0.0 ) } -} diff --git a/shared/database/src/main/kotlin/no/iktdev/mediaprocessing/shared/database/stores/TaskStore.kt b/shared/database/src/main/kotlin/no/iktdev/mediaprocessing/shared/database/stores/TaskStore.kt index b844c5fe..cf070594 100644 --- a/shared/database/src/main/kotlin/no/iktdev/mediaprocessing/shared/database/stores/TaskStore.kt +++ b/shared/database/src/main/kotlin/no/iktdev/mediaprocessing/shared/database/stores/TaskStore.kt @@ -218,5 +218,13 @@ object TaskStore: TaskStore { }.getOrDefault(emptyList()) } + fun getFailedTasks(): List { + return withTransaction { + TasksTable.getWhere { + (TasksTable.status eq TaskStatus.Failed) + } + }.getOrDefault(emptyList()) + } + } \ No newline at end of file