More Health
This commit is contained in:
parent
c8aeb3759f
commit
7cb2eed79d
@ -1,7 +1,9 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
package no.iktdev.mediaprocessing.coordinator.controller
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.CoordinatorHealth
|
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.CoordinatorHealthService
|
import no.iktdev.mediaprocessing.coordinator.services.CoordinatorHealthService
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
|
||||||
import org.springframework.web.bind.annotation.GetMapping
|
import org.springframework.web.bind.annotation.GetMapping
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
import org.springframework.web.bind.annotation.RequestMapping
|
||||||
import org.springframework.web.bind.annotation.RestController
|
import org.springframework.web.bind.annotation.RestController
|
||||||
@ -16,4 +18,10 @@ class HealthController(
|
|||||||
fun getHealth(): CoordinatorHealth {
|
fun getHealth(): CoordinatorHealth {
|
||||||
return healthService.getHealth()
|
return healthService.getHealth()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping("/health/events")
|
||||||
|
fun getEventRate(): EventRate = healthService.getEventRate()
|
||||||
|
|
||||||
|
@GetMapping("/health/storage")
|
||||||
|
fun getDiskStatus(): List<DiskInfo> = healthService.getDiskHealth()
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,8 +3,8 @@ package no.iktdev.mediaprocessing.coordinator.controller
|
|||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.EventService
|
import no.iktdev.mediaprocessing.coordinator.services.EventService
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.TaskService
|
import no.iktdev.mediaprocessing.coordinator.services.TaskService
|
||||||
import no.iktdev.mediaprocessing.coordinator.translateDto.CoordinatorTaskTransferDto
|
import no.iktdev.mediaprocessing.coordinator.dto.translate.CoordinatorTaskTransferDto
|
||||||
import no.iktdev.mediaprocessing.coordinator.translateDto.toCoordinatorTransferDto
|
import no.iktdev.mediaprocessing.coordinator.dto.translate.toCoordinatorTransferDto
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.util.UtcNow
|
import no.iktdev.mediaprocessing.ffmpeg.util.UtcNow
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
|
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.ResetTaskResponse
|
import no.iktdev.mediaprocessing.shared.common.dto.ResetTaskResponse
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto
|
package no.iktdev.mediaprocessing.coordinator.dto.health
|
||||||
|
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
|
|
||||||
@ -8,6 +8,7 @@ data class CoordinatorHealth(
|
|||||||
val stalledTasks: Int,
|
val stalledTasks: Int,
|
||||||
val activeTasks: Int,
|
val activeTasks: Int,
|
||||||
val queuedTasks: Int,
|
val queuedTasks: Int,
|
||||||
|
val failedTasks: Int,
|
||||||
val lastActivity: Instant?,
|
val lastActivity: Instant?,
|
||||||
|
|
||||||
// IDs for UI linking
|
// IDs for UI linking
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto
|
package no.iktdev.mediaprocessing.coordinator.dto.health
|
||||||
|
|
||||||
enum class CoordinatorHealthStatus {
|
enum class CoordinatorHealthStatus {
|
||||||
HEALTHY,
|
HEALTHY,
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto
|
package no.iktdev.mediaprocessing.coordinator.dto.health
|
||||||
|
|
||||||
import java.time.Duration
|
import java.time.Duration
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator.dto.rate
|
||||||
|
|
||||||
|
data class EventRate(
|
||||||
|
val lastMinute: Long,
|
||||||
|
val lastFiveMinutes: Long
|
||||||
|
)
|
||||||
@ -1,4 +1,4 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.translateDto
|
package no.iktdev.mediaprocessing.coordinator.dto.translate
|
||||||
|
|
||||||
import no.iktdev.eventi.models.store.PersistedTask
|
import no.iktdev.eventi.models.store.PersistedTask
|
||||||
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
||||||
@ -1,8 +1,12 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.services
|
package no.iktdev.mediaprocessing.coordinator.services
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.CoordinatorHealth
|
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.CoordinatorHealthStatus
|
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.SequenceHealth
|
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealthStatus
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.dto.health.SequenceHealth
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.util.getDiskInfoFor
|
||||||
import no.iktdev.mediaprocessing.shared.common.rules.EventLifecycleRules
|
import no.iktdev.mediaprocessing.shared.common.rules.EventLifecycleRules
|
||||||
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||||
@ -13,7 +17,8 @@ import java.time.Instant
|
|||||||
@Service
|
@Service
|
||||||
class CoordinatorHealthService(
|
class CoordinatorHealthService(
|
||||||
private val taskService: TaskService,
|
private val taskService: TaskService,
|
||||||
private val eventService: EventService
|
private val eventService: EventService,
|
||||||
|
private val coordinatorEnv: CoordinatorEnv
|
||||||
) {
|
) {
|
||||||
|
|
||||||
fun getHealth(): CoordinatorHealth {
|
fun getHealth(): CoordinatorHealth {
|
||||||
@ -31,6 +36,8 @@ class CoordinatorHealthService(
|
|||||||
.filter { TaskLifecycleRules.isStalled(it) }
|
.filter { TaskLifecycleRules.isStalled(it) }
|
||||||
.map { it.taskId }
|
.map { it.taskId }
|
||||||
|
|
||||||
|
val failedTasks = taskService.getFailedTasks()
|
||||||
|
|
||||||
// --- SEQUENCE HEALTH ---
|
// --- SEQUENCE HEALTH ---
|
||||||
val overdueSequences = incompleteSequences
|
val overdueSequences = incompleteSequences
|
||||||
.filter { EventLifecycleRules.isOverdue(it) }
|
.filter { EventLifecycleRules.isOverdue(it) }
|
||||||
@ -75,8 +82,6 @@ class CoordinatorHealthService(
|
|||||||
else -> CoordinatorHealthStatus.HEALTHY
|
else -> CoordinatorHealthStatus.HEALTHY
|
||||||
}
|
}
|
||||||
|
|
||||||
val eventsLastMinute = eventService.getEventsLast(1)
|
|
||||||
val eventsLastFive = eventService.getEventsLast(5)
|
|
||||||
|
|
||||||
val lastActivityCandidates = listOfNotNull(
|
val lastActivityCandidates = listOfNotNull(
|
||||||
tasks.maxOfOrNull { it.persistedAt },
|
tasks.maxOfOrNull { it.persistedAt },
|
||||||
@ -88,6 +93,7 @@ class CoordinatorHealthService(
|
|||||||
abandonedTasks = abandonedTaskIds.size,
|
abandonedTasks = abandonedTaskIds.size,
|
||||||
stalledTasks = stalledTaskIds.size,
|
stalledTasks = stalledTaskIds.size,
|
||||||
activeTasks = tasks.count { !it.consumed },
|
activeTasks = tasks.count { !it.consumed },
|
||||||
|
failedTasks = failedTasks.count(),
|
||||||
queuedTasks = TaskStore.getPendingTasks().size,
|
queuedTasks = TaskStore.getPendingTasks().size,
|
||||||
lastActivity = lastActivityCandidates.maxOrNull(),
|
lastActivity = lastActivityCandidates.maxOrNull(),
|
||||||
|
|
||||||
@ -99,10 +105,23 @@ class CoordinatorHealthService(
|
|||||||
details = mapOf(
|
details = mapOf(
|
||||||
"oldestActiveTaskAgeMinutes" to tasks.minOfOrNull {
|
"oldestActiveTaskAgeMinutes" to tasks.minOfOrNull {
|
||||||
Duration.between(it.persistedAt, Instant.now()).toMinutes()
|
Duration.between(it.persistedAt, Instant.now()).toMinutes()
|
||||||
},
|
}
|
||||||
"eventsLastMinute" to eventsLastMinute,
|
|
||||||
"eventsLastFiveMinutes" to eventsLastFive,
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun getEventRate(): EventRate {
|
||||||
|
return EventRate(
|
||||||
|
lastMinute = eventService.getEventsLast(1),
|
||||||
|
lastFiveMinutes = eventService.getEventsLast(5)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getDiskHealth(): List<DiskInfo> {
|
||||||
|
val paths = listOf(coordinatorEnv.incomingContent,
|
||||||
|
coordinatorEnv.cachedContent,
|
||||||
|
coordinatorEnv.outgoingContent)
|
||||||
|
.map { it -> it.absolutePath }
|
||||||
|
return getDiskInfoFor(paths)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -28,4 +28,8 @@ class TaskService {
|
|||||||
val resetSuccess = TaskStore.resetTaskById(taskId).isSuccess
|
val resetSuccess = TaskStore.resetTaskById(taskId).isSuccess
|
||||||
return resetSuccess
|
return resetSuccess
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun getFailedTasks(): List<PersistedTask> {
|
||||||
|
return TaskStore.getFailedTasks()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@ -1,35 +1,33 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.util
|
package no.iktdev.mediaprocessing.coordinator.util
|
||||||
|
|
||||||
import java.nio.file.FileSystems
|
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Paths.get
|
import java.nio.file.Paths
|
||||||
|
|
||||||
data class DiskInfo(
|
data class DiskInfo(
|
||||||
val mount: String,
|
val mount: String,
|
||||||
val device: String,
|
val device: String,
|
||||||
val totalBytes: Long,
|
val totalBytes: Long,
|
||||||
val freeBytes: Long
|
val freeBytes: Long,
|
||||||
|
val usedBytes: Long,
|
||||||
|
val usedPercent: Double
|
||||||
)
|
)
|
||||||
|
|
||||||
fun getDiskInfoFor(mounts: List<String>): List<DiskInfo> {
|
|
||||||
val fileStores = FileSystems.getDefault().fileStores
|
|
||||||
|
|
||||||
return mounts.mapNotNull { mount ->
|
fun getDiskInfoFor(mounts: List<String>): List<DiskInfo> =
|
||||||
val path = get(mount)
|
mounts.mapNotNull { mount ->
|
||||||
|
val path = Paths.get(mount)
|
||||||
|
|
||||||
val store = fileStores.find { fs ->
|
val store = runCatching { Files.getFileStore(path) }.getOrNull()
|
||||||
try {
|
?: return@mapNotNull null
|
||||||
Files.getFileStore(path) == fs
|
|
||||||
} catch (e: Exception) {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
} ?: return@mapNotNull null
|
|
||||||
|
|
||||||
DiskInfo(
|
DiskInfo(
|
||||||
mount = mount,
|
mount = mount,
|
||||||
device = store.name(),
|
device = store.name(),
|
||||||
totalBytes = store.totalSpace,
|
totalBytes = store.totalSpace,
|
||||||
freeBytes = store.usableSpace
|
freeBytes = store.usableSpace,
|
||||||
|
usedBytes = store.totalSpace - store.usableSpace,
|
||||||
|
usedPercent = if (store.totalSpace > 0)
|
||||||
|
((store.totalSpace - store.usableSpace).toDouble() / store.totalSpace.toDouble()) * 100
|
||||||
|
else 0.0
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|||||||
@ -218,5 +218,13 @@ object TaskStore: TaskStore {
|
|||||||
}.getOrDefault(emptyList())
|
}.getOrDefault(emptyList())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun getFailedTasks(): List<PersistedTask> {
|
||||||
|
return withTransaction {
|
||||||
|
TasksTable.getWhere {
|
||||||
|
(TasksTable.status eq TaskStatus.Failed)
|
||||||
|
}
|
||||||
|
}.getOrDefault(emptyList())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
Loading…
Reference in New Issue
Block a user