diff --git a/.gitignore b/.gitignore index fe782ac1..711b703d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,10 +36,12 @@ bin/ /.nb-gradle/ ### VS Code ### -.vscode/ + ### Mac OS ### .DS_Store -.idea/runConfigurations \ No newline at end of file +.idea/runConfigurations +/apps/pyMetadata/venv/ +/apps/pyWatcher/venv/ diff --git a/apps/converter/build.gradle.kts b/apps/converter/build.gradle.kts index 10ecd66e..2c03b8aa 100644 --- a/apps/converter/build.gradle.kts +++ b/apps/converter/build.gradle.kts @@ -39,7 +39,7 @@ dependencies { implementation("no.iktdev:exfl:0.0.16-SNAPSHOT") implementation("no.iktdev.library:subtitle:1.8.1-SNAPSHOT") - implementation("no.iktdev:eventi:1.0-rc13") + implementation("no.iktdev:eventi:1.0-rc15") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") @@ -52,6 +52,8 @@ dependencies { testImplementation("io.mockk:mockk:1.12.0") testImplementation("org.springframework.boot:spring-boot-starter-test") + testImplementation(project(":shared:common", configuration = "testArtifacts")) + } tasks.test { diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt index cf24cc34..7468cbc4 100644 --- a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt @@ -6,15 +6,15 @@ import no.iktdev.eventi.tasks.TaskTypeRegistry import no.iktdev.exfl.coroutines.CoroutinesDefault import no.iktdev.exfl.coroutines.CoroutinesIO import no.iktdev.exfl.observable.Observables +import no.iktdev.mediaprocessing.shared.common.DatabaseApplication +import no.iktdev.mediaprocessing.shared.common.MediaProcessingApp import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry -import no.iktdev.mediaprocessing.shared.common.DatabaseApplication import no.iktdev.mediaprocessing.shared.common.getAppVersion -import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.Configuration - +@MediaProcessingApp open class ConverterApplication: DatabaseApplication() { } diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/TaskPoller.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/TaskPoller.kt index a93b1087..85098023 100644 --- a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/TaskPoller.kt +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/TaskPoller.kt @@ -4,7 +4,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import no.iktdev.eventi.models.Event -import no.iktdev.eventi.tasks.AbstractTaskPoller +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.tasks.TaskPollerImplementation import no.iktdev.eventi.tasks.TaskReporter import no.iktdev.mediaprocessing.shared.common.stores.EventStore import no.iktdev.mediaprocessing.shared.common.stores.TaskStore @@ -30,7 +31,7 @@ class PollerAdministrator( @Service class TaskPoller( private val reporter: TaskReporter, -) : AbstractTaskPoller( +) : TaskPollerImplementation( taskStore = TaskStore, reporterFactory = { reporter } // én reporter brukes for alle tasks ) { @@ -49,7 +50,7 @@ class DefaultTaskReporter() : TaskReporter { } override fun markConsumed(taskId: UUID) { - TaskStore.markConsumed(taskId) + TaskStore.markConsumed(taskId, TaskStatus.Completed) } override fun updateProgress(taskId: UUID, progress: Int) { diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt index 5d732cf9..e6a98610 100644 --- a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt @@ -1,7 +1,5 @@ package no.iktdev.mediaprocessing.converter.convert -import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.Data -import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.SubtitleFormats import no.iktdev.library.subtitle.Configuration import no.iktdev.library.subtitle.Syncro import no.iktdev.library.subtitle.classes.Dialog @@ -10,11 +8,12 @@ import no.iktdev.library.subtitle.export.Export import no.iktdev.library.subtitle.reader.BaseReader import no.iktdev.library.subtitle.reader.Reader import no.iktdev.mediaprocessing.converter.ConverterEnv +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask +import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat import java.io.File -import kotlin.jvm.Throws -class Converter2(val data: Data, - private val listener: ConvertListener) { +class Converter2(val data: ConvertTask.Data, + private val listener: ConvertListener) { @Throws(FileUnavailableException::class) private fun getReader(): BaseReader? { @@ -55,13 +54,13 @@ class Converter2(val data: Data, exporter.write(syncOrNotSync) } else { val exported = mutableListOf() - if (data.formats.contains(SubtitleFormats.SRT)) { + if (data.formats.contains(SubtitleFormat.SRT)) { exported.add(exporter.writeSrt(syncOrNotSync)) } - if (data.formats.contains(SubtitleFormats.SMI)) { + if (data.formats.contains(SubtitleFormat.SMI)) { exported.add(exporter.writeSmi(syncOrNotSync)) } - if (data.formats.contains(SubtitleFormats.VTT)) { + if (data.formats.contains(SubtitleFormat.VTT)) { exported.add(exporter.writeVtt(syncOrNotSync)) } exported diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListener.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListener.kt index d0f365bc..ebd9babd 100644 --- a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListener.kt +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListener.kt @@ -8,7 +8,6 @@ import no.iktdev.eventi.tasks.TaskType import no.iktdev.mediaprocessing.converter.convert.ConvertListener import no.iktdev.mediaprocessing.converter.convert.Converter2 import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertedData import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask import org.springframework.stereotype.Component import java.util.* @@ -44,10 +43,10 @@ class ConvertTaskListener: TaskListener(TaskType.CPU_INTENSIVE) { return try { val result = converter.getResult() val newEvent = ConvertTaskResultEvent( - data = ConvertedData( + data = ConvertTaskResultEvent.ConvertedData( language = task.data.language, outputFiles = result, - baseName = task.data.storeFileName + baseName = task.data.outputFileName ), status = TaskStatus.Completed ).producedFrom(task) diff --git a/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplicationTest.kt b/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplicationTest.kt index 6ab0a024..5b3c1e54 100644 --- a/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplicationTest.kt +++ b/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplicationTest.kt @@ -1,21 +1,24 @@ package no.iktdev.mediaprocessing.converter -import io.mockk.junit5.MockKExtension -import mu.KotlinLogging import no.iktdev.eventi.models.Task +import no.iktdev.mediaprocessing.shared.common.TestBase +import no.iktdev.mediaprocessing.shared.common.config.DatasourceConfiguration import no.iktdev.mediaprocessing.shared.common.stores.TaskStore import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith -import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.TestPropertySource import org.springframework.test.context.junit.jupiter.SpringExtension -@SpringBootTest(classes = [ConverterApplication::class]) +@SpringBootTest( + classes = [ConverterApplication::class, + DatasourceConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +@TestPropertySource(properties = ["spring.flyway.enabled=true"]) @ExtendWith(SpringExtension::class) -class ConverterApplicationTest { - private val log = KotlinLogging.logger {} +class ConverterApplicationTest: TestBase() { data class TestTask( val success: Boolean diff --git a/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListenerTest.kt b/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListenerTest.kt new file mode 100644 index 00000000..65e8fe67 --- /dev/null +++ b/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListenerTest.kt @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing.converter.listeners + +class ConvertTaskListenerTest { + + + +} \ No newline at end of file diff --git a/apps/converter/src/test/resources/application.yml b/apps/converter/src/test/resources/application.yml new file mode 100644 index 00000000..3df1331c --- /dev/null +++ b/apps/converter/src/test/resources/application.yml @@ -0,0 +1,28 @@ +spring: + main: + allow-bean-definition-overriding: true + flyway: + enabled: false + locations: classpath:flyway + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration + + output: + ansi: + enabled: always + +springdoc: + swagger-ui: + path: /open/swagger-ui + +logging: + level: + org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN + org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping: DEBUG + +management: + endpoints: + web: + exposure: + include: mappings diff --git a/apps/coordinator/build.gradle.kts b/apps/coordinator/build.gradle.kts index b8dab861..d5529f86 100644 --- a/apps/coordinator/build.gradle.kts +++ b/apps/coordinator/build.gradle.kts @@ -22,7 +22,7 @@ repositories { } -val exposedVersion = "0.44.0" +val exposedVersion = "0.61.0" dependencies { /*Spring boot*/ @@ -39,7 +39,7 @@ dependencies { implementation("no.iktdev:exfl:0.0.16-SNAPSHOT") implementation("no.iktdev.streamit.library:streamit-library-db:1.0.0-alpha14") - implementation("no.iktdev:eventi:1.0-rc13") + implementation("no.iktdev:eventi:1.0-rc16") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") @@ -60,6 +60,7 @@ dependencies { implementation(kotlin("stdlib-jdk8")) testImplementation("org.assertj:assertj-core:3.21.0") + testImplementation("junit:junit:4.12") implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2") @@ -79,12 +80,24 @@ dependencies { testImplementation("org.mockito:mockito-core:3.+") testImplementation("org.assertj:assertj-core:3.4.1") testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testImplementation("io.mockk:mockk:1.13.9") + testImplementation("org.mockito:mockito-inline:5.2.0") + testImplementation("org.mockito.kotlin:mockito-kotlin:5.2.1") + testImplementation("org.mockito:mockito-junit-jupiter:5.11.0") + testImplementation(project(":shared:common", configuration = "testArtifacts")) + testImplementation("org.springframework.boot:spring-boot-starter-test") + + + } tasks.withType { useJUnitPlatform() } + + kotlin { jvmToolchain(21) } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt index 6d5b14dc..b0c64450 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt @@ -7,12 +7,14 @@ import no.iktdev.exfl.coroutines.CoroutinesDefault import no.iktdev.exfl.coroutines.CoroutinesIO import no.iktdev.exfl.observable.Observables import no.iktdev.mediaprocessing.shared.common.DatabaseApplication +import no.iktdev.mediaprocessing.shared.common.MediaProcessingApp import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry import no.iktdev.mediaprocessing.shared.common.getAppVersion import org.springframework.boot.runApplication import org.springframework.context.annotation.Configuration +@MediaProcessingApp class CoordinatorApplication: DatabaseApplication() { } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorEnv.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorEnv.kt index 8db7dc8d..8f397570 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorEnv.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorEnv.kt @@ -4,8 +4,14 @@ import java.io.File class CoordinatorEnv { companion object { + val streamitAddress = System.getenv("STREAMIT_ADDRESS") ?: "http://streamit.service" + val ffprobe: String = System.getenv("SUPPORTING_EXECUTABLE_FFPROBE") ?: "ffprobe" val preference: File = File("/data/config/preference.json") + + var cachedContent: File = if (!System.getenv("DIRECTORY_CONTENT_CACHE").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_CACHE")) else File("/src/cache") + val outgoingContent: File = if (!System.getenv("DIRECTORY_CONTENT_OUTGOING").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_OUTGOING")) else File("/src/output") + } } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/EventPoller.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/EventPoller.kt index cfec6c01..c7d34049 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/EventPoller.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/EventPoller.kt @@ -3,8 +3,8 @@ package no.iktdev.mediaprocessing.coordinator import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch -import no.iktdev.eventi.events.AbstractEventPoller import no.iktdev.eventi.events.EventDispatcher +import no.iktdev.eventi.events.EventPollerImplementation import no.iktdev.eventi.events.SequenceDispatchQueue import no.iktdev.mediaprocessing.shared.common.stores.EventStore import org.springframework.boot.ApplicationArguments @@ -26,5 +26,6 @@ class PollerAdministrator( val sequenceDispatcher = SequenceDispatchQueue(8) val dispatcher = EventDispatcher(eventStore = EventStore) -class EventPoller: AbstractEventPoller(eventStore = EventStore, dispatchQueue = sequenceDispatcher, dispatcher = dispatcher) { +@Component +class EventPoller: EventPollerImplementation(eventStore = EventStore, dispatchQueue = sequenceDispatcher, dispatcher = dispatcher) { } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Preference.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Preference.kt index e14094f9..1a82b636 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Preference.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Preference.kt @@ -7,10 +7,10 @@ import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoCodec import no.iktdev.mediaprocessing.shared.common.silentTry import java.io.File -class ProcesserPreference { - val videoPreference: VideoPreference? = null +data class ProcesserPreference( + val videoPreference: VideoPreference? = null, val audioPreference: AudioPreference? = null -} +) data class VideoPreference( val codec: VideoCodec, @@ -18,7 +18,7 @@ data class VideoPreference( ) data class AudioPreference( - val language: String, + val language: String? = null, val codec: AudioCodec ) diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/RestTemplateConfig.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/RestTemplateConfig.kt new file mode 100644 index 00000000..f567966b --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/RestTemplateConfig.kt @@ -0,0 +1,22 @@ +package no.iktdev.mediaprocessing.coordinator + +import org.springframework.boot.web.client.RestTemplateBuilder +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.web.client.RestTemplate + +@Configuration +class RestTemplateConfig { + + @Configuration + class RestTemplateConfig { + + @Bean + fun streamitRestTemplate(): RestTemplate { + return RestTemplateBuilder() + .rootUri(CoordinatorEnv.streamitAddress) + .build() + } + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/CollectEventsListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/CollectEventsListener.kt new file mode 100644 index 00000000..70c3145e --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/CollectEventsListener.kt @@ -0,0 +1,36 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import mu.KotlinLogging +import no.iktdev.eventi.events.EventListener +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent +import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection +import org.springframework.stereotype.Component + +@Component +class CollectEventsListener: EventListener() { + private val log = KotlinLogging.logger {} + + val undesiredStates = listOf(CollectProjection.TaskStatus.Failed, CollectProjection.TaskStatus.Pending) + override fun onEvent( + event: Event, + history: List + ): Event? { + + val collectProjection = CollectProjection(history) + log.info { collectProjection.prettyPrint() } + + val taskStatus = collectProjection.getTaskStatus() + if (taskStatus.all { it == CollectProjection.TaskStatus.NotInitiated }) { + // No work has been done, so we are not ready + return null + } + val statusAcceptable = taskStatus.none { it in undesiredStates } + if (!statusAcceptable) { + log.warn { "One or more tasks have failed in ${event.referenceId}" } + return null + } + + return CollectedEvent(history.map { it.eventId }.toSet()).derivedOf(event) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateConvertTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateConvertTaskListener.kt index 33caee5e..0c7547e5 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateConvertTaskListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateConvertTaskListener.kt @@ -1,15 +1,60 @@ package no.iktdev.mediaprocessing.coordinator.listeners.events +import mu.KotlinLogging import no.iktdev.eventi.events.EventListener import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore import org.springframework.stereotype.Component +import java.io.File +import java.nio.file.Files +import java.nio.file.Path @Component class MediaCreateConvertTaskListener: EventListener() { + private val log = KotlinLogging.logger {} + + fun allowOverwrite(): Boolean { + return true + } + override fun onEvent( event: Event, history: List ): Event? { - return null; + + val startedEvent = history.filterIsInstance().firstOrNull() ?: return null + if (startedEvent.data.operation.isNotEmpty()) { + if (!startedEvent.data.operation.contains(OperationType.Convert)) + return null + } + val selectedEvent = event as? ProcesserExtractResultEvent ?: return null + if (selectedEvent.status != TaskStatus.Completed) + return null + + + val result = selectedEvent.data ?: return null + if (!Files.exists(Path.of(result.cachedOutputFile))) + return null + val useFile = File(result.cachedOutputFile) + + val convertTask = ConvertTask( + data = ConvertTask.Data( + inputFile = result.cachedOutputFile, + language = result.language, + allowOverwrite = allowOverwrite(), + outputDirectory = useFile.parentFile.absolutePath, + outputFileName = useFile.nameWithoutExtension, + ) + ).derivedOf(event) + TaskStore.persist(convertTask) + + return ConvertTaskCreatedEvent(convertTask.taskId) + } } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateCoverDownloadTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateCoverDownloadTaskListener.kt new file mode 100644 index 00000000..4236e31a --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateCoverDownloadTaskListener.kt @@ -0,0 +1,48 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import mu.KotlinLogging +import no.iktdev.eventi.events.EventListener +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.springframework.stereotype.Component + +@Component +class MediaCreateCoverDownloadTaskListener: EventListener() { + private val log = KotlinLogging.logger {} + + override fun onEvent( + event: Event, + history: List + ): Event? { + val useEvent = event as? MetadataSearchResultEvent ?: return null + if (useEvent.status != TaskStatus.Completed) { + log.warn("MetadataResult on ${event.referenceId} did not complete successfully") + return null + } + + val downloadData = useEvent.results.map { + val data = it.data + val outputFileName = "${data.title}-${data.source}" + CoverDownloadTask.CoverDownloadData( + url = it.data.cover, + source = it.data.source, + outputFileName = outputFileName + ) + } + + val downloadTasks = downloadData.map { + CoverDownloadTask(it) + .derivedOf(useEvent) + } + + downloadTasks.forEach { TaskStore.persist(it) } + + return CoverDownloadTaskCreatedEvent( + downloadTasks.map { it.taskId } + ) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateEncodeTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateEncodeTaskListener.kt index 41791b60..853e2134 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateEncodeTaskListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateEncodeTaskListener.kt @@ -10,6 +10,8 @@ import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoCodec import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoTarget import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksEncodeSelectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeTaskCreatedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeData import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask @@ -20,7 +22,6 @@ import java.io.File @Component class MediaCreateEncodeTaskListener : EventListener() { - override fun onEvent( event: Event, history: List @@ -28,6 +29,10 @@ class MediaCreateEncodeTaskListener : EventListener() { val preference = Preference.getProcesserPreference() val startedEvent = history.filterIsInstance().firstOrNull() ?: return null + if (startedEvent.data.operation.isNotEmpty()) { + if (!startedEvent.data.operation.contains(OperationType.Encode)) + return null + } val selectedEvent = event as? MediaTracksEncodeSelectedEvent ?: return null val streams = history.filterIsInstance().firstOrNull()?.data ?: return null @@ -53,20 +58,21 @@ class MediaCreateEncodeTaskListener : EventListener() { audioTracks = audioTargets ) val args = plan.toFfmpegArgs(streams.videoStream, streams.audioStream) + val filename = startedEvent.data.fileUri.let { File(it) }.nameWithoutExtension + val extension = plan.toContainer() val task = EncodeTask( data = EncodeData( arguments = args, - outputFileName = startedEvent.data.fileUri.let { File(it).nameWithoutExtension }, + outputFileName = "$filename.$extension", inputFile = startedEvent.data.fileUri ) ).derivedOf(event) TaskStore.persist(task) - return null // Create task instead of event + return ProcesserEncodeTaskCreatedEvent( + taskCreated = task.taskId + ).derivedOf(event) } - - - } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateExtractTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateExtractTaskListener.kt index f06835fc..9b95188a 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateExtractTaskListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateExtractTaskListener.kt @@ -2,8 +2,19 @@ package no.iktdev.mediaprocessing.coordinator.listeners.events import no.iktdev.eventi.events.EventListener import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream +import no.iktdev.mediaprocessing.ffmpeg.dsl.SubtitleCodec +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksExtractSelectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleTask +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore import org.springframework.stereotype.Component +import java.io.File +import java.util.UUID @Component class MediaCreateExtractTaskListener: EventListener() { @@ -11,9 +22,59 @@ class MediaCreateExtractTaskListener: EventListener() { event: Event, history: List ): Event? { - val useEvent = event as? MediaTracksExtractSelectedEvent ?: return null + val startedEvent = history.filterIsInstance().firstOrNull() ?: return null + if (startedEvent.data.operation.isNotEmpty()) { + if (!startedEvent.data.operation.contains(OperationType.Extract)) + return null + } - return null + val selectedEvent = event as? MediaTracksExtractSelectedEvent ?: return null + val streams = history.filterIsInstance().firstOrNull()?.data ?: return null + + val selectedStreams: Map = selectedEvent.selectedSubtitleTracks.associateWith { + streams.subtitleStream[it] + } + + val entries = selectedStreams.mapNotNull { (idx, stream )-> + toSubtitleArgumentData(idx, startedEvent.data.fileUri.let { File(it) }, stream) + } + + val createdTaskIds: MutableList = mutableListOf() + entries.forEach { entry -> + ExtractSubtitleTask(data = entry).derivedOf(event).also { + TaskStore.persist(it) + createdTaskIds.add(it.taskId) + } + } + + return ProcesserExtractTaskCreatedEvent( + tasksCreated = createdTaskIds + ).derivedOf(event) } + + fun toSubtitleArgumentData(index: Int, inputFile: File, stream: SubtitleStream): ExtractSubtitleData? { + val codec = SubtitleCodec.getCodec(stream.codec_name) ?: return null + val extension = codec.getExtension() + + // ffmpeg-args for å mappe og copy akkurat dette subtitle-sporet + val args = mutableListOf() + args += listOf("-map", "0:s:$index") + args += codec.buildFfmpegArgs(stream) + + val language = stream.tags.language?: return null + + // outputfilnavn basert på index og extension + val outputFileName = "${inputFile.nameWithoutExtension}-${language}.${extension}" + + return ExtractSubtitleData( + inputFile = inputFile.absolutePath, + arguments = args, + outputFileName = outputFileName, + language = language + ) + + } + + } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateMetadataSearchTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateMetadataSearchTaskListener.kt new file mode 100644 index 00000000..c7d7861d --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateMetadataSearchTaskListener.kt @@ -0,0 +1,75 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import no.iktdev.eventi.ListenerOrder +import no.iktdev.eventi.events.EventListener +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MetadataSearchTask +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.springframework.stereotype.Component +import java.util.* +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit + +@Component +@ListenerOrder(5) +class MediaCreateMetadataSearchTaskListener: EventListener() { + + private val scheduledExpiries = ConcurrentHashMap>() + private val scheduler = Executors.newScheduledThreadPool(1) + + override fun onEvent( + event: Event, + history: List + ): Event? { + // For replay + if (event is MetadataSearchTaskCreatedEvent) { + val hasResult = history.filter { it is MetadataSearchResultEvent } + .any { it.metadata.derivedFromId?.contains(event.taskId) == true } + + if (!hasResult) { + scheduleTaskExpiry(event.taskId, event.eventId, event.referenceId) + } + } else if (event is MetadataSearchResultEvent) { + val cancelKeys = event.metadata.derivedFromId ?: emptySet() + scheduledExpiries.filter { it -> it.key in cancelKeys }.keys.forEach { key -> + scheduledExpiries.remove(key)?.cancel(true) + } + } + + val useEvent = event as? MediaParsedInfoEvent ?: return null + + val task = MetadataSearchTask( + MetadataSearchTask.SearchData( + searchTitles = useEvent.data.parsedSearchTitles, + collection = useEvent.data.parsedCollection + ) + ).derivedOf(useEvent) + val finalResult = MetadataSearchTaskCreatedEvent(task.taskId).derivedOf(useEvent) + scheduleTaskExpiry(task.taskId, finalResult.eventId, task.referenceId) + return finalResult + } + + private fun scheduleTaskExpiry(taskId: UUID, eventId: UUID, referenceId: UUID) { + if (scheduledExpiries.containsKey(taskId)) return + + val future = scheduler.schedule({ + // Hvis tasken fortsatt ikke har result/failed → marker som failed + TaskStore.claim(taskId, "Coordinator-MetadataSearchTaskListener-TimeoutScheduler") + TaskStore.markConsumed(taskId, TaskStatus.Failed) + val failureEvent = MetadataSearchResultEvent( + status = TaskStatus.Failed, + ).apply { setFailed(listOf(taskId)) } + //publishEvent(MetadataSearchFailedEvent(taskId, "Timeout").derivedOf(referenceId)) + scheduledExpiries.remove(taskId) + }, 10, TimeUnit.MINUTES) + + scheduledExpiries[taskId] = future + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaDetermineSubtitleTrackTypeListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaDetermineSubtitleTrackTypeListener.kt index 65113324..b6e965f5 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaDetermineSubtitleTrackTypeListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaDetermineSubtitleTrackTypeListener.kt @@ -38,8 +38,8 @@ class MediaDetermineSubtitleTrackTypeListener: EventListener() { } - fun getCommentaryFilters(): Set = setOf("commentary", "kommentar", "kommentaar") - fun getSongFilters(): Set = setOf("song", "sign") + fun getCommentaryFilters(): Set = setOf("commentary", "comentary", "kommentar", "kommentaar") + fun getSongFilters(): Set = setOf("song", "sign", "lyrics") fun getClosedCaptionFilters(): Set = setOf("closed caption", "cc", "close caption", "closed-caption", "cc.") fun getSHDFilters(): Set = setOf("shd", "hh", "hard of hearing", "hard-of-hearing") diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParseStreamsListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParseStreamsListener.kt index c0c50658..655fa495 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParseStreamsListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParseStreamsListener.kt @@ -3,18 +3,19 @@ package no.iktdev.mediaprocessing.coordinator.listeners.events import com.google.gson.Gson import com.google.gson.JsonObject import mu.KotlinLogging +import no.iktdev.eventi.ListenerOrder import no.iktdev.eventi.events.EventListener import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamReadEvent -import org.springframework.core.annotation.Order import org.springframework.stereotype.Component -@Order(4) +@ListenerOrder(4) @Component class MediaParseStreamsListener: EventListener() { val log = KotlinLogging.logger {} @@ -23,7 +24,13 @@ class MediaParseStreamsListener: EventListener() { event: Event, history: List ): Event? { - if (event !is MediaStreamReadEvent) return null + if (event !is CoordinatorReadStreamsResultEvent) return null + if (event.status != TaskStatus.Completed) + return null + if (event.data == null) { + log.error { "No data to parse in CoordinatorReadStreamsResultEvent" } + return null + } val streams = parseStreams(event.data) return MediaStreamParsedEvent( diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParsedInfoListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParsedInfoListener.kt index 610ced6f..6eb3950c 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParsedInfoListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParsedInfoListener.kt @@ -1,15 +1,15 @@ package no.iktdev.mediaprocessing.coordinator.listeners.events +import no.iktdev.eventi.ListenerOrder import no.iktdev.eventi.events.EventListener import no.iktdev.eventi.models.Event import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent import no.iktdev.mediaprocessing.shared.common.model.MediaType -import org.springframework.core.annotation.Order import org.springframework.stereotype.Component import java.io.File -@Order(2) +@ListenerOrder(2) @Component class MediaParsedInfoListener : EventListener() { override fun onEvent( @@ -24,12 +24,22 @@ class MediaParsedInfoListener : EventListener() { val searchTitles = file.guessSearchableTitle() val mediaType = file.guessMovieOrSeries() + val episodeInfo = if (mediaType == MediaType.Serie) { + val serieInfo = file.guessSerieInfo() + MediaParsedInfoEvent.ParsedData.EpisodeInfo( + episodeNumber = serieInfo.episodeNumber, + seasonNumber = serieInfo.seasonNumber, + episodeTitle = serieInfo.episodeTitle, + ) + } else null + return MediaParsedInfoEvent( MediaParsedInfoEvent.ParsedData( parsedFileName = filename, parsedCollection = collection, parsedSearchTitles = searchTitles, - mediaType = mediaType + mediaType = mediaType, + episodeInfo = episodeInfo, ) ).derivedOf(event) } @@ -99,6 +109,9 @@ class MediaParsedInfoListener : EventListener() { return when (type) { MediaType.Movie -> this.guessDesiredMovieTitle() MediaType.Serie -> this.guessDesiredSerieTitle() + MediaType.Subtitle -> { + this.nameWithoutExtension + } } } @@ -106,6 +119,9 @@ class MediaParsedInfoListener : EventListener() { val collection = when (this.guessMovieOrSeries()) { MediaType.Movie -> this.guessDesiredMovieTitle() MediaType.Serie -> this.guessDesiredSerieTitle() + MediaType.Subtitle -> { + this.parentFile.parentFile.nameWithoutExtension + } } return collection.noParens().noYear().split(" - ").first().trim() } @@ -131,6 +147,26 @@ class MediaParsedInfoListener : EventListener() { * @return A fully cleaned title including season and episode with possible episode title */ fun File.guessDesiredSerieTitle(): String { + val parsedSerieInfo = this.guessSerieInfo() + + val tag = buildString { + append("S${(parsedSerieInfo.seasonNumber ?: 1).toString().padStart(2, '0')}") + append("E${(parsedSerieInfo.episodeNumber ?: 1).toString().padStart(2, '0')}") + if (parsedSerieInfo.revision != null) append(" (v$parsedSerieInfo.revision)") + } + + return buildString { + append(parsedSerieInfo.serieTitle) + append(" - ") + append(tag) + if (parsedSerieInfo.episodeTitle.isNotEmpty()) { + append(" - ") + append(parsedSerieInfo.episodeTitle) + } + }.trim() + } + + fun File.guessSerieInfo(): ParsedSerieInfo { val raw = this.nameWithoutExtension val seasonRegex = Regex("""(?i)(?:S|Season|Series)\s*(\d{1,2})""") @@ -174,25 +210,18 @@ class MediaParsedInfoListener : EventListener() { baseTitle = this.parentFile?.name?.getCleanedTitle() ?: "Dumb ways to die" } - val tag = buildString { - append("S${(season ?: 1).toString().padStart(2, '0')}") - append("E${(episode ?: 1).toString().padStart(2, '0')}") - if (revision != null) append(" (v$revision)") - } - - return buildString { - append(baseTitle) - append(" - ") - append(tag) - if (episodeTitle.isNotEmpty()) { - append(" - ") - append(episodeTitle) - } - }.trim() + return ParsedSerieInfo( + serieTitle = baseTitle, + episodeNumber = episode ?: 1, + seasonNumber = season ?: 1, + revision = revision, + episodeTitle + ) } + fun File.guessSearchableTitle(): List { val cleaned = this.guessDesiredFileName() .noResolutionAndAfter() @@ -230,5 +259,13 @@ class MediaParsedInfoListener : EventListener() { return titles.distinct() } + data class ParsedSerieInfo( + val serieTitle: String, + val episodeNumber: Int, + val seasonNumber: Int, + val revision: Int? = null, + val episodeTitle: String, + ) + } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaReadStreamsTaskCreatedListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaReadStreamsTaskCreatedListener.kt index 5bbac291..1a7c94ec 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaReadStreamsTaskCreatedListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaReadStreamsTaskCreatedListener.kt @@ -1,16 +1,16 @@ package no.iktdev.mediaprocessing.coordinator.listeners.events +import no.iktdev.eventi.ListenerOrder import no.iktdev.eventi.events.EventListener import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsTaskCreatedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamReadTaskCreatedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask import no.iktdev.mediaprocessing.shared.common.stores.TaskStore -import org.springframework.core.annotation.Order import org.springframework.stereotype.Component -@Order(3) +@ListenerOrder(3) @Component class MediaReadStreamsTaskCreatedListener: EventListener() { override fun onEvent( @@ -24,9 +24,9 @@ class MediaReadStreamsTaskCreatedListener: EventListener() { val readTask = MediaReadTask( fileUri = startEvent.data.fileUri - ) + ).derivedOf(event) TaskStore.persist(readTask) - return null // Create task instead of event + return CoordinatorReadStreamsTaskCreatedEvent(readTask.taskId) // Create task instead of event } } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectEncodeTracksListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectEncodeTracksListener.kt index f150ea8a..24df9c96 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectEncodeTracksListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectEncodeTracksListener.kt @@ -33,7 +33,7 @@ class MediaSelectEncodeTracksListener: EventListener() { ).derivedOf(event) } - private fun getAudioExtendedTrackToUse(audioStream: List, selectedDefaultTrack: Int): Int? { + protected fun getAudioExtendedTrackToUse(audioStream: List, selectedDefaultTrack: Int): Int? { val durationFiltered = audioStream.filterOnPreferredLanguage() .filter { (it.duration_ts ?: 0) > 0 } .filter { it.channels > 2 } @@ -47,13 +47,13 @@ class MediaSelectEncodeTracksListener: EventListener() { * If no default track is found, select the first audio track. * If audio track with preferred language (e.g., "nor") is not found, selects "eng" or first available. */ - private fun getAudioDefaultTrackToUse(audioStream: List): Int { + protected fun getAudioDefaultTrackToUse(audioStream: List): Int { val durationFiltered = audioStream.filterOnPreferredLanguage() .filter { (it.duration_ts ?: 0) > 0 } val selected = durationFiltered .filter { it.channels == 2 }.ifEmpty { durationFiltered } - .maxByOrNull { it.index } ?: audioStream.minByOrNull { it.index } ?: durationFiltered.firstOrNull() + .minByOrNull { it.index } ?: audioStream.minByOrNull { it.index } ?: durationFiltered.firstOrNull() return audioStream.indexOf(selected) } @@ -62,11 +62,11 @@ class MediaSelectEncodeTracksListener: EventListener() { * Filters audio streams based on preferred languages. * If no streams match the preferred languages, returns the original list. */ - private fun List.filterOnPreferredLanguage(): List { + protected fun List.filterOnPreferredLanguage(): List { return this.filter { it.tags.language in getAudioLanguagePreference() }.ifEmpty { this } } - private fun getVideoTrackToUse(streams: List): Int { + protected fun getVideoTrackToUse(streams: List): Int { val selectStream = streams.filter { (it.duration_ts ?: 0) > 0 } .maxByOrNull { it.duration_ts ?: 0 } ?: streams.minByOrNull { it.index } ?: throw Exception("No video streams found") return streams.indexOf(selectStream) diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectExtractTracksListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectExtractTracksListener.kt index ba7c90da..4c27b492 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectExtractTracksListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectExtractTracksListener.kt @@ -11,7 +11,7 @@ import org.springframework.stereotype.Component @Component class MediaSelectExtractTracksListener: EventListener() { - fun limitToLanguages(): Set { + open fun limitToLanguages(): Set { return emptySet() } @@ -36,7 +36,7 @@ class MediaSelectExtractTracksListener: EventListener() { } - private fun List.filterOnPreferredLanguage(): List { + protected fun List.filterOnPreferredLanguage(): List { val languages = limitToLanguages() if (languages.isEmpty()) return this return this.filter { it.tags.language != null }.filter { languages.contains(it.tags.language) } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MigrateCreateStoreTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MigrateCreateStoreTaskListener.kt new file mode 100644 index 00000000..7edceb37 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MigrateCreateStoreTaskListener.kt @@ -0,0 +1,59 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import mu.KotlinLogging +import no.iktdev.eventi.events.EventListener +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask +import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection +import no.iktdev.mediaprocessing.shared.common.projection.MigrateContentProject +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.springframework.stereotype.Component + +@Component +class MigrateCreateStoreTaskListener: EventListener() { + private val log = KotlinLogging.logger {} + + override fun onEvent( + event: Event, + history: List + ): Event? { + val useEvent = event as? CollectedEvent ?: return null + val useHistory = history.filter { useEvent.eventIds.contains(it.eventId) } + + val collectProjection = CollectProjection(useHistory) + log.info { collectProjection.prettyPrint() } + + val statusAcceptable = collectProjection.getTaskStatus().none { it == CollectProjection.TaskStatus.Failed } + if (!statusAcceptable) { + log.warn { "One or more tasks have failed in ${event.referenceId}" } + } + + val migrateContentProjection = MigrateContentProject(useHistory, CoordinatorEnv.outgoingContent) + + val collection = migrateContentProjection.useStore?.name ?: + throw RuntimeException("No content store configured for migration in ${event.referenceId}") + + val videContent = migrateContentProjection.getVideoStoreFile()?.let { MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath) } + val subtitleContent = migrateContentProjection.getSubtitleStoreFiles()?.map { + MigrateToContentStoreTask.Data.SingleSubtitle(it.language, it.cts.cachedFile.absolutePath, it.cts.storeFile.absolutePath, ) + } + val coverContent = migrateContentProjection.getCoverStoreFiles()?.map { + MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath) + } + val storeTask = MigrateToContentStoreTask( + MigrateToContentStoreTask.Data( + collection = collection, + videoContent = videContent, + subtitleContent = subtitleContent, + coverContent = coverContent + ) + ).derivedOf(event) + + TaskStore.persist(storeTask) + + return MigrateContentToStoreTaskCreatedEvent(storeTask.taskId) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StartedListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StartedListener.kt index 00dde704..326d9728 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StartedListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StartedListener.kt @@ -1,16 +1,12 @@ package no.iktdev.mediaprocessing.coordinator.listeners.events +import no.iktdev.eventi.ListenerOrder import no.iktdev.eventi.events.EventListener import no.iktdev.eventi.models.Event -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.FileReadyEvent -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcessFlow -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent -import org.springframework.core.annotation.Order +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.* import org.springframework.stereotype.Component -@Order(1) +@ListenerOrder(1) @Component class StartedListener : EventListener() { override fun onEvent( @@ -21,7 +17,7 @@ class StartedListener : EventListener() { return StartProcessingEvent( data = StartData( - flow = ProcessFlow.Auto, + flow = StartFlow.Auto, fileUri = useEvent.data.fileUri, operation = setOf( OperationType.Extract, diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StoreContentAndMetadataListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StoreContentAndMetadataListener.kt new file mode 100644 index 00000000..60bea0ed --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StoreContentAndMetadataListener.kt @@ -0,0 +1,54 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import mu.KotlinLogging +import no.iktdev.eventi.events.EventListener +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask +import no.iktdev.mediaprocessing.shared.common.model.ContentExport +import no.iktdev.mediaprocessing.shared.common.projection.StoreProjection +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.springframework.stereotype.Component + +@Component +class StoreContentAndMetadataListener: EventListener() { + val log = KotlinLogging.logger {} + + override fun onEvent( + event: Event, + history: List + ): Event? { + val useEvent = event as? MigrateContentToStoreTaskResultEvent ?: return null + val collectionEvent = history.lastOrNull { it is CollectedEvent } as? CollectedEvent + ?: return null + + val useHistory = (history.filter { collectionEvent.eventIds.contains(it.eventId) }) + listOf(useEvent) + val projection = StoreProjection(useHistory) + + val collection = projection.getCollection() + if (collection.isNullOrBlank()) { + log.error { "Collection is null @ ${useEvent.referenceId}" } + return null + } + val metadata = projection.projectMetadata() + if (metadata == null) { + log.error { "Metadata is null @ ${useEvent.referenceId}"} + return null + } + + + val exportInfo = ContentExport( + collection = collection, + media = projection.projectMediaFiles(), + episodeInfo = projection.projectEpisodeInfo(), + metadata = metadata + ) + + val task = StoreContentAndMetadataTask(exportInfo).derivedOf(useEvent) + TaskStore.persist(task) + + return StoreContentAndMetadataTaskCreatedEvent(task.taskId).derivedOf(useEvent) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/DownloadCoverTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/DownloadCoverTaskListener.kt new file mode 100644 index 00000000..e7729c58 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/DownloadCoverTaskListener.kt @@ -0,0 +1,55 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.tasks + +import mu.KotlinLogging +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.tasks.TaskListener +import no.iktdev.eventi.tasks.TaskType +import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv +import no.iktdev.mediaprocessing.shared.common.DownloadClient +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask +import org.springframework.stereotype.Component +import java.util.UUID + +@Component +class DownloadCoverTaskListener: TaskListener(TaskType.MIXED) { + val log = KotlinLogging.logger {} + + override fun getWorkerId(): String { + return "${this::class.java.simpleName}-${TaskType.CPU_INTENSIVE}-${UUID.randomUUID()}" + } + + override fun supports(task: Task): Boolean { + return task is CoverDownloadTask + } + + override suspend fun onTask(task: Task): Event? { + val pickedTask = task as? CoverDownloadTask ?: return null + log.info { "Downloading cover from ${pickedTask.data.url}" } + val taskData = pickedTask.data + + val downloadClient = DownloadClient(taskData.url, CoordinatorEnv.cachedContent, taskData.outputFileName) + val downloadedFile = downloadClient.download() + + + if (downloadedFile?.exists() == true) { + log.info { "Downloaded cover to ${downloadedFile.absolutePath}" } + return CoverDownloadResultEvent( + status = TaskStatus.Completed, + data = CoverDownloadResultEvent.CoverDownloadedData( + source = taskData.source, + outputFile = downloadedFile.absolutePath + ) + ).producedFrom(pickedTask) + } else { + log.error { "Failed to download cover from ${taskData.url}" } + return CoverDownloadResultEvent( + status = TaskStatus.Failed, + ) + } + } + + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MediaStreamReadTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MediaStreamReadTaskListener.kt index f2bd4429..6bd4badb 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MediaStreamReadTaskListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MediaStreamReadTaskListener.kt @@ -3,10 +3,11 @@ package no.iktdev.mediaprocessing.coordinator.listeners.tasks import mu.KotlinLogging import no.iktdev.eventi.models.Event import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus import no.iktdev.eventi.tasks.TaskType import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv import no.iktdev.mediaprocessing.ffmpeg.FFprobe -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamReadEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask import org.springframework.stereotype.Component import java.util.UUID @@ -29,14 +30,20 @@ class MediaStreamReadTaskListener: FfprobeTaskListener(TaskType.CPU_INTENSIVE) { val probeResult = getFfprobe() .readJsonStreams(pickedTask.fileUri) - val result = probeResult.data - assert(result != null) { "No data returned from ffprobe for ${pickedTask.fileUri}" } + val result = + probeResult.data ?: throw RuntimeException("No data returned from ffprobe for ${pickedTask.fileUri}") - return MediaStreamReadEvent(data = result!!).producedFrom(task) + return CoordinatorReadStreamsResultEvent( + status = TaskStatus.Completed, + data = result + ).producedFrom(task) } catch (e: Exception) { log.error(e) { "Error reading media streams for ${pickedTask.fileUri}" } - return null + return CoordinatorReadStreamsResultEvent( + status = TaskStatus.Failed, + data = null + ) } } @@ -44,6 +51,6 @@ class MediaStreamReadTaskListener: FfprobeTaskListener(TaskType.CPU_INTENSIVE) { return JsonFfinfo(CoordinatorEnv.ffprobe) } - class JsonFfinfo(override val executable: String): FFprobe() { + class JsonFfinfo(executable: String): FFprobe(executable) { } } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MigrateContentToStoreTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MigrateContentToStoreTaskListener.kt new file mode 100644 index 00000000..2e0eb1b1 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MigrateContentToStoreTaskListener.kt @@ -0,0 +1,114 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.tasks + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.tasks.TaskListener +import no.iktdev.eventi.tasks.TaskType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask +import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus +import org.springframework.stereotype.Component +import java.io.File +import java.nio.file.Files +import java.util.UUID + +@Component +class MigrateContentToStoreTaskListener: TaskListener(TaskType.IO_INTENSIVE) { + override fun getWorkerId(): String { + return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}" + } + + override fun supports(task: Task): Boolean { + return task is MigrateToContentStoreTask + } + + override suspend fun onTask(task: Task): Event? { + val pickedTask = task as? MigrateToContentStoreTask ?: return null + + val videoStatus = migrateVideo(pickedTask.data.videoContent) + val subtitleStatus = migrateSubtitle(pickedTask.data.subtitleContent ?: emptyList()) + val coverStatus = migrateCover(pickedTask.data.coverContent ?: emptyList()) + + var status = TaskStatus.Completed + if (videoStatus.status != MigrateStatus.Failed && + subtitleStatus.none { it.status == MigrateStatus.Failed } && + coverStatus.none { it.status == MigrateStatus.Failed }) + { + pickedTask.data.videoContent?.cachedUri?.let { File(it) }?.deleteOnExit() + pickedTask.data.subtitleContent?.forEach { File(it.cachedUri).deleteOnExit() } + pickedTask.data.coverContent?.forEach { File(it.cachedUri).deleteOnExit() } + } else { + status = TaskStatus.Failed + } + + + val completedEvent = MigrateContentToStoreTaskResultEvent( + status = status, + collection = pickedTask.data.collection, + videoMigrate = videoStatus, + subtitleMigrate = subtitleStatus, + coverMigrate = coverStatus + ).producedFrom(task) + + return completedEvent + } + + private fun migrateVideo(videoContent: MigrateToContentStoreTask.Data.SingleContent?): MigrateContentToStoreTaskResultEvent.FileMigration { + if (videoContent == null) return MigrateContentToStoreTaskResultEvent.FileMigration(null, MigrateStatus.NotPresent) + val source = File(videoContent.cachedUri) + val destination = File(videoContent.storeUri) + return try { + source.copyTo(destination, overwrite = true) + val identical = Files.mismatch(source.toPath(), destination.toPath()) == -1L + if (!identical) { + return MigrateContentToStoreTaskResultEvent.FileMigration(null, MigrateStatus.Failed) + } + MigrateContentToStoreTaskResultEvent.FileMigration(destination.absolutePath, MigrateStatus.Completed) + } catch (e: Exception) { + MigrateContentToStoreTaskResultEvent.FileMigration(null, MigrateStatus.Failed) + } + } + + private fun migrateSubtitle(subtitleContents: List): List { + if (subtitleContents.isEmpty()) return listOf(MigrateContentToStoreTaskResultEvent.SubtitleMigration(null, null, MigrateStatus.NotPresent)) + val results = mutableListOf() + for (subtitle in subtitleContents) { + val source = File(subtitle.cachedUri) + val destination = File(subtitle.storeUri) + try { + source.copyTo(destination, overwrite = true) + val identical = Files.mismatch(source.toPath(), destination.toPath()) == -1L + if (!identical) { + results.add(MigrateContentToStoreTaskResultEvent.SubtitleMigration(subtitle.language, destination.absolutePath, MigrateStatus.Failed)) + } else { + results.add(MigrateContentToStoreTaskResultEvent.SubtitleMigration(subtitle.language,destination.absolutePath, MigrateStatus.Completed)) + } + } catch (e: Exception) { + results.add(MigrateContentToStoreTaskResultEvent.SubtitleMigration(subtitle.language,destination.absolutePath, MigrateStatus.Failed)) + } + } + return results + } + + private fun migrateCover(coverContents: List): List { + if (coverContents.isEmpty()) return listOf(MigrateContentToStoreTaskResultEvent.FileMigration(null, MigrateStatus.NotPresent)) + val results = mutableListOf() + for (cover in coverContents) { + val source = File(cover.cachedUri) + val destination = File(cover.storeUri) + try { + source.copyTo(destination, overwrite = true) + val identical = Files.mismatch(source.toPath(), destination.toPath()) == -1L + if (!identical) { + results.add(MigrateContentToStoreTaskResultEvent.FileMigration(destination.absolutePath, MigrateStatus.Failed)) + } else { + results.add(MigrateContentToStoreTaskResultEvent.FileMigration(destination.absolutePath, MigrateStatus.Completed)) + } + } catch (e: Exception) { + results.add(MigrateContentToStoreTaskResultEvent.FileMigration(destination.absolutePath, MigrateStatus.Failed)) + } + } + return results + } +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/StoreContentAndMetadataTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/StoreContentAndMetadataTaskListener.kt new file mode 100644 index 00000000..31868008 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/StoreContentAndMetadataTaskListener.kt @@ -0,0 +1,56 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.tasks + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.tasks.TaskListener +import no.iktdev.eventi.tasks.TaskType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.HttpEntity +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.stereotype.Component +import org.springframework.web.client.RestTemplate +import java.util.UUID + +@Component +class StoreContentAndMetadataTaskListener: TaskListener(TaskType.MIXED) { + @Autowired + lateinit var streamitRestTemplate: RestTemplate + + override fun getWorkerId(): String { + return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}" + } + + override fun supports(task: Task): Boolean { + return task is StoreContentAndMetadataTask + } + + override suspend fun onTask(task: Task): Event? { + val pickedTask = task as? StoreContentAndMetadataTask ?: return null + + val headers = HttpHeaders().apply { contentType = MediaType.APPLICATION_JSON } + val entity = HttpEntity(pickedTask.data, headers) + + val response = try { + val res = streamitRestTemplate.exchange( + "open/api/mediaprocesser/import", + HttpMethod.POST, + entity, + Void::class.java, + ) + res.statusCode.is2xxSuccessful + } catch (e: Exception) { + false + } + + return StoreContentAndMetadataTaskResultEvent( + if (response) TaskStatus.Completed else TaskStatus.Failed + ) + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/ListenerInformOrderTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/ListenerInformOrderTest.kt new file mode 100644 index 00000000..5fd47232 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/ListenerInformOrderTest.kt @@ -0,0 +1,57 @@ +package no.iktdev.mediaprocessing + +import no.iktdev.eventi.ListenerOrder +import no.iktdev.eventi.events.EventListenerRegistry +import no.iktdev.mediaprocessing.coordinator.CoordinatorApplication +import no.iktdev.mediaprocessing.coordinator.listeners.events.* +import no.iktdev.mediaprocessing.shared.common.config.DatasourceConfiguration +import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry +import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.ApplicationContext +import org.springframework.context.annotation.ComponentScan +import org.springframework.test.context.TestPropertySource +import org.springframework.test.context.junit.jupiter.SpringExtension + + +@SpringBootTest( + classes = [CoordinatorApplication::class, + DatasourceConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +@TestPropertySource(properties = ["spring.flyway.enabled=true"]) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ComponentScan("no.iktdev.mediaprocessing.coordinator.listeners.events") +@ExtendWith(SpringExtension::class) +class ListenerInformOrderTest(): TestBase() { + @Autowired lateinit var ctx: ApplicationContext + + @Test + fun verifyTaskRegistryIsNotEmpty() { + assertThat { TaskRegistry.getTasks().isNotEmpty() } + } + @Test + fun verifyEventRegistryIsNotEmpty() { + assertThat { EventRegistry.getEvents().isNotEmpty() } + } + + + @Test + fun `only ordered handlers should be in correct order`() { + val handlers = EventListenerRegistry.getListeners() + assertThat(handlers).isNotEmpty + val filtered = handlers.filter { it::class.java.isAnnotationPresent(ListenerOrder::class.java) } + assertThat (filtered.map { it::class.simpleName }).containsExactly( + StartedListener::class.simpleName, + MediaParsedInfoListener::class.java.simpleName, + MediaReadStreamsTaskCreatedListener::class.java.simpleName, + MediaParseStreamsListener::class.java.simpleName, + MediaCreateMetadataSearchTaskListener::class.java.simpleName, + ) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/MockData.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/MockData.kt new file mode 100644 index 00000000..79534cdc --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/MockData.kt @@ -0,0 +1,122 @@ +package no.iktdev.mediaprocessing + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.TestBase.DummyTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.* +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import java.util.* + +object MockData { + + fun mediaParsedEvent( + collection: String, + fileName: String, + mediaType: MediaType + ) = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = collection, + parsedFileName = fileName, + parsedSearchTitles = listOf(collection, fileName), + mediaType = mediaType + ) + ) + + fun metadataEvent(derivedFrom: Event): List { + val dummyTask = DummyTask().derivedOf(derivedFrom) + val create = MetadataSearchTaskCreatedEvent(dummyTask.taskId).derivedOf(derivedFrom) + + val result = MetadataSearchResultEvent( + results = listOf( + MetadataSearchResultEvent.SearchResult( + simpleScore = 10, + prefixScore = 10, + advancedScore = 10, + sourceWeight = 1f, + data = MetadataSearchResultEvent.SearchResult.MetadataResult( + source = "test", + title = "MyCollection", + cover = "cover.jpg", + type = MediaType.Movie, + summary = listOf( + MetadataSearchResultEvent.SearchResult.MetadataResult.Summary( + language = "en", + description = "desc" + ) + ), + genres = listOf("Drama") + ) + ) + ), + recommended = null, + status = TaskStatus.Completed + ).producedFrom(dummyTask) + return listOf(create, result) + } + + fun encodeEvent(cachedFile: String, derivedFrom: Event, status: TaskStatus = TaskStatus.Completed,): List { + val dummyTask = DummyTask().derivedOf(derivedFrom) + val create = ProcesserEncodeTaskCreatedEvent(dummyTask.taskId) + .derivedOf(derivedFrom) + + val result = ProcesserEncodeResultEvent( + data = ProcesserEncodeResultEvent.EncodeResult( + cachedOutputFile = cachedFile + ), + status = status + ).producedFrom(dummyTask) + return listOf(create, result) + } + + fun extractEvent(language: String, cachedFile: String, derivedFrom: Event): List { + val dummyTask = DummyTask().derivedOf(derivedFrom) + val create = ProcesserExtractTaskCreatedEvent(listOf(dummyTask.taskId) as MutableList) + .derivedOf(derivedFrom) + + val result = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + language = language, + cachedOutputFile = cachedFile + ) + ).producedFrom(dummyTask) + return listOf(create, result) + } + + fun convertEvent( + language: String, + baseName: String, + outputFiles: List, + derivedFrom: Event + ): List { + val dummyTask = DummyTask().derivedOf(derivedFrom) + val createdTaskEvent = ConvertTaskCreatedEvent( + taskId = dummyTask.taskId + ).derivedOf(derivedFrom) + + val resultTask = ConvertTaskResultEvent( + data = ConvertTaskResultEvent.ConvertedData( + language = language, + baseName = baseName, + outputFiles = outputFiles + ), + status = TaskStatus.Completed + ).producedFrom(dummyTask) + return listOf(createdTaskEvent, resultTask) + } + + fun coverEvent(cacheFile: String, derivedFrom: Event, source: String = "test"): List { + val dummyTask = DummyTask().derivedOf(derivedFrom) + val start = CoverDownloadTaskCreatedEvent(listOf(dummyTask.taskId)).derivedOf(derivedFrom) + + val result = CoverDownloadResultEvent( + data = CoverDownloadResultEvent.CoverDownloadedData( + source = source, + outputFile = cacheFile + ), + status = TaskStatus.Completed + ).producedFrom(dummyTask) + return listOf(start, result) + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/MockFFprobe.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/MockFFprobe.kt new file mode 100644 index 00000000..b6776a74 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/MockFFprobe.kt @@ -0,0 +1,32 @@ +package no.iktdev.mediaprocessing + +import com.google.gson.JsonObject +import kotlinx.coroutines.delay +import no.iktdev.mediaprocessing.ffmpeg.FFprobe +import no.iktdev.mediaprocessing.ffmpeg.data.FFinfoOutput + +class MockFFprobe( + private val delayMillis: Long = 0, + private val result: FFinfoOutput? = null, + private val throwException: Boolean = false +) : FFprobe("") { + + var lastInputFile: String? = null + + override suspend fun readJsonStreams(inputFile: String): FFinfoOutput { + lastInputFile = inputFile + if (delayMillis > 0) delay(delayMillis) + if (throwException) throw RuntimeException("Simulated ffprobe failure") + return result ?: FFinfoOutput(success = false, data = null, error = "No result configured") + } + + companion object { + fun success(json: JsonObject) = MockFFprobe( + result = FFinfoOutput(success = true, data = json, error = null) + ) + fun failure(errorMsg: String) = MockFFprobe( + result = FFinfoOutput(success = false, data = null, error = errorMsg) + ) + fun exception() = MockFFprobe(throwException = true) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/TestBase.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/TestBase.kt new file mode 100644 index 00000000..4cc36507 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/TestBase.kt @@ -0,0 +1,51 @@ +package no.iktdev.mediaprocessing + +import io.mockk.* +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Task +import no.iktdev.mediaprocessing.coordinator.AudioPreference +import no.iktdev.mediaprocessing.coordinator.Preference +import no.iktdev.mediaprocessing.coordinator.ProcesserPreference +import no.iktdev.mediaprocessing.coordinator.VideoPreference +import no.iktdev.mediaprocessing.ffmpeg.dsl.AudioCodec +import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoCodec +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.junit.jupiter.api.BeforeEach +import java.io.File +import java.util.* + +open class TestBase { + class DummyEvent: Event() + class DummyTask: Task() + + @BeforeEach + fun setup() { + mockkObject(TaskStore) + every { TaskStore.persist(any()) } just Runs + mockkObject(Preference) + every { Preference.getProcesserPreference() } returns ProcesserPreference( + videoPreference = VideoPreference(codec = VideoCodec.Hevc()), + audioPreference = AudioPreference(codec = AudioCodec.Aac(channels = 2)) + ) + } + + fun mockkIO() { + mockkConstructor(File::class) + every { anyConstructed().exists() } returns true + } + + fun defaultStartEvent(): StartProcessingEvent { + val start = StartProcessingEvent( + data = StartData( + operation = setOf(OperationType.Encode, OperationType.Extract, OperationType.Convert), + fileUri = "file:///unit/${UUID.randomUUID()}.mkv" + ) + ) + start.newReferenceId() + return start + + } +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/CollectEventsListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/CollectEventsListenerTest.kt new file mode 100644 index 00000000..172da787 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/CollectEventsListenerTest.kt @@ -0,0 +1,275 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.MockData.convertEvent +import no.iktdev.mediaprocessing.MockData.coverEvent +import no.iktdev.mediaprocessing.MockData.encodeEvent +import no.iktdev.mediaprocessing.MockData.extractEvent +import no.iktdev.mediaprocessing.MockData.mediaParsedEvent +import no.iktdev.mediaprocessing.MockData.metadataEvent +import no.iktdev.mediaprocessing.TestBase +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + + +class CollectEventsListenerTest : TestBase() { + + private val listener = CollectEventsListener() + + @Test + @DisplayName( + """ + Hvis historikken har alle påkrevde hendelser og alle oppgaver er i en gyldig tisltand + Når onEvent kalles og projeksjonen tilsier gyldig status + Så: + Opprettes CollectEvent basert på historikken + """ + ) + fun success1() { + val started = defaultStartEvent() + + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + val metadata = metadataEvent(parsed) + + val encode = encodeEvent("/tmp/video.mp4", parsed) + val extract = extractEvent("en", "/tmp/sub1.srt", encode.last()) + val convert = convertEvent(language = "en", baseName = "sub1", outputFiles = listOf("/tmp/sub1.vtt"), derivedFrom = extract.last()) + val cover = coverEvent("/tmp/cover.jpg", metadata.last()) + + val history = listOf( + started, + parsed, + *metadata.toTypedArray(), + *encode.toTypedArray(), + *extract.toTypedArray(), + *convert.toTypedArray(), + *cover.toTypedArray(), + ) + + val result = listener.onEvent(history.last(), history) + + assertThat(result).isNotNull() + assertThat { + result is CollectedEvent + } + } + + + @Test + @DisplayName( + """ + Hvis vi har kun encoded hendelse, men vi har sagt at vi også skal ha extract, men ikke har opprettet extract + Når encode result kommer inn + Så: + Opprettes CollectEvent basert på historikken + """ + ) + fun success2() { + val started = defaultStartEvent().let { ev -> + ev.copy(data = ev.data.copy(operation = setOf(OperationType.Encode, OperationType.Extract))) + } + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + val encode = encodeEvent("/tmp/video.mp4", parsed) + + val history = listOf( + started, + parsed, + *encode.toTypedArray(), + ) + val result = listener.onEvent(history.last(), history) + assertThat(result).isNotNull() + assertThat { + result is CollectedEvent + } + } + + + @Test + @DisplayName( + """ + Hvis vi har kun convert hendelse + Når convert har komment inn + Så: + Opprettes CollectEvent basert på historikken + """ + ) + fun success3() { + val started = defaultStartEvent().let { ev -> + ev.copy(data = ev.data.copy(operation = setOf(OperationType.Convert))) + } + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + val convert = encodeEvent("/tmp/fancy.srt", parsed) + + val history = listOf( + started, + parsed, + *convert.toTypedArray(), + ) + val result = listener.onEvent(history.last(), history) + assertThat(result).isNotNull() + assertThat { + result is CollectedEvent + } + } + + + @Test + @DisplayName( + """ + Hvis vi har kun encoded og extracted hendelser, men vi har sagt at vi også skal konvertere + Når extract result kommer inn + Så: + Skal vi si pending på convert + Listener skal returnerere null + """ + ) + fun failure1() { + val started = defaultStartEvent() + + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + val encode = encodeEvent("/tmp/video.mp4", parsed) + val extract = extractEvent("en", "/tmp/sub1.srt", encode.last()) + + val history = listOf( + started, + parsed, + *encode.toTypedArray(), + *extract.toTypedArray(), + ) + val result = listener.onEvent(history.last(), history) + assertThat(result).isNull() + } + + @Test + @DisplayName( + """ + Hvis historikken har alle påkrevde media hendelser, men venter på metadata + Når onEvent kalles og projeksjonen tilsier ugyldig tilstand + Så: + Returerer vi failure + """ + ) + fun failure2() { + val started = defaultStartEvent() + + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + val metadata = metadataEvent(parsed).first() + val encode = encodeEvent("/tmp/video.mp4", parsed) + val extract = extractEvent("en", "/tmp/sub1.srt", encode.last()) + val convert = convertEvent(language = "en", baseName = "sub1", outputFiles = listOf("/tmp/sub1.vtt"), derivedFrom = extract.last()) + + val history = listOf( + started, + parsed, + metadata, + *encode.toTypedArray(), + *extract.toTypedArray(), + *convert.toTypedArray(), + ) + + val result = listener.onEvent(history.last(), history) + + assertThat(result).isNull() + } + @Test + @DisplayName( + """ + Hvis historikken har alle påkrevde hendelser og encode feilet + Når onEvent kalles og projeksjonen tilsier ugyldig tilstand + Så: + Collect feiler + """ + ) + fun failure3() { + val started = defaultStartEvent() + + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + val metadata = metadataEvent(parsed) + + val encode = encodeEvent("/tmp/video.mp4", parsed, TaskStatus.Failed) + val extract = extractEvent("en", "/tmp/sub1.srt", encode.last()) + val convert = convertEvent(language = "en", baseName = "sub1", outputFiles = listOf("/tmp/sub1.vtt"), derivedFrom = extract.last()) + val cover = coverEvent("/tmp/cover.jpg", metadata.last()) + + val history = listOf( + started, + parsed, + *metadata.toTypedArray(), + *encode.toTypedArray(), + *extract.toTypedArray(), + *convert.toTypedArray(), + *cover.toTypedArray(), + ) + + val result = listener.onEvent(history.last(), history) + + assertThat(result).isNull() + } + @Test + @DisplayName( + """ + Hvis ingen oppgaver har blitt gjort + Når onEvent kalles + Så: + Skal projeksjonen gi ugyldig tilstand og returnere null + """ + ) + fun failure4() { + val started = defaultStartEvent().let { ev -> + ev.copy(data = ev.data.copy(operation = setOf(OperationType.Encode))) + } + + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + + + val history = listOf( + started, + parsed, + ) + + val result = listener.onEvent(history.last(), history) + + assertThat(result).isNull() + } + + +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateConvertTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateConvertTaskListenerTest.kt new file mode 100644 index 00000000..0048bc8d --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateConvertTaskListenerTest.kt @@ -0,0 +1,247 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.TestBase +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import java.io.File + +import io.mockk.* +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.mockito.Mockito.mockStatic +import org.mockito.kotlin.any +import org.mockito.kotlin.whenever +import java.nio.file.Files +import java.nio.file.Path + +class MediaCreateConvertTaskListenerTest : TestBase() { + + private val listener = MediaCreateConvertTaskListener() + + @Test + @DisplayName(""" + Når en ProcesserExtractResultEvent mottas + Hvis historikken inneholder StartProcessingEvent med Convert og filen eksisterer + Så: + Skal det opprettes ConvertTask og returneres ConvertTaskCreatedEvent + """) + fun verifyConvertTaskCreatedOnValidHistory() { + val tempFile = File.createTempFile("test", ".srt") + tempFile.writeText("dummy subtitle") + + val startEvent = StartProcessingEvent( + data = StartData( + fileUri = tempFile.absolutePath, + operation = setOf(OperationType.Convert) + ) + ) + val extractEvent = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + cachedOutputFile = tempFile.absolutePath, + language = "en" + ) + ) + + val history = listOf(startEvent) + val result = listener.onEvent(extractEvent, history) + + assertNotNull(result) + assertTrue(result is ConvertTaskCreatedEvent) + + // verifiser at TaskStore.persist ble kalt med ConvertTask + verify { TaskStore.persist(match { it is ConvertTask }) } + } + + @Test + @DisplayName(""" + Når en ProcesserExtractResultEvent mottas + Hvis StartProcessingEvent mangler i historikken + Så: + Skal onEvent returnere null og TaskStore.persist ikke kalles + """) + fun verifyNullWhenNoStartEvent() { + val tempFile = File.createTempFile("test", ".srt") + val extractEvent = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + cachedOutputFile = tempFile.absolutePath, + language = "en" + ) + ) + + val history = emptyList() + val result = listener.onEvent(extractEvent, history) + + assertNull(result) + verify(exactly = 0) { TaskStore.persist(any()) } + } + + @Test + @DisplayName(""" + Når en ProcesserExtractResultEvent mottas + Hvis StartProcessingEvent finnes men operation ikke inneholder Convert + Så: + Skal onEvent returnere null + """) + fun verifyNullWhenOperationNotConvert() { + val tempFile = File.createTempFile("test", ".srt") + val startEvent = StartProcessingEvent( + data = StartData( + fileUri = tempFile.absolutePath, + operation = setOf(OperationType.Encode) // Ikke Convert + ) + ) + val extractEvent = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + cachedOutputFile = tempFile.absolutePath, + language = "en" + ) + ) + + val history = listOf(startEvent) + val result = listener.onEvent(extractEvent, history) + + assertNull(result) + verify(exactly = 0) { TaskStore.persist(any()) } + } + + @Test + @DisplayName(""" + Når en ProcesserExtractResultEvent mottas + Hvis status ikke er Completed + Så: + Skal onEvent returnere null + """) + fun verifyNullWhenStatusNotCompleted() { + val tempFile = File.createTempFile("test", ".srt") + val startEvent = StartProcessingEvent( + data = StartData( + fileUri = tempFile.absolutePath, + operation = setOf(OperationType.Convert) + ) + ) + val extractEvent = ProcesserExtractResultEvent( + status = TaskStatus.Failed, + data = ProcesserExtractResultEvent.ExtractResult( + cachedOutputFile = tempFile.absolutePath, + language = "en" + ) + ) + + val history = listOf(startEvent) + val result = listener.onEvent(extractEvent, history) + + assertNull(result) + verify(exactly = 0) { TaskStore.persist(any()) } + } + + @Test + @DisplayName(""" + Når en ProcesserExtractResultEvent mottas + Hvis data mangler (er null) + Så: + Skal onEvent returnere null + """) + fun verifyNullWhenDataIsNull() { + val startEvent = StartProcessingEvent( + data = StartData( + fileUri = "video.mp4", + operation = setOf(OperationType.Convert) + ) + ) + val extractEvent = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = null + ) + + val history = listOf(startEvent) + val result = listener.onEvent(extractEvent, history) + + assertNull(result) + verify(exactly = 0) { TaskStore.persist(any()) } + } + + @Test + @DisplayName(""" + Når en ProcesserExtractResultEvent mottas + Hvis cachedOutputFile ikke eksisterer + Så: + Skal onEvent returnere null + """) + fun verifyNullWhenFileDoesNotExist() { + val startEvent = StartProcessingEvent( + data = StartData( + fileUri = "nonexistent.srt", + operation = setOf(OperationType.Convert) + ) + ) + val extractEvent = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + cachedOutputFile = "nonexistent.srt", + language = "en" + ) + ) + + val history = listOf(startEvent) + val result = listener.onEvent(extractEvent, history) + + assertNull(result) + verify(exactly = 0) { TaskStore.persist(any()) } + } + + @Test + @DisplayName(""" + Når en ProcesserExtractResultEvent mottas + Hvis historikken inneholder StartEvent med Convert og File.exists() returnerer true + Så: + Skal det opprettes ConvertTask og returneres ConvertTaskCreatedEvent + """) + fun verifyConvertTaskCreatedWithMockedFileExists() { + // Intercept File konstruktør og mock exists() + mockStatic(Files::class.java).use { filesMock -> + + filesMock.`when` { + Files.exists(any()) + }.thenReturn(true) + + val startEvent = StartProcessingEvent( + data = StartData( + fileUri = "/tmp/video.srt", + operation = setOf(OperationType.Convert) + ) + ) + + val extractEvent = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + cachedOutputFile = "/tmp/video.srt", + language = "en" + ) + ) + + val history = listOf(startEvent) + val result = listener.onEvent(extractEvent, history) + + assertNotNull(result) + assertTrue(result is ConvertTaskCreatedEvent) + + filesMock.verify { + Files.exists(any()) + } + } + } +} + diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateCoverDownloadTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateCoverDownloadTaskListenerTest.kt new file mode 100644 index 00000000..7518f488 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateCoverDownloadTaskListenerTest.kt @@ -0,0 +1,5 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +class MediaCreateCoverDownloadTaskListenerTest { + +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateEncodeTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateEncodeTaskListenerTest.kt new file mode 100644 index 00000000..17a1377a --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateEncodeTaskListenerTest.kt @@ -0,0 +1,242 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.mockkObject +import io.mockk.verify +import no.iktdev.mediaprocessing.coordinator.AudioPreference +import no.iktdev.mediaprocessing.coordinator.Preference +import no.iktdev.mediaprocessing.coordinator.ProcesserPreference +import no.iktdev.mediaprocessing.coordinator.VideoPreference +import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream +import no.iktdev.mediaprocessing.ffmpeg.data.Disposition +import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams +import no.iktdev.mediaprocessing.ffmpeg.data.Tags +import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream +import no.iktdev.mediaprocessing.ffmpeg.dsl.AudioCodec +import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoCodec +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksEncodeSelectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + +class MediaCreateEncodeTaskListenerTest { + + private val listener = MediaCreateEncodeTaskListener() + + @BeforeEach + fun setup() { + mockkObject(TaskStore) + every { TaskStore.persist(any()) } just Runs + mockkObject(Preference) + every { Preference.getProcesserPreference() } returns ProcesserPreference( + videoPreference = VideoPreference(codec = VideoCodec.Hevc()), + audioPreference = AudioPreference(codec = AudioCodec.Aac(channels = 2)) + ) + } + + @Test + @DisplayName(""" + Hvis en video- og audio-track er valgt + Når onEvent kalles + Så: + TaskStore.persist mottar et EncodeTask + data-feltet har korrekt inputFile, outputFileName og arguments fra MediaPlan + """) + fun testOnEventWithSingleAudioTrack() { + val startEvent = StartProcessingEvent( + StartData(setOf(OperationType.Encode), fileUri = "/tmp/movie.mkv") + ) + val parsedEvent = MediaStreamParsedEvent( + data = ParsedMediaStreams( + videoStream = listOf(mockVideoStream(index = 0, codec = "h264", disposition = mockDisposition(), tags = mockTags())), + audioStream = listOf(mockAudioStream(index = 1, codec = "aac", disposition = mockDisposition(), tags = mockTags())) + ) + ) + val selectedEvent = MediaTracksEncodeSelectedEvent( + selectedVideoTrack = 0, + selectedAudioTrack = 0 + ) + + val history = listOf(startEvent, parsedEvent) + + val result = listener.onEvent(selectedEvent, history) + + verify { + TaskStore.persist(withArg { task -> + assertTrue(task is EncodeTask) + val data = (task as EncodeTask).data + assertEquals("/tmp/movie.mkv", data.inputFile) + assertEquals("movie.mp4", data.outputFileName) + assertTrue(data.arguments.isNotEmpty(), "Arguments from MediaPlan should not be empty") + }) + } + + assertTrue(result is ProcesserEncodeTaskCreatedEvent) + } + + @Test + @DisplayName(""" + Hvis en video- og to audio-tracks (inkludert extended) er valgt + Når onEvent kalles + Så: + TaskStore.persist mottar et EncodeTask + data-feltet inkluderer begge audio-targets i arguments + """) + fun testOnEventWithExtendedAudioTrack() { + val startEvent = StartProcessingEvent( + StartData(setOf(OperationType.Encode), fileUri = "/tmp/movie.mkv") + ) + val parsedEvent = MediaStreamParsedEvent( + data = ParsedMediaStreams( + videoStream = listOf(mockVideoStream(index = 0, codec = "h264", disposition = mockDisposition(), tags = mockTags())), + audioStream = listOf( + mockAudioStream(index = 1, codec = "aac", disposition = mockDisposition(), tags = mockTags()), + mockAudioStream(index = 2, codec = "aac", disposition = mockDisposition(), tags = mockTags()) + ) + ) + ) + val selectedEvent = MediaTracksEncodeSelectedEvent( + selectedVideoTrack = 0, + selectedAudioTrack = 0, + selectedAudioExtendedTrack = 1 + ) + + val history = listOf(startEvent, parsedEvent) + + val result = listener.onEvent(selectedEvent, history) + + verify { + TaskStore.persist(withArg { task -> + val data = (task as EncodeTask).data + assertEquals("/tmp/movie.mkv", data.inputFile) + assertEquals("movie.mp4", data.outputFileName) + // her kan du sjekke at begge audio-tracks er med i ffmpeg-args + assertTrue(data.arguments.any { it.contains("0:a:0") }) + assertTrue(data.arguments.any { it.contains("0:a:1") }) + }) + } + + assertTrue(result is ProcesserEncodeTaskCreatedEvent) + } + + // Dummy streams for test + fun mockVideoStream( + index: Int = 0, + codec: String = "h264", + width: Int = 1920, + height: Int = 1080, + disposition: Disposition, + tags: Tags + ) = VideoStream( + index = index, + codec_name = codec, + codec_long_name = "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10", + codec_type = "video", + codec_tag_string = "avc1", + codec_tag = "0x31637661", + r_frame_rate = "25/1", + avg_frame_rate = "25/1", + time_base = "1/90000", + start_pts = 0, + start_time = "0.000000", + disposition = disposition, + tags = tags, + duration = "60.0", + duration_ts = 54000, + profile = "High", + width = width, + height = height, + coded_width = width, + coded_height = height, + closed_captions = 0, + has_b_frames = 2, + sample_aspect_ratio = "1:1", + display_aspect_ratio = "16:9", + pix_fmt = "yuv420p", + level = 40, + color_range = "tv", + color_space = "bt709", + color_transfer = "bt709", + color_primaries = "bt709", + chroma_location = "left", + refs = 1 + ) + + fun mockAudioStream( + index: Int = 0, + codec: String = "aac", + channels: Int = 2, + profile: String = "LC", + disposition: Disposition, + tags: Tags + ) = AudioStream( + index = index, + codec_name = codec, + codec_long_name = "AAC (Advanced Audio Coding)", + codec_type = "audio", + codec_tag_string = "mp4a", + codec_tag = "0x6134706d", + r_frame_rate = "0/0", + avg_frame_rate = "0/0", + time_base = "1/48000", + start_pts = 0, + start_time = "0.000000", + duration = "60.0", + duration_ts = 2880000, + disposition = disposition, + tags = tags, + profile = profile, + sample_fmt = "fltp", + sample_rate = "48000", + channels = channels, + channel_layout = "stereo", + bits_per_sample = 0 + ) + + fun mockDisposition( + default: Int = 1, + forced: Int = 0 + ) = Disposition( + default = default, + dub = 0, + original = 0, + comment = 0, + lyrics = 0, + karaoke = 0, + forced = forced, + hearing_impaired = 0, + captions = 0, + visual_impaired = 0, + clean_effects = 0, + attached_pic = 0, + timed_thumbnails = 0 + ) + + fun mockTags( + language: String? = "eng", + title: String? = null, + filename: String? = null + ) = Tags( + title = title, + BPS = null, + DURATION = null, + NUMBER_OF_FRAMES = 0, + NUMBER_OF_BYTES = null, + _STATISTICS_WRITING_APP = null, + _STATISTICS_WRITING_DATE_UTC = null, + _STATISTICS_TAGS = null, + language = language, + filename = filename, + mimetype = null + ) +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateExtractTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateExtractTaskListenerTest.kt new file mode 100644 index 00000000..12a439f5 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateExtractTaskListenerTest.kt @@ -0,0 +1,264 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + + +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.mockkObject +import io.mockk.verify +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.PersistedTask +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleTags +import no.iktdev.mediaprocessing.ffmpeg.data.Tags +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksExtractSelectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleTask +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import java.io.File +import java.time.Duration +import java.util.UUID + +class MediaCreateExtractTaskListenerTest { + + object FakeTaskStore: no.iktdev.eventi.stores.TaskStore { + val persisted = mutableListOf() + override fun persist(task: Task) { + persisted.add(task) + } + + override fun findByTaskId(taskId: UUID): PersistedTask? { TODO("Not yet implemented") } + override fun findByReferenceId(referenceId: UUID): List { TODO("Not yet implemented") } + override fun findUnclaimed(referenceId: UUID): List { TODO("Not yet implemented") } + override fun claim(taskId: UUID, workerId: String): Boolean { TODO("Not yet implemented") } + override fun heartbeat(taskId: UUID) { TODO("Not yet implemented") } + override fun markConsumed(taskId: UUID, status: TaskStatus) { TODO("Not yet implemented") } + override fun releaseExpiredTasks(timeout: Duration) { TODO("Not yet implemented") } + override fun getPendingTasks(): List { TODO("Not yet implemented") } + } + + @BeforeEach + fun setup() { + mockkObject(TaskStore) + every { TaskStore.persist(any()) } just Runs + } + + private val listener = MediaCreateExtractTaskListener() + + private fun dummyStream( + index: Int, + codecName: String, + language: String? = null + ): SubtitleStream { + return SubtitleStream( + index = index, + codec_name = codecName, + codec_long_name = codecName, + codec_type = "subtitle", + codec_tag_string = "", + codec_tag = "", + r_frame_rate = "0/0", + avg_frame_rate = "0/0", + time_base = "1/1000", + start_pts = 0, + start_time = "0", + duration = null, + duration_ts = null, + disposition = null, + tags = Tags( + title = null, + BPS = null, + DURATION = null, + NUMBER_OF_FRAMES = 0, + NUMBER_OF_BYTES = null, + _STATISTICS_WRITING_APP = null, + _STATISTICS_WRITING_DATE_UTC = null, + _STATISTICS_TAGS = null, + language = language, + filename = null, + mimetype = null + ), + subtitle_tags = SubtitleTags( + language = language, + filename = null, + mimetype = null + ) + ) + } + + @Test + @DisplayName(""" + Hvis en SRT-subtitle med språk er valgt + Når toSubtitleArgumentData kalles + Så: + Returneres et ExtractSubtitleData-objekt + Outputfilen får .srt-extension og språk i navnet + Argumentlisten inneholder -map og -c:s copy + """) + fun testSrtSubtitle() { + val stream = dummyStream(0, "subrip", "eng") + val inputFile = File("/tmp/movie.mkv") + + val result = listener.toSubtitleArgumentData(0, inputFile, stream) + + assertNotNull(result) + assertEquals("movie-eng.srt", result!!.outputFileName) + assertEquals("eng", result.language) + assertEquals(listOf("-map", "0:s:0", "-c:s", "copy"), result.arguments) + } + + @Test + @DisplayName(""" + Hvis codec ikke støttes (f.eks pgssub) + Når toSubtitleArgumentData kalles + Så: + Returneres null + Ingen ExtractSubtitleData opprettes + """) + fun testUnsupportedCodec() { + val stream = dummyStream(1, "pgssub", "eng") + val inputFile = File("/tmp/movie.mkv") + + val result = listener.toSubtitleArgumentData(1, inputFile, stream) + + assertNull(result) + } + + @Test + @DisplayName(""" + Hvis språk mangler i subtitle-stream + Når toSubtitleArgumentData kalles + Så: + Returneres null + Ingen ExtractSubtitleData opprettes + """) + fun testMissingLanguage() { + val stream = dummyStream(2, "subrip", null) + val inputFile = File("/tmp/movie.mkv") + + val result = listener.toSubtitleArgumentData(2, inputFile, stream) + + assertNull(result) + } + + @Test + @DisplayName(""" + Hvis en ASS-subtitle med språk er valgt + Når toSubtitleArgumentData kalles + Så: + Returneres et ExtractSubtitleData-objekt + Outputfilen får .ass-extension og språk i navnet + Argumentlisten inneholder -map og -c:s copy + """) + fun testAssSubtitle() { + val stream = dummyStream(3, "ass", "jpn") + val inputFile = File("/tmp/anime.mkv") + + val result = listener.toSubtitleArgumentData(3, inputFile, stream) + + assertNotNull(result) + assertEquals("anime-jpn.ass", result!!.outputFileName) + assertEquals("jpn", result.language) + assertEquals(listOf("-map", "0:s:3", "-c:s", "copy"), result.arguments) + } + + @Test + @DisplayName(""" + Hvis en StartProcessingEvent og MediaStreamParsedEvent finnes i historikken + Når onEvent kalles med MediaTracksExtractSelectedEvent som velger en SRT-subtitle + Så: + Returneres et ProcesserExtractTaskCreatedEvent + tasksCreated-listen inneholder minst én UUID + """) + fun testOnEventCreatesTasks() { + val startEvent = StartProcessingEvent( + StartData(setOf(OperationType.Extract), fileUri = "/tmp/movie.mkv") + ) + val parsedEvent = MediaStreamParsedEvent( + data = ParsedMediaStreams(subtitleStream = listOf(dummyStream(0, "subrip", "eng"))) + ) + val selectedEvent = MediaTracksExtractSelectedEvent(selectedSubtitleTracks = listOf(0)) + + val history = listOf(startEvent, parsedEvent) + + val result = listener.onEvent(selectedEvent, history) + + assertNotNull(result) + assertTrue(result is ProcesserExtractTaskCreatedEvent) + val created = result as ProcesserExtractTaskCreatedEvent + assertTrue(created.tasksCreated.isNotEmpty()) + verify { + TaskStore.persist(withArg { task -> + assertTrue(task is ExtractSubtitleTask) + val data = (task as ExtractSubtitleTask).data + assertEquals("/tmp/movie.mkv", data.inputFile) + assertEquals("movie-eng.srt", data.outputFileName) + assertEquals("eng", data.language) + assertEquals(listOf("-map", "0:s:0", "-c:s", "copy"), data.arguments) + }) + } + } + + @Test + @DisplayName(""" + Hvis flere undertekster (SRT og ASS) er valgt + Når onEvent kalles + Så: + TaskStore.persist skal kalles én gang per valgt spor + Hvert ExtractSubtitleTask skal ha korrekt data (filnavn, språk, arguments) + """) + fun testOnEventWithMultipleSubtitles() { + // Hvis: vi har en StartProcessingEvent og to subtitle streams + val startEvent = StartProcessingEvent( + StartData(setOf(OperationType.Extract), fileUri = "/tmp/movie.mkv") + ) + val parsedEvent = MediaStreamParsedEvent( + data = ParsedMediaStreams( + subtitleStream = listOf( + dummyStream(0, "subrip", "eng"), + dummyStream(1, "ass", "jpn") + ) + ) + ) + val selectedEvent = MediaTracksExtractSelectedEvent(selectedSubtitleTracks = listOf(0, 1)) + + val history = listOf(startEvent, parsedEvent) + + // Når: vi kaller onEvent + val result = listener.onEvent(selectedEvent, history) + + // Så: TaskStore.persist skal ha blitt kalt to ganger + verify(exactly = 2) { TaskStore.persist(any()) } + + // Fang begge objektene + val slot = mutableListOf() + verify { TaskStore.persist(capture(slot)) } + + // Sjekk første (SRT) + val srtTask = slot[0] as ExtractSubtitleTask + assertEquals("movie-eng.srt", srtTask.data.outputFileName) + assertEquals("eng", srtTask.data.language) + assertEquals(listOf("-map", "0:s:0", "-c:s", "copy"), srtTask.data.arguments) + + // Sjekk andre (ASS) + val assTask = slot[1] as ExtractSubtitleTask + assertEquals("movie-jpn.ass", assTask.data.outputFileName) + assertEquals("jpn", assTask.data.language) + assertEquals(listOf("-map", "0:s:1", "-c:s", "copy"), assTask.data.arguments) + + // Og: resultatet er et ProcesserExtractTaskCreatedEvent med to taskIds + assertTrue(result is ProcesserExtractTaskCreatedEvent) + val created = result as ProcesserExtractTaskCreatedEvent + assertEquals(2, created.tasksCreated.size) + } +} diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateMetadataSearchTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateMetadataSearchTaskListenerTest.kt new file mode 100644 index 00000000..64aaec06 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaCreateMetadataSearchTaskListenerTest.kt @@ -0,0 +1,5 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +class MediaCreateMetadataSearchTaskListenerTest { + +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaDetermineSubtitleTrackTypeListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaDetermineSubtitleTrackTypeListenerTest.kt new file mode 100644 index 00000000..8553692d --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaDetermineSubtitleTrackTypeListenerTest.kt @@ -0,0 +1,176 @@ +@file:Suppress("JUnitMalformedDeclaration") + +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleTags +import no.iktdev.mediaprocessing.ffmpeg.data.Tags +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent +import no.iktdev.mediaprocessing.shared.common.model.SubtitleType +import org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Named +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import java.util.stream.Stream + +class MediaDetermineSubtitleTrackTypeListenerTest { + + private val listener = MediaDetermineSubtitleTrackTypeListener() + + + + data class SubtitleTestCase( + val stream: SubtitleStream, + val expectedType: SubtitleType, + val expectedKept: Boolean + ) + + companion object { + + private fun makeStream(codec: String, title: String?, language: String = "eng"): SubtitleStream { + return SubtitleStream( + index = 0, + codec_name = codec, + codec_long_name = codec, + codec_type = codec, // NB: her brukes codec_type i onlySupportedCodecs + codec_tag_string = "", + codec_tag = "", + r_frame_rate = "0/0", + avg_frame_rate = "0/0", + time_base = "1/1000", + start_pts = 0, + start_time = "0", + duration = null, + duration_ts = null, + disposition = null, + tags = Tags( + title = title, + BPS = null, + DURATION = null, + NUMBER_OF_FRAMES = 0, + NUMBER_OF_BYTES = null, + _STATISTICS_WRITING_APP = null, + _STATISTICS_WRITING_DATE_UTC = null, + _STATISTICS_TAGS = null, + language = language, + filename = null, + mimetype = null + ), + subtitle_tags = SubtitleTags(language = language, filename = null, mimetype = null) + ) + } + + @JvmStatic + fun subtitleCases(): Stream> { + return Stream.of( + Named.of("Commentary filtered out", + SubtitleTestCase( + stream = makeStream("ass", "Director Commentary"), + expectedType = SubtitleType.Commentary, + expectedKept = false + ) + ), + Named.of("Song filtered out", + SubtitleTestCase( + stream = makeStream("subrip", "Song Lyrics"), + expectedType = SubtitleType.Song, + expectedKept = false + ) + ), + Named.of("Closed Caption filtered out", + SubtitleTestCase( + stream = makeStream("webvtt", "Closed Caption"), + expectedType = SubtitleType.ClosedCaption, + expectedKept = false + ) + ), + Named.of("SHD filtered out", + SubtitleTestCase( + stream = makeStream("smi", "SHD"), + expectedType = SubtitleType.SHD, + expectedKept = false + ) + ), + Named.of("Dialogue kept", + SubtitleTestCase( + stream = makeStream("ass", "Normal Dialogue"), + expectedType = SubtitleType.Dialogue, + expectedKept = true + ) + ), + Named.of("Unsupported codec filtered out", + SubtitleTestCase( + stream = makeStream("pgssub", "Dialogue"), + expectedType = SubtitleType.Dialogue, + expectedKept = false + ) + ), + Named.of("Commentary with typo", + SubtitleTestCase( + stream = makeStream("ass", "Comentary track"), // missing 'm' + expectedType = SubtitleType.Commentary, + expectedKept = false + ) + ), + Named.of("Song with variant spelling", + SubtitleTestCase( + stream = makeStream("subrip", "Sogn lyrics"), // 'song' misspelled + expectedType = SubtitleType.Song, + expectedKept = false + ) + ), + Named.of("Closed Caption with dash", + SubtitleTestCase( + stream = makeStream("webvtt", "Closed-caption subs"), + expectedType = SubtitleType.ClosedCaption, + expectedKept = false + ) + ), + Named.of("SHD with abbreviation", + SubtitleTestCase( + stream = makeStream("smi", "HH subs"), // 'hh' is in SHD filters + expectedType = SubtitleType.SHD, + expectedKept = false + ) + ), + Named.of("Dialogue with extra tags", + SubtitleTestCase( + stream = makeStream("ass", "Dialogue [ENG] normal"), + expectedType = SubtitleType.Dialogue, + expectedKept = true + ) + ), + Named.of("Unsupported codec with random title", + SubtitleTestCase( + stream = makeStream("pgssub", "Commentary track"), + expectedType = SubtitleType.Commentary, + expectedKept = false + ) + ) + ) + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("subtitleCases") + @DisplayName("Hvis ulike subtitles testes → riktig type og filtrering") + fun testSubtitleCases(testCase: SubtitleTestCase) { + val event = MediaStreamParsedEvent( + ParsedMediaStreams(subtitleStream = listOf(testCase.stream)) + ) + val result = listener.onEvent(event, emptyList()) as MediaTracksDetermineSubtitleTypeEvent + + if (testCase.expectedKept) { + assertEquals(1, result.subtitleTrackItems.size) + assertEquals(testCase.expectedType, result.subtitleTrackItems[0].type) + } else { + assertTrue(result.subtitleTrackItems.isEmpty()) + } + } +} diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParseStreamsListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParseStreamsListenerTest.kt new file mode 100644 index 00000000..99cda469 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParseStreamsListenerTest.kt @@ -0,0 +1,188 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import org.junit.jupiter.api.Assertions.* + +import com.google.gson.JsonParser +import no.iktdev.eventi.models.Event +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + +class MediaParseStreamsListenerTest { + + private val listener = MediaParseStreamsListener() + + class DummyEvent(): Event() {} + + @Test + @DisplayName(""" + Hvis JSON inneholder video, audio og subtitle streams + Når parseStreams kalles + Så: + Alle tre typer havner i riktig liste + """) + fun testparseMappingCorrectly() { + val json = """ + { + "streams": [ + { + "codec_name":"h264", + "codec_long_name":"H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10", + "codec_type":"video", + "codec_tag_string":"avc1", + "codec_tag":"0x31637661", + "r_frame_rate":"25/1", + "avg_frame_rate":"25/1", + "time_base":"1/90000", + "start_pts":0, + "start_time":"0.000000", + "disposition": { "default":1,"dub":0,"original":0,"comment":0,"lyrics":0,"karaoke":0,"forced":0,"hearing_impaired":0,"captions":0,"visual_impaired":0,"clean_effects":0,"attached_pic":0,"timed_thumbnails":0 }, + "tags": { "title":"Main Video","language":"eng" }, + "profile":"High", + "width":1920, + "height":1080, + "coded_width":1920, + "coded_height":1080, + "closed_captions":0, + "has_b_frames":2, + "sample_aspect_ratio":"1:1", + "display_aspect_ratio":"16:9", + "pix_fmt":"yuv420p", + "level":40, + "color_range":"tv", + "color_space":"bt709", + "color_transfer":"bt709", + "color_primaries":"bt709", + "chroma_location":"left", + "refs":1 + }, + { + "codec_name":"aac", + "codec_long_name":"AAC (Advanced Audio Coding)", + "codec_type":"audio", + "codec_tag_string":"mp4a", + "codec_tag":"0x6134706d", + "r_frame_rate":"0/0", + "avg_frame_rate":"0/0", + "time_base":"1/48000", + "start_pts":0, + "start_time":"0.000000", + "disposition": { "default":1,"dub":0,"original":0,"comment":0,"lyrics":0,"karaoke":0,"forced":0,"hearing_impaired":0,"captions":0,"visual_impaired":0,"clean_effects":0,"attached_pic":0,"timed_thumbnails":0 }, + "tags": { "title":"Stereo Track","language":"eng" }, + "profile":"LC", + "sample_fmt":"fltp", + "sample_rate":"48000", + "channels":2, + "channel_layout":"stereo", + "bits_per_sample":0 + }, + { + "codec_name":"ass", + "codec_long_name":"ASS (Advanced SSA Subtitle)", + "codec_type":"subtitle", + "codec_tag_string":"[0][0][0][0]", + "codec_tag":"0x0000", + "r_frame_rate":"0/0", + "avg_frame_rate":"0/0", + "time_base":"1/1000", + "start_pts":0, + "start_time":"0.000000", + "disposition": { "default":1,"dub":0,"original":0,"comment":0,"lyrics":0,"karaoke":0,"forced":0,"hearing_impaired":0,"captions":0,"visual_impaired":0,"clean_effects":0,"attached_pic":0,"timed_thumbnails":0 }, + "tags": { "title":"English Subs","language":"eng" }, + "subtitle_tags": { "language":"eng","filename":"subs.ass","mimetype":"text/x-ssa" } + } + ] + } + """.trimIndent() + + val parsed = listener.parseStreams(JsonParser.parseString(json).asJsonObject) + + assertEquals(1, parsed.videoStream.size) + assertEquals("h264", parsed.videoStream[0].codec_name) + + assertEquals(1, parsed.audioStream.size) + assertEquals("aac", parsed.audioStream[0].codec_name) + + assertEquals(1, parsed.subtitleStream.size) + assertEquals("ass", parsed.subtitleStream[0].codec_name) + } + + + + @Test + @DisplayName(""" + Hvis event ikke er MediaStreamReadEvent + Når onEvent kalles + Så: + Returneres null + """) + fun testOnEventNonMediaStreamReadEvent() { + val result = listener.onEvent(DummyEvent(), emptyList()) + assertNull(result) + } + + @Test + @DisplayName(""" + Hvis JSON inneholder video, audio og subtitle streams + Når parseStreams kalles + Så: + Alle tre typer havner i riktig liste + """) + fun testParseStreamsMapsCorrectly() { + val json = """ + { + "streams": [ + {"codec_name":"h264","codec_type":"video"}, + {"codec_name":"aac","codec_type":"audio"}, + {"codec_name":"ass","codec_type":"subtitle"} + ] + } + """.trimIndent() + + val parsed = listener.parseStreams(JsonParser.parseString(json).asJsonObject) + + assertEquals(1, parsed.videoStream.size) + assertEquals("h264", parsed.videoStream[0].codec_name) + + assertEquals(1, parsed.audioStream.size) + assertEquals("aac", parsed.audioStream[0].codec_name) + + assertEquals(1, parsed.subtitleStream.size) + assertEquals("ass", parsed.subtitleStream[0].codec_name) + } + + @Test + @DisplayName(""" + Hvis JSON inneholder codec_name png og mjpeg + Når parseStreams kalles + Så: + Disse ignoreres og videoStream blir tom + """) + fun testParseStreamsIgnoresPngAndMjpeg() { + val json = """ + { + "streams": [ + {"codec_name":"png","codec_type":"video"}, + {"codec_name":"mjpeg","codec_type":"video"} + ] + } + """.trimIndent() + + val parsed = listener.parseStreams(JsonParser.parseString(json).asJsonObject) + assertTrue(parsed.videoStream.isEmpty()) + } + + @Test + @DisplayName(""" + Hvis JSON mangler streams array + Når parseStreams kalles + Så: + Kastes Exception + """) + fun testParseStreamsThrowsOnInvalidJson() { + val json = """{}""" + assertThrows(Exception::class.java) { + listener.parseStreams(JsonParser.parseString(json).asJsonObject) + } + } +} diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/events/MediaParsedInfoListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParsedInfoListenerTest.kt similarity index 86% rename from apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/events/MediaParsedInfoListenerTest.kt rename to apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParsedInfoListenerTest.kt index 70a0aaf2..9989f6d1 100644 --- a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/events/MediaParsedInfoListenerTest.kt +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaParsedInfoListenerTest.kt @@ -1,41 +1,43 @@ -package no.iktdev.mediaprocessing.coordinator.events +package no.iktdev.mediaprocessing.coordinator.listeners.events -import no.iktdev.mediaprocessing.coordinator.listeners.events.MediaParsedInfoListener +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent import no.iktdev.mediaprocessing.shared.common.model.MediaType -import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions import org.junit.jupiter.api.Named +import org.junit.jupiter.api.TestInstance import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource import java.io.File +@TestInstance(TestInstance.Lifecycle.PER_CLASS) class MediaParsedInfoListenerTest : MediaParsedInfoListener() { - - @MethodSource("parsedInfoTest") @ParameterizedTest(name = "{0}") + @MethodSource("parsedInfoTestCases") fun parsedInfoTest(testCase: ParsedInfoTestCase) { val testFile = testCase.file val collection = testFile.getDesiredCollection() val fileName = testFile.guessDesiredFileName() val searchTitles = testFile.guessSearchableTitle() - assertThat(collection).isEqualTo(testCase.expectedTitle) - assertThat(fileName).isEqualTo(testCase.expectedFileName) - assertThat(searchTitles).isEqualTo(testCase.expectedSearchTitles) + Assertions.assertThat(collection).isEqualTo(testCase.expectedTitle) + Assertions.assertThat(fileName).isEqualTo(testCase.expectedFileName) + Assertions.assertThat(searchTitles).isEqualTo(testCase.expectedSearchTitles) } - @MethodSource("parseVideoType") + @MethodSource("parseVideoTypeCases") @ParameterizedTest(name = "{0}") fun parseVideoType(testCase: ParseVideoTypeTestCase) { val testFile = testCase.file val mediaType = testFile.guessMovieOrSeries() - assertThat(mediaType).isEqualTo(testCase.expectedType) + Assertions.assertThat(mediaType).isEqualTo(testCase.expectedType) } data class ParsedInfoTestCase( val file: File, val expectedTitle: String, val expectedFileName: String, - val expectedSearchTitles: List + val expectedSearchTitles: List, + val expectedEpisodeInfo: MediaParsedInfoEvent.ParsedData.EpisodeInfo? = null ) data class ParseVideoTypeTestCase( @@ -46,7 +48,7 @@ class MediaParsedInfoListenerTest : MediaParsedInfoListener() { companion object { @JvmStatic - fun parsedInfoTest() = listOf( + fun parsedInfoTestCases() = listOf( // existing parsed cases Named.of( "Series episode parsing", @@ -54,7 +56,12 @@ class MediaParsedInfoListenerTest : MediaParsedInfoListener() { file = File("Fancy.Thomas.S03E03.Enemy.1080p.AMAZING.WEB-VALUE.DDP5AN.1.H.264.mkv"), expectedTitle = "Fancy Thomas", expectedFileName = "Fancy Thomas - S03E03 - Enemy", - expectedSearchTitles = listOf("Fancy Thomas", "Fancy Thomas - S03E03 - Enemy") + expectedSearchTitles = listOf("Fancy Thomas", "Fancy Thomas - S03E03 - Enemy"), + expectedEpisodeInfo = MediaParsedInfoEvent.ParsedData.EpisodeInfo( + seasonNumber = 3, + episodeNumber = 3, + episodeTitle = "Enemy" + ) ) ), Named.of( @@ -63,7 +70,8 @@ class MediaParsedInfoListenerTest : MediaParsedInfoListener() { file = File("Epic.Potato.Movie.2021.1080p.BluRay.x264.mkv"), expectedTitle = "Epic Potato Movie", expectedFileName = "Epic Potato Movie", - expectedSearchTitles = listOf("Epic Potato Movie") + expectedSearchTitles = listOf("Epic Potato Movie"), + expectedEpisodeInfo = null ) ), Named.of( @@ -72,7 +80,12 @@ class MediaParsedInfoListenerTest : MediaParsedInfoListener() { file = File("Like.a.Potato.Chef.S01E01.Departure.\\u0026.Skills.1080p.Potato.mkv"), expectedTitle = "Like a Potato Chef", expectedFileName = "Like a Potato Chef - S01E01 - Departure \\u0026 Skills", - expectedSearchTitles = listOf("Like a Potato Chef", "Like a Potato Chef - S01E01 - Departure \\u0026 Skills") + expectedSearchTitles = listOf("Like a Potato Chef", "Like a Potato Chef - S01E01 - Departure \\u0026 Skills"), + expectedEpisodeInfo = MediaParsedInfoEvent.ParsedData.EpisodeInfo( + seasonNumber = 1, + episodeNumber = 1, + episodeTitle = "Departure \\u0026 Skills" + ) ) ), Named.of( @@ -121,6 +134,11 @@ class MediaParsedInfoListenerTest : MediaParsedInfoListener() { expectedSearchTitles = listOf( "Dumb ways to die", "Dumb ways to die - S01E03 - How to unlucky i am" + ), + expectedEpisodeInfo = MediaParsedInfoEvent.ParsedData.EpisodeInfo( + episodeTitle = "How to unlucky i am", + episodeNumber = 3, + seasonNumber = 1 ) ) ), @@ -202,7 +220,11 @@ class MediaParsedInfoListenerTest : MediaParsedInfoListener() { file = File("Show.Name.S01.E02.720p.HDTV.x264-Group_v2.mkv"), expectedTitle = "Show Name", expectedFileName = "Show Name - S01E02", - expectedSearchTitles = listOf("Show Name", "Show Name - S01E02") + expectedSearchTitles = listOf("Show Name", "Show Name - S01E02"), + expectedEpisodeInfo = MediaParsedInfoEvent.ParsedData.EpisodeInfo( + episodeNumber = 2, + seasonNumber = 1 + ) ) ), Named.of( @@ -235,7 +257,7 @@ class MediaParsedInfoListenerTest : MediaParsedInfoListener() { ) @JvmStatic - fun parseVideoType() = listOf( + fun parseVideoTypeCases() = listOf( Named.of( "Series file detection full block", ParseVideoTypeTestCase( diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaReadStreamsTaskCreatedListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaReadStreamsTaskCreatedListenerTest.kt new file mode 100644 index 00000000..e5f414e3 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaReadStreamsTaskCreatedListenerTest.kt @@ -0,0 +1,75 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import no.iktdev.mediaprocessing.TestBase +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + +class MediaReadStreamsTaskCreatedListenerTest: TestBase() { + + private val listener = MediaReadStreamsTaskCreatedListener() + + @Test + @DisplayName(""" + Hvis event ikke er MediaParsedInfoEvent + Når onEvent kalles + Så: + Returneres null + """) + fun testOnEventNonParsedInfoEvent() { + val result = listener.onEvent(DummyEvent(), emptyList()) + assertNull(result) + } + + @Test + @DisplayName(""" + Hvis event er MediaParsedInfoEvent men history mangler StartProcessingEvent + Når onEvent kalles + Så: + Returneres null + """) + fun testOnEventParsedInfoEventWithoutStartProcessing() { + val parsedEvent = MediaParsedInfoEvent( + MediaParsedInfoEvent.ParsedData( + parsedCollection = "collection", + parsedFileName = "file.mkv", + parsedSearchTitles = listOf("title"), + mediaType = MediaType.Movie + ) + ) + val result = listener.onEvent(parsedEvent, emptyList()) + assertNull(result) + } + + @Test + @DisplayName(""" + Hvis event er MediaParsedInfoEvent og history inneholder StartProcessingEvent + Når onEvent kalles + Så: + Returneres CoordinatorReadStreamsTaskCreatedEvent med riktig taskId + """) + fun testOnEventParsedInfoEventWithStartProcessing() { + val parsedEvent = MediaParsedInfoEvent( + MediaParsedInfoEvent.ParsedData( + parsedCollection = "collection", + parsedFileName = "file.mkv", + parsedSearchTitles = listOf("title"), + mediaType = MediaType.Movie + ) + ) + val startEvent = StartProcessingEvent(StartData(fileUri = "file://test.mkv", operation = emptySet())) + + val result = listener.onEvent(parsedEvent, listOf(startEvent)) + + assertNotNull(result) + assertTrue(result is CoordinatorReadStreamsTaskCreatedEvent) + + val coordinatorEvent = result as CoordinatorReadStreamsTaskCreatedEvent + assertNotNull(coordinatorEvent.taskId) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectEncodeTracksListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectEncodeTracksListenerTest.kt new file mode 100644 index 00000000..1ac67620 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectEncodeTracksListenerTest.kt @@ -0,0 +1,219 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream +import no.iktdev.mediaprocessing.ffmpeg.data.Disposition +import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams +import no.iktdev.mediaprocessing.ffmpeg.data.Tags +import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksEncodeSelectedEvent +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + +class MediaTracksEncodeSelectorTest: MediaSelectEncodeTracksListener() { + + private fun dummyAudioStream( + index: Int, + language: String, + channels: Int, + durationTs: Long = 1000 + ): AudioStream { + return AudioStream( + index = index, + codec_name = "aac", + codec_long_name = "AAC", + codec_type = "audio", + codec_tag_string = "", + codec_tag = "", + r_frame_rate = "0/0", + avg_frame_rate = "0/0", + time_base = "1/1000", + start_pts = 0, + start_time = "0", + duration = null, + duration_ts = durationTs, + disposition = Disposition(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + tags = Tags( + title = null, BPS = null, DURATION = null, NUMBER_OF_FRAMES = 0, + NUMBER_OF_BYTES = null, _STATISTICS_WRITING_APP = null, _STATISTICS_WRITING_DATE_UTC = null, + _STATISTICS_TAGS = null, language = language, filename = null, mimetype = null + ), + profile = "LC", + sample_fmt = "fltp", + sample_rate = "48000", + channels = channels, + channel_layout = "stereo", + bits_per_sample = 0 + ) + } + + private fun dummyVideoStream(index: Int, durationTs: Long = 1000): VideoStream { + return VideoStream( + index = index, + codec_name = "h264", + codec_long_name = "H.264", + codec_type = "video", + codec_tag_string = "", + codec_tag = "", + r_frame_rate = "25/1", + avg_frame_rate = "25/1", + time_base = "1/1000", + start_pts = 0, + start_time = "0", + duration = null, + duration_ts = durationTs, + disposition = Disposition(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), + tags = Tags( + title = null, BPS = null, DURATION = null, NUMBER_OF_FRAMES = 0, + NUMBER_OF_BYTES = null, _STATISTICS_WRITING_APP = null, _STATISTICS_WRITING_DATE_UTC = null, + _STATISTICS_TAGS = null, language = "eng", filename = null, mimetype = null + ), + profile = "main", + width = 1920, + height = 1080, + coded_width = 1920, + coded_height = 1080, + closed_captions = 0, + has_b_frames = 0, + sample_aspect_ratio = "1:1", + display_aspect_ratio = "16:9", + pix_fmt = "yuv420p", + level = 30, + color_range = "tv", + color_space = "bt709", + color_transfer = "bt709", + color_primaries = "bt709", + chroma_location = "left", + refs = 1 + ) + } + + @Test + @DisplayName(""" + Hvis video streams har ulik varighet + Når getVideoTrackToUse kalles + Så: + Returneres index til stream med lengst varighet + """) + fun testVideoTrackSelection() { + val streams = listOf(dummyVideoStream(0, 1000), dummyVideoStream(1, 5000)) + val index = getVideoTrackToUse(streams) + assertEquals(1, index) + } + + @Test + @DisplayName(""" + Hvis audio streams inneholder foretrukket språk jpn med 2 kanaler + Når getAudioDefaultTrackToUse kalles + Så: + Returneres index til jpn stereo track + """) + fun testAudioDefaultTrackSelectionPreferredLanguageStereo() { + val streams = listOf( + dummyAudioStream(0, "eng", 2), + dummyAudioStream(1, "jpn", 2), + dummyAudioStream(2, "jpn", 6) + ) + val index = getAudioDefaultTrackToUse(streams) + assertEquals(1, index) + } + + @Test + @DisplayName(""" + Hvis audio streams inneholder foretrukket språk jpn med 6 kanaler + Når getAudioExtendedTrackToUse kalles + Så: + Returneres index til jpn 6-kanals track + """) + fun testAudioExtendedTrackSelectionPreferredLanguageSurround() { + val streams = listOf( + dummyAudioStream(0, "jpn", 2), + dummyAudioStream(1, "jpn", 6) + ) + val defaultIndex = getAudioDefaultTrackToUse(streams) + val extendedIndex = getAudioExtendedTrackToUse(streams, defaultIndex) + assertEquals(0, defaultIndex) + assertEquals(1, extendedIndex) + } + + @Test + @DisplayName(""" + Hvis audio streams ikke matcher foretrukket språk + Når filterOnPreferredLanguage kalles + Så: + Returneres original liste uten filtrering + """) + fun testFilterOnPreferredLanguageFallback() { + val streams = listOf( + dummyAudioStream(0, "eng", 2), + dummyAudioStream(1, "fra", 2) + ) + val filtered = streams.filterOnPreferredLanguage() + assertEquals(streams.size, filtered.size) + } + + @Test + @DisplayName(""" + Hvis audio streams ikke matcher foretrukket språk + Når getAudioDefaultTrackToUse kalles + Så: + Velges et spor (fallback) selv om ingen matcher + """) + fun testAudioDefaultTrackFallbackSelection() { + val streams = listOf( + dummyAudioStream(0, "eng", 2), + dummyAudioStream(1, "fra", 2) + ) + + // filterOnPreferredLanguage skal returnere original listen + val filtered = streams.filterOnPreferredLanguage() + assertEquals(streams.size, filtered.size) + + // getAudioDefaultTrackToUse skal likevel velge et spor + val selectedIndex = getAudioDefaultTrackToUse(streams) + + // Sjekk at det faktisk er en gyldig index + assertTrue(selectedIndex in streams.indices) + + // I dette tilfellet velges siste med høyest index (1) + assertEquals(0, selectedIndex) + } + + + + class DummyEvent: Event() + + @Test + @DisplayName(""" + Hvis event ikke er MediaStreamParsedEvent + Når onEvent kalles + Så: + Returneres null + """) + fun testOnEventNonParsedEvent() { + val result = onEvent(DummyEvent(), emptyList()) + assertNull(result) + } + + @Test + @DisplayName(""" + Hvis event er MediaStreamParsedEvent med video og audio + Når onEvent kalles + Så: + Returneres MediaTracksEncodeSelectedEvent med riktige spor + """) + fun testOnEventParsedEvent() { + val videoStreams = listOf(dummyVideoStream(0, 1000)) + val audioStreams = listOf(dummyAudioStream(0, "jpn", 2), dummyAudioStream(1, "jpn", 6)) + val parsedEvent = MediaStreamParsedEvent( + ParsedMediaStreams(videoStream = videoStreams, audioStream = audioStreams, subtitleStream = emptyList()) + ) + val result = onEvent(parsedEvent, emptyList()) as MediaTracksEncodeSelectedEvent + assertEquals(0, result.selectedVideoTrack) + assertEquals(0, result.selectedAudioTrack) + assertEquals(1, result.selectedAudioExtendedTrack) + } +} diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectExtractTracksListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectExtractTracksListenerTest.kt new file mode 100644 index 00000000..d92c6b68 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MediaSelectExtractTracksListenerTest.kt @@ -0,0 +1,121 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import no.iktdev.mediaprocessing.TestBase +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleTags +import no.iktdev.mediaprocessing.ffmpeg.data.Tags +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksExtractSelectedEvent +import no.iktdev.mediaprocessing.shared.common.model.SubtitleItem +import no.iktdev.mediaprocessing.shared.common.model.SubtitleType +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + +class MediaSelectExtractTracksListenerTest: TestBase() { + + + // Vi lager en subclass som gir oss tilgang til alt og lar oss overstyre språkpreferanser + class TestableMediaSelectExtractTracksListener( + private val preferredLanguages: Set = emptySet() + ) : MediaSelectExtractTracksListener() { + override fun limitToLanguages(): Set = preferredLanguages + // gjør private extension tilgjengelig via wrapper + fun callFilterOnPreferredLanguage(streams: List): List { + return streams.filterOnPreferredLanguage() + } + } + + private fun dummySubtitleStream(index: Int, language: String?, type: SubtitleType): SubtitleItem { + val stream = SubtitleStream( + index = index, + codec_name = "ass", + codec_long_name = "ASS", + codec_type = "subtitle", + codec_tag_string = "", + codec_tag = "", + r_frame_rate = "0/0", + avg_frame_rate = "0/0", + time_base = "1/1000", + start_pts = 0, + start_time = "0", + duration = null, + duration_ts = 1000, + disposition = null, + tags = Tags( + title = null, BPS = null, DURATION = null, NUMBER_OF_FRAMES = 0, + NUMBER_OF_BYTES = null, _STATISTICS_WRITING_APP = null, _STATISTICS_WRITING_DATE_UTC = null, + _STATISTICS_TAGS = null, language = language, filename = null, mimetype = null + ), + subtitle_tags = SubtitleTags(language = language, filename = null, mimetype = null) + ) + return SubtitleItem(stream = stream, type = type) + } + + @Test + @DisplayName(""" + Hvis event ikke er MediaTracksDetermineSubtitleTypeEvent + Når onEvent kalles + Så: + Returneres null + """) + fun testOnEventNonSubtitleEvent() { + val listener = TestableMediaSelectExtractTracksListener() + val result = listener.onEvent(DummyEvent(), emptyList()) + assertNull(result) + } + + @Test + @DisplayName(""" + Hvis event inneholder Dialogue subtitles + Når onEvent kalles + Så: + Returneres MediaTracksExtractSelectedEvent med index til Dialogue tracks + """) + fun testOnEventDialogueTracksSelected() { + val listener = TestableMediaSelectExtractTracksListener() + val items = listOf( + dummySubtitleStream(0, "eng", SubtitleType.Dialogue), + dummySubtitleStream(1, "eng", SubtitleType.Commentary) + ) + val event = MediaTracksDetermineSubtitleTypeEvent(subtitleTrackItems = items) + val result = listener.onEvent(event, emptyList()) as MediaTracksExtractSelectedEvent + assertEquals(listOf(0), result.selectedSubtitleTracks) + } + + @Test + @DisplayName(""" + Hvis limitToLanguages returnerer jpn + Når filterOnPreferredLanguage kalles + Så: + Returneres kun spor med språk jpn + """) + fun testFilterOnPreferredLanguageWithLimit() { + val listener = TestableMediaSelectExtractTracksListener(setOf("jpn")) + val streams = listOf( + dummySubtitleStream(0, "eng", SubtitleType.Dialogue).stream, + dummySubtitleStream(1, "jpn", SubtitleType.Dialogue).stream + ) + val filtered = listener.callFilterOnPreferredLanguage(streams) + assertEquals(1, filtered.size) + assertEquals("jpn", filtered[0].tags.language) + } + + @Test + @DisplayName(""" + Hvis limitToLanguages er tom + Når filterOnPreferredLanguage kalles + Så: + Returneres original liste uten filtrering + """) + fun testFilterOnPreferredLanguageNoLimit() { + val listener = TestableMediaSelectExtractTracksListener() + val streams = listOf( + dummySubtitleStream(0, "eng", SubtitleType.Dialogue).stream, + dummySubtitleStream(1, "fra", SubtitleType.Dialogue).stream + ) + val filtered = listener.callFilterOnPreferredLanguage(streams) + assertEquals(streams.size, filtered.size) + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MigrateCreateStoreTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MigrateCreateStoreTaskListenerTest.kt new file mode 100644 index 00000000..ff75c1b4 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/MigrateCreateStoreTaskListenerTest.kt @@ -0,0 +1,302 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import io.mockk.slot +import io.mockk.verify +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.MockData.convertEvent +import no.iktdev.mediaprocessing.MockData.coverEvent +import no.iktdev.mediaprocessing.MockData.encodeEvent +import no.iktdev.mediaprocessing.MockData.extractEvent +import no.iktdev.mediaprocessing.MockData.mediaParsedEvent +import no.iktdev.mediaprocessing.MockData.metadataEvent +import no.iktdev.mediaprocessing.TestBase +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import java.io.File + +class MigrateCreateStoreTaskListenerTest : TestBase() { + + private val listener = MigrateCreateStoreTaskListener() + + @Test + @DisplayName( + """ + Hvis historikken inneholder gyldig parsed info, metadata og migreringsdata + Når onEvent kalles med CollectedEvent + Så: + Opprettes MigrateToContentStoreTask og sendes til TaskStore.persist +""" + ) + fun `creates migrate-to-store task`() { + val started = defaultStartEvent() + + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Movie + ).derivedOf(started) + + val metadata = metadataEvent(parsed) + + val encode = encodeEvent("/tmp/video.mp4", metadata.last()) + + val extract = extractEvent("en", "/tmp/sub1.srt", encode.last()) + + val coverDownload = coverEvent("/tmp/cover.jpg", metadata.last()) + + val convert = convertEvent( + language = "en", + baseName = "sub1", + outputFiles = listOf("/tmp/sub1.vtt"), + derivedFrom = extract.last() + ) + + val migrate = migrateResultEvent( + collection = "MyCollection", + videoUri = "file:///video.mp4", + coverUri = "file:///cover.jpg", + subtitleUris = listOf("file:///sub1.srt", "file://sub1.vtt") + ).derivedOf(convert.last()) + + val collected = CollectedEvent( + setOf( + started.eventId, + parsed.eventId, + *metadata.map { it.eventId }.toTypedArray(), + *encode.map { it.eventId }.toTypedArray(), + *extract.map { it.eventId }.toTypedArray(), + *convert.map { it.eventId }.toTypedArray(), + *coverDownload.map { it.eventId }.toTypedArray(), + migrate.eventId + ) + ).derivedOf(migrate) + + val history = listOf( + started, + parsed, + *metadata.toTypedArray(), + *encode.toTypedArray(), + *extract.toTypedArray(), + *convert.toTypedArray(), + *coverDownload.toTypedArray(), + migrate, + collected + ) + + val result = listener.onEvent(collected, history) + + assertThat(result).isNotNull() + + verify(exactly = 1) { + TaskStore.persist(withArg { task -> + val storeTask = task as MigrateToContentStoreTask + + assertThat(storeTask.data.collection).isEqualTo("MyCollection") + assertThat(storeTask.data.videoContent).isNotNull() + assertThat(storeTask.data.subtitleContent).hasSize(2) + assertThat(storeTask.data.coverContent).hasSize(1) + }) + } + + } + + @Test + @DisplayName( + """ + Hvis historikken inneholder gyldig parsed info, metadata og migreringsdata + Når onEvent kalles med CollectedEvent + Så: + Opprettes MigrateToContentStoreTask og sendes til TaskStore.persist +""" + ) + fun success1() { + val started = defaultStartEvent() + + val parsed = mediaParsedEvent( + collection = "Baking Bread", + fileName = "Baking Bread - S01E01 - Flour", + mediaType = MediaType.Serie + ).derivedOf(started) + + val metadata = metadataEvent(parsed) + + val encode = encodeEvent("/tmp/video.mp4", metadata.last()) + + val extract = extractEvent("en", "/tmp/sub1.srt", encode.last()) + + val coverDownload = coverEvent("/tmp/cover.jpg", metadata.last()) + val coverDownload2 = coverEvent("/tmp/cover.jpg", metadata.last(), "potet") + + val convert = convertEvent( + language = "en", + baseName = "sub1", + outputFiles = listOf("/tmp/sub1.vtt"), + derivedFrom = extract.last() + ) + + + val collected = CollectedEvent( + setOf( + started.eventId, + parsed.eventId, + *metadata.map { it.eventId }.toTypedArray(), + *encode.map { it.eventId }.toTypedArray(), + *extract.map { it.eventId }.toTypedArray(), + *convert.map { it.eventId }.toTypedArray(), + *coverDownload.map { it.eventId }.toTypedArray(), + *coverDownload2.map { it.eventId }.toTypedArray(), + ) + ).derivedOf(coverDownload.last()) + + val history = listOf( + started, + parsed, + *metadata.toTypedArray(), + *encode.toTypedArray(), + *extract.toTypedArray(), + *convert.toTypedArray(), + *coverDownload.toTypedArray(), + *coverDownload2.toTypedArray(), + collected, + ) + + val result = listener.onEvent(collected, history) + + assertThat(result).isNotNull() + + val slot = slot() + + verify(exactly = 1) { + TaskStore.persist(capture(slot)) + } + + val storeTask = slot.captured + + assertThat(storeTask.data.collection).isEqualTo("Baking Bread") + assertThat(storeTask.data.videoContent).isNotNull() + assertThat(storeTask.data.videoContent?.storeUri.let { f -> File(f).name }) + .isEqualTo("Baking Bread - S01E01 - Flour.mp4") + + assertThat(storeTask.data.subtitleContent).hasSize(2) + assertThat( + storeTask.data.subtitleContent!! + .map { File(it.storeUri).nameWithoutExtension } + ).containsOnly("Baking Bread - S01E01 - Flour") + + assertThat(storeTask.data.coverContent).hasSize(2) + assertThat(File(storeTask.data.coverContent!!.first().storeUri).name) + .isEqualTo("Baking Bread-test.jpg") + assertThat(File(storeTask.data.coverContent!!.last().storeUri).name) + .isEqualTo("Baking Bread-potet.jpg") + + + } + + + @Test + @DisplayName( + """ + Hvis start hendelsen kun inneholder converter, og vi har gjennomført konvertering + Når onEvent kalles med CollectedEvent + Så: + Opprettes det migrate task + """ + ) + fun createMigrateForConvert() { + val started = defaultStartEvent() + + val parsed = mediaParsedEvent( + collection = "MyCollection", + fileName = "MyCollection 1", + mediaType = MediaType.Subtitle + ).derivedOf(started) + + val convert = convertEvent( + language = "en", + baseName = "sub1", + outputFiles = listOf("/tmp/sub1.vtt"), + derivedFrom = started + ) + + val migrate = migrateResultEvent( + collection = "MyCollection", + videoUri = "file:///video.mp4", + coverUri = "file:///cover.jpg", + subtitleUris = listOf("file:///sub1.srt", "file://sub1.vtt") + ).derivedOf(convert.last()) + + val collected = CollectedEvent( + setOf( + started.eventId, + parsed.eventId, + *convert.map { it.eventId }.toTypedArray(), + migrate.eventId, + ) + ).derivedOf(migrate) + + val history = listOf( + started, + parsed, + *convert.toTypedArray(), + migrate, + collected + ) + + val result = listener.onEvent(collected, history) + + assertThat(result).isNotNull() + + verify(exactly = 1) { + TaskStore.persist(withArg { task -> + val storeTask = task as MigrateToContentStoreTask + + assertThat(storeTask.data.collection).isEqualTo("MyCollection") + assertThat(storeTask.data.videoContent).isNull() + assertThat(storeTask.data.subtitleContent).hasSize(1) + assertThat(storeTask.data.coverContent).isEmpty() + }) + } + + } + + + // --------------------------------------------------------- + // Helpers for generating events + // --------------------------------------------------------- + + private fun migrateResultEvent( + collection: String, + videoUri: String?, + coverUri: String?, + subtitleUris: List + ) = MigrateContentToStoreTaskResultEvent( + status = TaskStatus.Completed, + collection = collection, + videoMigrate = MigrateContentToStoreTaskResultEvent.FileMigration( + storedUri = videoUri, + status = if (videoUri != null) MigrateStatus.Completed else MigrateStatus.Failed + ), + subtitleMigrate = subtitleUris.map { + MigrateContentToStoreTaskResultEvent.SubtitleMigration( + language = "en", + storedUri = it, + status = MigrateStatus.Completed + ) + }, + coverMigrate = listOfNotNull( + coverUri?.let { + MigrateContentToStoreTaskResultEvent.FileMigration( + storedUri = it, + status = MigrateStatus.Completed + ) + } + ) + ) +} diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StoreContentAndMetadataListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StoreContentAndMetadataListenerTest.kt new file mode 100644 index 00000000..ce254429 --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/events/StoreContentAndMetadataListenerTest.kt @@ -0,0 +1,181 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.events + +import io.mockk.slot +import io.mockk.verify +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.TestBase +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + +class StoreContentAndMetadataListenerTest : TestBase() { + + private val listener = StoreContentAndMetadataListener() + + @Test + @DisplayName( + """ + Hvis event ikke er et MigrateContentToStoreTaskResultEvent + Når onEvent kalles + Så: + Returneres null + """ + ) + fun `ignores non migrate events`() { + val startedEvent = defaultStartEvent() + val event = DummyEvent().derivedOf(startedEvent) + val history = emptyList() + + val result = listener.onEvent(event, history) + + assertThat(result).isNull() + } + + @Test + @DisplayName( + """ + Hvis historikken ikke inneholder CollectedEvent + Når onEvent kalles + Så: + Returneres null + """ + ) + fun `returns null when no collected event exists`() { + val startedEvent = defaultStartEvent() + val event = migrateEvent().derivedOf(startedEvent) + val history = listOf(DummyEvent().derivedOf(startedEvent)) + + val result = listener.onEvent(event, history) + + assertThat(result).isNull() + } + + @Test + @DisplayName( + """ + Hvis collection eller metadata ikke kan projiseres + Når onEvent kalles + Så: + Returneres null + """ + ) + fun `returns null when projection lacks collection or metadata`() { + val startedEvent = defaultStartEvent() + val event = migrateEvent().derivedOf(startedEvent) + val collected = CollectedEvent(setOf(startedEvent.eventId, event.eventId)) + + // Historikken inneholder kun collected-eventet, ingen metadata eller parsed info + val history = listOf(startedEvent, collected) + + val result = listener.onEvent(event, history) + + assertThat(result).isNull() + } + + @Test + @DisplayName( + """ + Hvis historikken inneholder gyldig collection og metadata + Når onEvent kalles + Så: + Opprettes StoreContentAndMetadataTask og det returneres et StoreContentAndMetadataTaskCreatedEvent + """ + ) + fun `creates task and returns created event`() { + val startedEvent = defaultStartEvent() + + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "MyCollection", + parsedFileName = "MyCollection", + parsedSearchTitles = listOf("MyCollection"), + mediaType = MediaType.Serie, + episodeInfo = null + ) + ).derivedOf(startedEvent) + + val migrate = migrateEvent( + status = TaskStatus.Completed, + collection = "Baking Bread", + videoUri = "file:///Baking Bread/Baking Bread - S01E01 - Flour.mp4", + coverUri = "file:///Baking Bread/Baking Bread.jpg", + subtitleUris = listOf("file:///Baking Bread/en/Baking Bread - S01E01 - Flour.srt") + ).derivedOf(parsed) + + val collected = CollectedEvent(setOf(startedEvent.eventId, parsed.eventId)) + .derivedOf(migrate) + + val history = listOf( + startedEvent, + parsed, + collected, + migrate + ) + + val result = listener.onEvent(migrate, history) + assertThat(result).isInstanceOf(StoreContentAndMetadataTaskCreatedEvent::class.java) + + val slot = slot() + + verify(exactly = 1) { + TaskStore.persist(capture(slot)) + } + + val storeTask = slot.captured + assertThat(storeTask.data.collection).isEqualTo("Baking Bread") + assertThat(storeTask.data.metadata.mediaType).isEqualTo(MediaType.Serie) + assertThat(storeTask.data.media?.videoFile).isEqualTo("Baking Bread - S01E01 - Flour.mp4") + assertThat(storeTask.data.media?.subtitles?.first()?.subtitleFile).isEqualTo("Baking Bread - S01E01 - Flour.srt") + assertThat(storeTask.data.media?.subtitles?.first()?.language).isEqualTo("en") + + } + + // --------------------------------------------------------- + // Helpers + // --------------------------------------------------------- + + private fun migrateEvent( + status: TaskStatus = TaskStatus.Completed, + collection: String = "TestCollection", + videoUri: String? = null, + coverUri: String? = null, + subtitleUris: List = emptyList() + ): MigrateContentToStoreTaskResultEvent { + return MigrateContentToStoreTaskResultEvent( + status = status, + collection = collection, + videoMigrate = MigrateContentToStoreTaskResultEvent.FileMigration( + storedUri = videoUri, + status = if (videoUri != null) MigrateStatus.Completed else MigrateStatus.Failed + ), + subtitleMigrate = subtitleUris.map { + MigrateContentToStoreTaskResultEvent.SubtitleMigration( + language = "en", + storedUri = it, + status = MigrateStatus.Completed + ) + }, + coverMigrate = listOfNotNull( + coverUri?.let { + MigrateContentToStoreTaskResultEvent.FileMigration( + storedUri = it, + status = MigrateStatus.Completed + ) + } + ) + ) + } + + class DummyEvent : Event() + + +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MediaStreamReadTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MediaStreamReadTaskListenerTest.kt new file mode 100644 index 00000000..63d80a9f --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/MediaStreamReadTaskListenerTest.kt @@ -0,0 +1,157 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.tasks + +import com.google.gson.JsonObject +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.MockFFprobe +import no.iktdev.mediaprocessing.ffmpeg.FFprobe +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test + +class MediaStreamReadTaskListenerTest { + + class MediaStreamReadTaskListenerTestImplementation(): MediaStreamReadTaskListener() { + + lateinit var probe: FFprobe + override fun getFfprobe(): FFprobe { + return probe + } + } + + private val listener = MediaStreamReadTaskListenerTestImplementation() + + @Test + @DisplayName( + "Når støtter sjekk for MediaReadTask" + + "Hvis task er av typen MediaReadTask" + + "Så:" + + " returnerer true" + ) + fun `supports returns true for MediaReadTask`() { + val mediaTask = mockk() + assertTrue(listener.supports(mediaTask)) + } + + @Test + @DisplayName( + "Når støtter sjekk for ikke-MediaReadTask" + + "Hvis task ikke er av typen MediaReadTask" + + "Så:" + + " returnerer false" + ) + fun `supports returns false for non MediaReadTask`() { + val otherTask = mockk() + assertFalse(listener.supports(otherTask)) + } + + @Test + @DisplayName( + "Når onTask kalles med ikke-MediaReadTask" + + "Hvis task ikke kan castes til MediaReadTask" + + "Så:" + + " returnerer null" + ) + fun `onTask returns null for non MediaReadTask`() = runBlocking { + val otherTask = mockk() + val result = listener.onTask(otherTask) + assertNull(result) + } + + @Test + @DisplayName( + "Når genererer worker id" + + "Hvis worker id blir forespurt" + + "Så:" + + " inneholder id klasse navn og task type" + ) + fun `getWorkerId contains class name and task type`() { + val id = listener.getWorkerId() + assertTrue(id.contains("MediaStreamReadTaskListener")) + assertTrue(id.contains("CPU_INTENSIVE")) + } + + @Test + @DisplayName(""" + Når en MediaReadTask med gyldig filUri prosesseres + Hvis FFprobe returnerer et gyldig JSON-objekt + Så: + Skal MediaStreamReadEvent produseres med data + """) + fun verifyEventProducedOnValidJson() = runTest { + val listener = MediaStreamReadTaskListenerTestImplementation() + val json = JsonObject().apply { addProperty("codec_type", "video") } + listener.probe = MockFFprobe.success(json) + + val task = MediaReadTask(fileUri = "test.mp4").newReferenceId() + val event = listener.onTask(task) + + assertNotNull(event) + assertTrue(event is CoordinatorReadStreamsResultEvent) + val result = event as CoordinatorReadStreamsResultEvent + assertEquals(json, result.data) + assertEquals(TaskStatus.Completed, result.status) + assertEquals("test.mp4", (listener.probe as MockFFprobe).lastInputFile) + } + + @Test + @DisplayName(""" + Når en MediaReadTask med ugyldig filUri prosesseres + Hvis FFprobe feiler med parsing + Så: + Skal onTask returnere null og ikke kaste unntak + """) + fun verifyNullOnParsingError() = runTest { + val listener = MediaStreamReadTaskListenerTestImplementation() + listener.probe = MockFFprobe.failure("Could not parse") + + val task = MediaReadTask(fileUri = "corrupt.mp4").newReferenceId() + val event = listener.onTask(task) + val result = event as CoordinatorReadStreamsResultEvent + + assertNull(result.data) + assertEquals(TaskStatus.Failed, result.status) + assertEquals("corrupt.mp4", (listener.probe as MockFFprobe).lastInputFile) + } + + @Test + @DisplayName(""" + Når en MediaReadTask prosesseres + Hvis FFprobe kaster exception + Så: + Skal onTask returnere null og logge feilen + """) + fun verifyExceptionHandling() = runTest { + val listener = MediaStreamReadTaskListenerTestImplementation() + listener.probe = MockFFprobe.exception() + + val task = MediaReadTask(fileUri = "broken.mp4").newReferenceId() + val event = listener.onTask(task) + assertInstanceOf(CoordinatorReadStreamsResultEvent::class.java, event) + val resultEvent = event as CoordinatorReadStreamsResultEvent + assertNull(event.data) + assertEquals(TaskStatus.Failed, event.status) + } + + @Test + @DisplayName(""" + Når en Task som ikke er MediaReadTask prosesseres + Hvis supports sjekkes + Så: + Skal supports returnere false og onTask returnere null + """) + fun verifySupportsOnlyMediaReadTask() = runTest { + val listener = MediaStreamReadTaskListenerTestImplementation() + listener.probe = MockFFprobe.failure("Not used") + + val otherTask = object : Task() {}.newReferenceId() + assertFalse(listener.supports(otherTask)) + val event = listener.onTask(otherTask) + assertNull(event) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/StoreContentAndMetadataTaskListenerTest.kt b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/StoreContentAndMetadataTaskListenerTest.kt new file mode 100644 index 00000000..efda1b4f --- /dev/null +++ b/apps/coordinator/src/test/kotlin/no/iktdev/mediaprocessing/coordinator/listeners/tasks/StoreContentAndMetadataTaskListenerTest.kt @@ -0,0 +1,165 @@ +package no.iktdev.mediaprocessing.coordinator.listeners.tasks + +import kotlinx.coroutines.test.runTest +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask +import no.iktdev.mediaprocessing.shared.common.model.ContentExport +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.whenever +import org.springframework.http.HttpMethod +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.web.client.RestTemplate + +@ExtendWith(MockitoExtension::class) +class StoreContentAndMetadataTaskListenerTest { + + @Mock + lateinit var restTemplate: RestTemplate + + lateinit var listener: StoreContentAndMetadataTaskListener + + @BeforeEach + fun setup() { + listener = StoreContentAndMetadataTaskListener() + listener.streamitRestTemplate = restTemplate + } + + private fun sampleContentExport(): ContentExport { + return ContentExport( + collection = "series", + episodeInfo = ContentExport.EpisodeInfo(episodeNumber = 1, seasonNumber = 1, episodeTitle = "Pilot"), + media = ContentExport.MediaExport( + videoFile = "bb.s01e01.mkv", + subtitles = listOf(ContentExport.MediaExport.Subtitle(subtitleFile = "bb.en.srt", language = "en")) + ), + metadata = ContentExport.MetadataExport( + title = "Breaking Bad", + genres = listOf("Drama"), + cover = "bb.jpg", + summary = emptyList(), + mediaType = MediaType.Serie, + source = "local" + ) + ) + } + + + @Test + @DisplayName( + """ + Gitt en StoreContentAndMetadataTask + Når supports() kalles + Så: + Returnerer true + """ + ) + fun supports_returnsTrueForCorrectTask() { + val task = StoreContentAndMetadataTask(data = sampleContentExport()) + + val result = listener.supports(task) + + assertThat(result).isTrue() + } + + @Test + @DisplayName( + """ + Gitt en annen type Task + Når supports() kalles + Så: + Returnerer false + """ + ) + fun supports_returnsFalseForWrongTask() { + val task = object : Task() {} + + val result = listener.supports(task) + + assertThat(result).isFalse() + } + + @Test + @DisplayName( + """ + Gitt at RestTemplate returnerer 200 OK + Når onTask() kalles + Så: + Returnerer Completed-event + """ + ) + fun onTask_returnsCompletedOnSuccess() = runTest { + val task = StoreContentAndMetadataTask(data = sampleContentExport()) + + whenever( + restTemplate.exchange( + eq("open/api/mediaprocesser/import"), + eq(HttpMethod.POST), + any(), + eq(Void::class.java) + ) + ).thenReturn(ResponseEntity(HttpStatus.OK)) + + val event = listener.onTask(task) + + assertThat(event).isInstanceOf(StoreContentAndMetadataTaskResultEvent::class.java) + val result = event as StoreContentAndMetadataTaskResultEvent + assertThat(result.taskStatus).isEqualTo(TaskStatus.Completed) + } + + @Test + @DisplayName( + """ + Gitt at RestTemplate kaster exception + Når onTask() kalles + Så: + Returnerer Failed-event + """ + ) + fun onTask_returnsFailedOnException() = runTest { + val task = StoreContentAndMetadataTask(data = sampleContentExport()) + + whenever( + restTemplate.exchange( + any(), + any(), + any(), + eq(Void::class.java) + ) + ).thenThrow(RuntimeException("boom")) + + val event = listener.onTask(task) + + assertThat(event).isInstanceOf(StoreContentAndMetadataTaskResultEvent::class.java) + val result = event as StoreContentAndMetadataTaskResultEvent + assertThat(result.taskStatus).isEqualTo(TaskStatus.Failed) + } + + @Test + @DisplayName( + """ + Gitt en gyldig task + Når getWorkerId() kalles + Så: + Returnerer en streng som inneholder klassenavn, tasktype og UUID + """ + ) + fun workerId_hasCorrectFormat() { + val id = listener.getWorkerId() + + assertThat(id).contains("StoreContentAndMetadataTaskListener-MIXED-") + assertThat(id.split("-").last().length).isGreaterThan(10) // UUID-ish + } +} diff --git a/apps/coordinator/src/test/resources/application.yml b/apps/coordinator/src/test/resources/application.yml new file mode 100644 index 00000000..3df1331c --- /dev/null +++ b/apps/coordinator/src/test/resources/application.yml @@ -0,0 +1,28 @@ +spring: + main: + allow-bean-definition-overriding: true + flyway: + enabled: false + locations: classpath:flyway + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration + + output: + ansi: + enabled: always + +springdoc: + swagger-ui: + path: /open/swagger-ui + +logging: + level: + org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN + org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping: DEBUG + +management: + endpoints: + web: + exposure: + include: mappings diff --git a/apps/processer/build.gradle.kts b/apps/processer/build.gradle.kts index a0697890..63b85529 100644 --- a/apps/processer/build.gradle.kts +++ b/apps/processer/build.gradle.kts @@ -35,7 +35,7 @@ dependencies { implementation("org.json:json:20210307") implementation("no.iktdev:exfl:0.0.16-SNAPSHOT") - implementation("no.iktdev:eventi:1.0-rc13") + implementation("no.iktdev:eventi:1.0-rc15") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") diff --git a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ProcesserEnv.kt b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ProcesserEnv.kt index 3ef9d278..33b741f1 100755 --- a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ProcesserEnv.kt +++ b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ProcesserEnv.kt @@ -11,7 +11,6 @@ class ProcesserEnv { val allowOverwrite = System.getenv("ALLOW_OVERWRITE").toBoolean() ?: false var cachedContent: File = if (!System.getenv("DIRECTORY_CONTENT_CACHE").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_CACHE")) else File("/src/cache") - val outgoingContent: File = if (!System.getenv("DIRECTORY_CONTENT_OUTGOING").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_OUTGOING")) else File("/src/output") val logDirectory = if (!System.getenv("LOG_DIR").isNullOrBlank()) File(System.getenv("LOG_DIR")) else diff --git a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/TaskPoller.kt b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/TaskPoller.kt index f9804080..6a4aa96e 100644 --- a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/TaskPoller.kt +++ b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/TaskPoller.kt @@ -4,7 +4,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import no.iktdev.eventi.models.Event -import no.iktdev.eventi.tasks.AbstractTaskPoller +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.tasks.TaskPollerImplementation import no.iktdev.eventi.tasks.TaskReporter import no.iktdev.mediaprocessing.shared.common.stores.EventStore import no.iktdev.mediaprocessing.shared.common.stores.TaskStore @@ -29,7 +30,7 @@ class PollerAdministrator( @Service class TaskPoller( private val reporter: TaskReporter, -) : AbstractTaskPoller( +) : TaskPollerImplementation( taskStore = TaskStore, reporterFactory = { reporter } // én reporter brukes for alle tasks ) { @@ -48,7 +49,7 @@ class DefaultTaskReporter() : TaskReporter { } override fun markConsumed(taskId: UUID) { - TaskStore.markConsumed(taskId) + TaskStore.markConsumed(taskId, TaskStatus.Completed) } override fun updateProgress(taskId: UUID, progress: Int) { diff --git a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/SubtitleTaskListener.kt b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/SubtitleTaskListener.kt index 0313141a..251ae19b 100644 --- a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/SubtitleTaskListener.kt +++ b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/SubtitleTaskListener.kt @@ -9,14 +9,13 @@ import no.iktdev.mediaprocessing.ffmpeg.FFmpeg import no.iktdev.mediaprocessing.ffmpeg.arguments.MpegArgument import no.iktdev.mediaprocessing.processer.ProcesserEnv import no.iktdev.mediaprocessing.processer.Util -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ExtractResult -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleTask import org.springframework.stereotype.Service import java.util.UUID @Service -class SubtitleTaskListener: TaskListener(TaskType.CPU_INTENSIVE) { +class SubtitleTaskListener: FfmpegTaskListener(TaskType.CPU_INTENSIVE) { override fun getWorkerId() = "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}" override fun supports(task: Task) = task is ExtractSubtitleTask @@ -31,10 +30,8 @@ class SubtitleTaskListener: TaskListener(TaskType.CPU_INTENSIVE) { } if (cachedOutFile.exists() && taskData.data.arguments.firstOrNull() != "-y") { - reporter?.publishEvent(ProcesserExtractEvent( - data = ExtractResult( - status = TaskStatus.Failed - ) + reporter?.publishEvent(ProcesserExtractResultEvent( + status = TaskStatus.Failed ).producedFrom(task)) throw IllegalStateException("${cachedOutFile.absolutePath} does already exist, and arguments does not permit overwrite") } @@ -44,23 +41,27 @@ class SubtitleTaskListener: TaskListener(TaskType.CPU_INTENSIVE) { .outputFile(cachedOutFile.absolutePath) .args(taskData.data.arguments) - val result = SubtitleFFmpeg() + val result = getFfmpeg() withHeartbeatRunner { reporter?.updateLastSeen(task.taskId) } result.run(arguments) if (result.result.resultCode != 0 ) { - return ProcesserExtractEvent(data = ExtractResult(status = TaskStatus.Failed)).producedFrom(task) + return ProcesserExtractResultEvent(status = TaskStatus.Failed).producedFrom(task) } - return ProcesserExtractEvent( - data = ExtractResult( - status = TaskStatus.Completed, + return ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + language = taskData.data.language, cachedOutputFile = cachedOutFile.absolutePath ) ).producedFrom(task) } + override fun getFfmpeg(): FFmpeg { + return SubtitleFFmpeg() + } class SubtitleFFmpeg(override val listener: Listener? = null): FFmpeg(executable = ProcesserEnv.ffmpeg, logDir = ProcesserEnv.subtitleExtractLogDirectory ) { diff --git a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListener.kt b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListener.kt index adb5ad40..b49726ed 100644 --- a/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListener.kt +++ b/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListener.kt @@ -9,8 +9,7 @@ import no.iktdev.mediaprocessing.ffmpeg.arguments.MpegArgument import no.iktdev.mediaprocessing.ffmpeg.decoder.FfmpegDecodedProgress import no.iktdev.mediaprocessing.processer.ProcesserEnv import no.iktdev.mediaprocessing.processer.Util -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.EncodeResult -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeResultEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask import org.springframework.stereotype.Service import java.util.* @@ -29,10 +28,8 @@ class VideoTaskListener: FfmpegTaskListener(TaskType.CPU_INTENSIVE) { } } if (cachedOutFile.exists() && taskData.data.arguments.firstOrNull() != "-y") { - reporter?.publishEvent(ProcesserEncodeEvent( - data = EncodeResult( - status = TaskStatus.Failed - ) + reporter?.publishEvent(ProcesserEncodeResultEvent( + status = TaskStatus.Failed ).producedFrom(task)) throw IllegalStateException("${cachedOutFile.absolutePath} does already exist, and arguments does not permit overwrite") } @@ -49,12 +46,12 @@ class VideoTaskListener: FfmpegTaskListener(TaskType.CPU_INTENSIVE) { } result.run(arguments) if (result.result.resultCode != 0 ) { - return ProcesserEncodeEvent(data = EncodeResult(status = TaskStatus.Failed)).producedFrom(task) + return ProcesserEncodeResultEvent(status = TaskStatus.Failed).producedFrom(task) } - return ProcesserEncodeEvent( - data = EncodeResult( - status = TaskStatus.Completed, + return ProcesserEncodeResultEvent( + status = TaskStatus.Completed, + data = ProcesserEncodeResultEvent.EncodeResult( cachedOutputFile = cachedOutFile.absolutePath ) ).producedFrom(task) diff --git a/apps/processer/src/test/kotlin/no/iktdev/mediaprocessing/processer/listeners/SubtitleTaskListenerTest.kt b/apps/processer/src/test/kotlin/no/iktdev/mediaprocessing/processer/listeners/SubtitleTaskListenerTest.kt new file mode 100644 index 00000000..a4b27ec9 --- /dev/null +++ b/apps/processer/src/test/kotlin/no/iktdev/mediaprocessing/processer/listeners/SubtitleTaskListenerTest.kt @@ -0,0 +1,82 @@ +package no.iktdev.mediaprocessing.processer.listeners + +import kotlinx.coroutines.test.runTest +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.tasks.TaskReporter +import no.iktdev.eventi.tasks.TaskTypeRegistry +import no.iktdev.mediaprocessing.ffmpeg.FFmpeg +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleTask +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.* +import kotlin.system.measureTimeMillis + +class SubtitleTaskListenerTest { + + class TestListener(val delay: Long): SubtitleTaskListener() { + fun getJob() = currentJob + + private var _result: Event? = null + fun getResult(): Event? { + return _result + } + override fun onComplete(task: Task, result: Event?) { + this._result = result + } + + override fun getFfmpeg(): FFmpeg { + return MockFFmpeg(delayMillis = delay, listener = MockFFmpeg.emptyListener()) + } + } + + val overrideReporter = object : TaskReporter { + override fun markClaimed(taskId: UUID, workerId: String) {} + override fun updateLastSeen(taskId: UUID) {} + override fun markConsumed(taskId: UUID) {} + override fun updateProgress(taskId: UUID, progress: Int) {} + override fun log(taskId: UUID, message: String) {} + override fun publishEvent(event: Event) { + + } + } + + @BeforeEach + fun setup() { + TaskTypeRegistry.register(ExtractSubtitleTask::class.java) + } + + @Test + fun `onTask waits for runner to complete`() = runTest { + val delay = 1000L + val testTask = ExtractSubtitleTask( + ExtractSubtitleData( + inputFile = "input.mp4", + outputFileName = "output.srt", + arguments = listOf("-y"), + language = "eng" + ) + ).newReferenceId() + + val listener = TestListener(delay) + + val time = measureTimeMillis { + val accepted = listener.accept(testTask, overrideReporter) + assertTrue(accepted, "Task listener did not accept the task.") + listener.getJob()?.join() + val event = listener.getResult() + assertTrue(event is ProcesserExtractResultEvent) + assertEquals(TaskStatus.Completed, (event as ProcesserExtractResultEvent).status) + } + + assertTrue(time >= delay, "Expected onTask to wait at least $delay ms, waited for $time ms") + assertTrue(time <= (delay*2), "Expected onTask to wait less than ${(delay*2)} ms, waited for $time ms") + + } + +} \ No newline at end of file diff --git a/apps/processer/src/test/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListenerTest.kt b/apps/processer/src/test/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListenerTest.kt index d40738a8..fd402198 100644 --- a/apps/processer/src/test/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListenerTest.kt +++ b/apps/processer/src/test/kotlin/no/iktdev/mediaprocessing/processer/listeners/VideoTaskListenerTest.kt @@ -7,14 +7,14 @@ import no.iktdev.eventi.models.store.TaskStatus import no.iktdev.eventi.tasks.TaskReporter import no.iktdev.eventi.tasks.TaskTypeRegistry import no.iktdev.mediaprocessing.ffmpeg.FFmpeg -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeResultEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeData import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import java.util.UUID +import java.util.* import kotlin.system.measureTimeMillis class VideoTaskListenerTest { @@ -68,8 +68,8 @@ class VideoTaskListenerTest { listener.accept(testTask, overrideReporter) listener.getJob()?.join() val event = listener.getResult() - assertTrue(event is ProcesserEncodeEvent) - assertEquals(TaskStatus.Completed, (event as ProcesserEncodeEvent).data.status) + assertTrue(event is ProcesserEncodeResultEvent) + assertEquals(TaskStatus.Completed, (event as ProcesserEncodeResultEvent).status) } assertTrue(time >= delay, "Expected onTask to wait at least $delay ms, waited for $time ms") diff --git a/apps/pyMetadata/.vscode/settings.json b/apps/pyMetadata/.vscode/settings.json new file mode 100644 index 00000000..0c8df84d --- /dev/null +++ b/apps/pyMetadata/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "python.defaultInterpreterPath": "venv/bin/python", + "python.terminal.activateEnvironment": true +} diff --git a/apps/pyMetadata/.vscode/tasks.json b/apps/pyMetadata/.vscode/tasks.json new file mode 100644 index 00000000..a6d83822 --- /dev/null +++ b/apps/pyMetadata/.vscode/tasks.json @@ -0,0 +1,11 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Install requirements", + "type": "shell", + "command": "${workspaceFolder}/venv/bin/pip install -r requirements.txt", + "group": "build" + } + ] +} \ No newline at end of file diff --git a/apps/pyMetadata/DryRun.py b/apps/pyMetadata/DryRun.py index 8abacd27..11756e23 100644 --- a/apps/pyMetadata/DryRun.py +++ b/apps/pyMetadata/DryRun.py @@ -12,7 +12,7 @@ from fuzzywuzzy import fuzz from algo.AdvancedMatcher import AdvancedMatcher from algo.SimpleMatcher import SimpleMatcher from algo.PrefixMatcher import PrefixMatcher -from clazz.Metadata import Metadata +from models.metadata import Metadata from clazz.shared import EventData, EventMetadata, MediaEvent from app import MetadataEventHandler diff --git a/apps/pyMetadata/README.md b/apps/pyMetadata/README.md new file mode 100644 index 00000000..5ee615bf --- /dev/null +++ b/apps/pyMetadata/README.md @@ -0,0 +1,3 @@ +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt diff --git a/apps/pyMetadata/algo/AdvancedMatcher.py b/apps/pyMetadata/algo/AdvancedMatcher.py index 74cce98a..774ce7b3 100644 --- a/apps/pyMetadata/algo/AdvancedMatcher.py +++ b/apps/pyMetadata/algo/AdvancedMatcher.py @@ -1,41 +1,29 @@ -from fuzzywuzzy import fuzz import re +from typing import List +from fuzzywuzzy import fuzz +from models.metadata import Metadata from .AlgorithmBase import AlgorithmBase, MatchResult -from clazz.Metadata import Metadata class AdvancedMatcher(AlgorithmBase): - def clean_title(self, title: str) -> str: - # Fjerner eventuelle ekstra tekster etter kolon eller andre skilletegn - return re.sub(r'[:\-\—].*', '', title).strip() + def clean(self, s: str) -> str: + # Fjern alt etter kolon eller bindestrek, normaliser til lowercase + return re.sub(r'[:\-\—].*', '', s).strip().lower() - def getBestMatch(self) -> Metadata | None: - best_match = None - best_score = -1 - match_results = [] + def getScore(self) -> int: + best_score = 0 - for title in self.titles: - cleaned_title = self.clean_title(title) # Renset tittel uten ekstra tekst - for metadata in self.metadata: - cleaned_metadata_title = self.clean_title(metadata.title) # Renset metadata-tittel + cleaned_title = self.clean(self.title) + cleaned_metadata_title = self.clean(self.metadata.title) - # Compute different match ratios for both the original and cleaned titles - original_title_ratio = fuzz.token_sort_ratio(title.lower(), metadata.title.lower()) - cleaned_title_ratio = fuzz.token_sort_ratio(cleaned_title.lower(), cleaned_metadata_title.lower()) + # Sammenlign original + best_score = max(best_score, fuzz.token_sort_ratio(self.title.lower(), self.metadata.title.lower())) - alt_title_ratios = [fuzz.token_sort_ratio(cleaned_title.lower(), self.clean_title(alt_title).lower()) for alt_title in metadata.altTitle] - max_alt_title_ratio = max(alt_title_ratios) if alt_title_ratios else 0 + # Sammenlign renset + best_score = max(best_score, fuzz.token_sort_ratio(cleaned_title, cleaned_metadata_title)) - # Combine ratios: take the best of original vs cleaned title, and alt title match - combined_score = max(original_title_ratio, cleaned_title_ratio, max_alt_title_ratio) + # Sammenlign mot altTitler + for alt in self.metadata.altTitle: + alt_score = fuzz.token_sort_ratio(cleaned_title, self.clean(alt)) + best_score = max(best_score, alt_score) - match_results.append(MatchResult(title, metadata.title, combined_score, metadata.source, metadata)) - - # Update best match if this one is better - if combined_score > best_score: - best_score = combined_score - best_match = metadata if combined_score >= 70 else None - - # Print match summary - self.print_match_summary(match_results) - - return best_match \ No newline at end of file + return best_score \ No newline at end of file diff --git a/apps/pyMetadata/algo/AlgorithmBase.py b/apps/pyMetadata/algo/AlgorithmBase.py index a8683fc9..2d428ef3 100644 --- a/apps/pyMetadata/algo/AlgorithmBase.py +++ b/apps/pyMetadata/algo/AlgorithmBase.py @@ -1,13 +1,9 @@ - - from abc import ABC, abstractmethod from dataclasses import dataclass from typing import List - -from fuzzywuzzy import fuzz, process +from fuzzywuzzy import fuzz from tabulate import tabulate - -from clazz.Metadata import Metadata +from models.metadata import Metadata @dataclass class MatchResult: @@ -17,17 +13,19 @@ class MatchResult: source: str data: Metadata - class AlgorithmBase(ABC): - def __init__(self, titles: List[str], metadata: List[Metadata]): - self.titles = titles + def __init__(self, title: str, metadata: Metadata): + self.title = title self.metadata = metadata @abstractmethod - def getBestMatch(self) -> Metadata | None: + def getScore(self) -> int: + """ + Returnerer alle matchresultater med scorer. + """ pass - def print_match_summary(self, match_results: List[MatchResult]): + def print_match_summary(self, match_results: List[MatchResult]) -> None: headers = ["Title", "Matched Title", "Score", "Source"] - data = [(result.title, result.matched_title, result.score, result.source) for result in match_results] - print(tabulate(data, headers=headers)) \ No newline at end of file + data = [(r.title, r.matched_title, r.score, r.source) for r in match_results] + print(tabulate(data, headers=headers)) diff --git a/apps/pyMetadata/algo/PrefixMatcher.py b/apps/pyMetadata/algo/PrefixMatcher.py index 5d6887f8..6aab14f7 100644 --- a/apps/pyMetadata/algo/PrefixMatcher.py +++ b/apps/pyMetadata/algo/PrefixMatcher.py @@ -2,53 +2,27 @@ import re from typing import List, Optional from fuzzywuzzy import fuzz, process from .AlgorithmBase import AlgorithmBase, MatchResult -from clazz.Metadata import Metadata +from models.metadata import Metadata class PrefixMatcher(AlgorithmBase): + def preprocess(self, s: str) -> str: + return re.sub(r'[^a-zA-Z0-9\s]', ' ', s).strip().lower() - def preprocess_text(self, text: str) -> str: - unitext = re.sub(r'[^a-zA-Z0-9\s]', ' ', text) - return unitext.strip().lower() - - def source_priority(self, source: str) -> int: - priority_map = {'mal': 1, 'anii': 2, 'imdb': 3} - return priority_map.get(source, 4) + def first_word(self, s: str) -> str: + return self.preprocess(s).split(" ")[0] if s else "" - def getBestMatch(self) -> Optional[Metadata]: - best_match = None - best_score = -1 - match_results: List[MatchResult] = [] + def getScore(self) -> int: + best_score = 0 + pt = self.first_word(self.title) - for title in self.titles: - preprocessed_title = self.preprocess_text(title)[:1] - - for metadata in self.metadata: - preprocessed_metadata_title = self.preprocess_text(metadata.title)[:1] - - # Match against metadata title - score = fuzz.token_sort_ratio(preprocessed_title, preprocessed_metadata_title) - match_results.append(MatchResult(title, metadata.title, score, metadata.source, metadata)) - if score > best_score: - best_score = score - best_match = metadata if score >= 70 else None + # Mot hovedtittel + meta_main = self.first_word(self.metadata.title) + best_score = max(best_score, fuzz.ratio(pt, meta_main)) - # Match against metadata altTitles - for alt_title in metadata.altTitle: - preprocessed_alt_title = self.preprocess_text(alt_title)[:1] - alt_score = fuzz.token_sort_ratio(preprocessed_title, preprocessed_alt_title) - match_results.append(MatchResult(title, alt_title, alt_score, metadata.source, metadata)) - if alt_score > best_score: - best_score = alt_score - best_match = metadata if alt_score >= 70 else None + # Mot altTitler + for alt in self.metadata.altTitle: + alt_score = fuzz.ratio(pt, self.first_word(alt)) + best_score = max(best_score, alt_score) - match_results.sort(key=lambda x: (-x.score, self.source_priority(x.source))) - - # Print match summary - self.print_match_summary(match_results) - - if match_results: - top_result = match_results[0].data - return top_result - - return best_match + return best_score diff --git a/apps/pyMetadata/algo/SimpleMatcher.py b/apps/pyMetadata/algo/SimpleMatcher.py index 01c07a68..4dc6b0bb 100644 --- a/apps/pyMetadata/algo/SimpleMatcher.py +++ b/apps/pyMetadata/algo/SimpleMatcher.py @@ -1,40 +1,17 @@ import logging +from typing import List from fuzzywuzzy import fuzz, process from .AlgorithmBase import AlgorithmBase, MatchResult -from clazz.Metadata import Metadata +from models.metadata import Metadata class SimpleMatcher(AlgorithmBase): - def getBestMatch(self) -> Metadata | None: - best_match = None - best_score = -1 - match_results = [] + def getScore(self) -> int: + best_score = fuzz.token_sort_ratio(self.title.lower(), self.metadata.title.lower()) - try: - for title in self.titles: - for metadata in self.metadata: - # Match against metadata title - score = fuzz.token_sort_ratio(title.lower(), metadata.title.lower()) - match_results.append(MatchResult(title, metadata.title, score, metadata.source, metadata)) - if score > best_score: - best_score = score - best_match = metadata if score >= 70 else None + for alt in self.metadata.altTitle: + alt_score = fuzz.token_sort_ratio(self.title.lower(), alt.lower()) + best_score = max(best_score, alt_score) - # Match against metadata altTitles - for alt_title in metadata.altTitle: - alt_score = fuzz.token_sort_ratio(title.lower(), alt_title.lower()) - match_results.append(MatchResult(title, alt_title, alt_score, metadata.source, metadata)) - if alt_score > best_score: - best_score = alt_score - best_match = metadata if alt_score >= 70 else None - except Exception as e: - logging.debug("Unntak: {e}") - logging.debug(f"type(title): {type(title)}, value: {title}") - logging.debug(f"type(alt_title): {type(alt_title)}, value: {alt_title}") - logging.debug(f"Metadata objekt:") - logging.debug(metadata.to_dict()) - # Print match summary - self.print_match_summary(match_results) - - return best_match \ No newline at end of file + return best_score diff --git a/apps/pyMetadata/algo/SourceWeighted.py b/apps/pyMetadata/algo/SourceWeighted.py deleted file mode 100644 index 223fe2f0..00000000 --- a/apps/pyMetadata/algo/SourceWeighted.py +++ /dev/null @@ -1,111 +0,0 @@ -from dataclasses import dataclass -from typing import List, Optional -from fuzzywuzzy import fuzz -from unidecode import unidecode -import logging -import re - -from clazz.Metadata import Metadata - -log = logging.getLogger(__name__) - -@dataclass -class WeightedData: - result: Metadata - weight: float - -@dataclass -class DataAndScore: - result: Metadata - score: float - weight: float - matched_title: str - - -class UseSource: - titles: List[str] = [] - dataWeighed: List[WeightedData] = [] - - def __init__(self, titles: List[str], mal: Optional[Metadata] = None, imdb: Optional[Metadata] = None, anii: Optional[Metadata] = None) -> None: - self.titles = titles - if mal is not None: - self.dataWeighed.append(WeightedData(mal, 1.5)) - - if imdb is not None: - self.dataWeighed.append(WeightedData(imdb, 1)) - - if anii is not None: - self.dataWeighed.append(WeightedData(anii, 1.3)) - - - def stripped(self, input_string) -> str: - unitext = unidecode(input_string) - unitext = re.sub(r'[^a-zA-Z0-9\s]', ' ', unitext) - unitext = re.sub(r'\s{2,}', ' ', unitext) - return unitext.strip() - - - def __calculate_score(self, title: str, weightData: List[WeightedData]) -> List[DataAndScore]: - result: List[DataAndScore] = [] - - for title_to_check in self.titles: - for wd in weightData: - if wd.result is None: - continue - - highScore = fuzz.ratio(self.stripped(title_to_check.lower()), self.stripped(wd.result.title.lower())) - for alt_title in wd.result.altTitle: - try: - altScore = fuzz.ratio(self.stripped(title_to_check.lower()), self.stripped(alt_title.lower())) - if altScore > highScore: - highScore = altScore - except Exception as e: - logging.debug("Unntak: {e}") - logging.debug(f"type(title): {type(title)}, value: {title}") - logging.debug(f"type(alt_title): {type(alt_title)}, value: {alt_title}") - logging.debug(f"Metadata objekt:") - logging.debug(weightData) - - givenScore = highScore * wd.weight - result.append(DataAndScore(wd.result, givenScore, wd.weight, title_to_check)) - - result.sort(key=lambda x: x.score, reverse=True) - return result - - def select_result_table(self) -> Optional[pd.DataFrame]: - scoredResults = [] - for title in self.titles: - scoredResult = self.__calculate_score(title=title, weightData=self.dataWeighed) - scoredResults.append(scoredResult) - - all_results = [item for sublist in scoredResults for item in sublist] - - if not all_results: - return None - - # Prepare data for DataFrame - data = { - "Title": [], - "Alt Title": [], - "Score": [], - "Weight": [], - "Matched Title": [] - } - - for ds in all_results: - metadata = ds.result - data["Title"].append(metadata.title) - data["Alt Title"].append(", ".join(metadata.altTitle)) - data["Score"].append(ds.score) - data["Weight"].append(ds.weight) - data["Matched Title"].append(ds.matched_title) - - df = pd.DataFrame(data) - df = df.sort_values(by="Score", ascending=False).reset_index(drop=True) - - try: - df.to_json(f"./logs/{self.titles[0]}.json", orient="records", indent=4) - except Exception as e: - log.error(f"Failed to dump JSON: {e}") - - return df diff --git a/apps/pyMetadata/app.py b/apps/pyMetadata/app.py index 2bd5a979..b12a2e00 100644 --- a/apps/pyMetadata/app.py +++ b/apps/pyMetadata/app.py @@ -1,348 +1,34 @@ -import logging import signal import sys -import os -from typing import List, Optional -import uuid -import threading -import json -import time -from fuzzywuzzy import fuzz -import mysql.connector -import mysql.connector.cursor -from datetime import datetime -import asyncio +from config.database_config import DatabaseConfig +from db.database import Database +from utils.logger import logger +from worker.poller import run_worker +# global flag for shutdown +shutdown_flag = False -from algo.AdvancedMatcher import AdvancedMatcher -from algo.SimpleMatcher import SimpleMatcher -from algo.PrefixMatcher import PrefixMatcher -from clazz.shared import EventMetadata, MediaEvent, event_data_to_json, json_to_media_event -from clazz.Metadata import Metadata +def handle_shutdown(signum, frame): + global shutdown_flag + logger.info("🛑 Shutdown signal mottatt, avslutter worker...") + shutdown_flag = True -from sources.anii import Anii -from sources.imdb import Imdb -from sources.mal import Mal - -from mysql.connector.abstracts import MySQLConnectionAbstract -from mysql.connector.pooling import PooledMySQLConnection -from mysql.connector.types import RowType as MySqlRowType - - -# Konfigurer Database -events_server_address = os.environ.get("DATABASE_ADDRESS") or "192.168.2.250" # "127.0.0.1" -events_server_port = os.environ.get("DATABASE_PORT") or "3306" -events_server_database_name = os.environ.get("DATABASE_NAME_E") or "eventsV3" # "events" -events_server_username = os.environ.get("DATABASE_USERNAME") or "root" -events_server_password = os.environ.get("DATABASE_PASSWORD") or "shFZ27eL2x2NoxyEDBMfDWkvFO" #"root" // default password -log_level = os.environ.get("LOG_LEVEL") or None - -configured_level = logging.INFO - -if (log_level is not None): - _log_level = log_level.lower() - if (_log_level.startswith("d")): - configured_level = logging.DEBUG - elif (_log_level.startswith("e")): - configured_level = logging.ERROR - elif (_log_level.startswith("w")): - configured_level = logging.WARNING - - - - -# Konfigurer logging -logging.basicConfig( - level=configured_level, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.StreamHandler(sys.stdout) - ] -) -logger = logging.getLogger(__name__) - -if (configured_level == logging.DEBUG): - logger.info("Logger configured with DEBUG") -elif (configured_level == logging.ERROR): - logger.info("Logger configured with ERROR") -elif (configured_level == logging.WARNING): - logger.info("Logger configured with WARNING") -else: - logger.info("Logger configured with INFO") - -class EventsPullerThread(threading.Thread): - - connection: PooledMySQLConnection | MySQLConnectionAbstract | None = None - - def __init__(self): - super().__init__() - self.shutdown = threading.Event() - - def getEventsAvailable(self, connection: PooledMySQLConnection | MySQLConnectionAbstract) -> List[MySqlRowType]: - logging.debug("Looking for new available events") - cursor = connection.cursor(dictionary=True) - cursor.execute(""" - SELECT * - FROM events - WHERE referenceId IN ( - SELECT referenceId - FROM events - GROUP BY referenceId - HAVING - SUM(event = 'BaseInfoRead') > 0 - AND SUM(event = 'MetadataSearchPerformed') = 0 - AND SUM(event = 'ProcessCompleted') = 0 - ) - AND event = 'BaseInfoRead' - AND JSON_UNQUOTE(JSON_EXTRACT(data, '$.metadata.status')) = 'Success'; - """) - row = cursor.fetchall() - cursor.close() - return row - - def storeProducedEvent(self, connection: PooledMySQLConnection | MySQLConnectionAbstract, event: MediaEvent) -> bool: - try: - cursor = connection.cursor() - - query = """ - INSERT INTO events (referenceId, eventId, event, data) - VALUES (%s, %s, %s, %s) - """ - cursor.execute(query, ( - event.metadata.referenceId, - event.metadata.eventId, - "MetadataSearchPerformed", - event_data_to_json(event) - )) - connection.commit() - cursor.close() - return True - except mysql.connector.Error as err: - logger.error("Error inserting into database: %s", err) - return False - - def __connect_to_datasource(self) -> bool: - try: - myConnection = mysql.connector.connect( - host=events_server_address, - port=events_server_port, - database=events_server_database_name, - user=events_server_username, - password=events_server_password - ) - if myConnection.is_connected(): - logging.debug(f"Successfully connected to {events_server_database_name} at {events_server_address}:{events_server_port}") - self.connection = myConnection - return True - else: - self.connection = None - except Exception as e: - logging.error(f"Error while connecting to database: {e}") - logging.exception(e) - self.connection = None - return False - - def __has_connection_to_database(self) -> bool: - if (self.connection == None or self.connection.is_connected() == False): - return False - else: - try: - self.connection.ping(reconnect=True, attempts=5, delay=5) - except Exception as e: - logging.warning("Incorrect state for connection! Ping yielded no connection!") - logging.exception(e) - return False - return True - - def run(self) -> None: - logger.info(f"Using {events_server_address}:{events_server_port} on table: {events_server_database_name} with user: {events_server_username}") - while not self.shutdown.is_set(): - producedMessage: bool = False - - while not self.shutdown.is_set() and self.__has_connection_to_database() != True: - logging.debug("Attempting to reconnect to the database...") - if (self.__connect_to_datasource() == False): - logger.info("Failed to connect to database, waiting 5 seconds before retrying") - time.sleep(5) # Wait 5 seconds before retrying - else: - logging.debug("A successful connection has been made!") - - try: - rows = self.getEventsAvailable(connection=self.connection) - if (len(rows) == 0): - logger.debug("No events found..") - for row in rows: - event: MediaEvent | None = None - if (row is not None): - try: - referenceId = row["referenceId"] - incomingEventType = row["event"] - logMessage = f""" -============================================================================ -Found message -{referenceId} -{incomingEventType} -============================================================================\n""" - logger.info(logMessage) - - - event = json_to_media_event(row["data"]) - producedEvent = asyncio.run(MetadataEventHandler(event).run()) - - producedMessage = f""" -============================================================================ -Producing message -{referenceId} -{incomingEventType} - -{event_data_to_json(producedEvent)} -============================================================================\n""" - logger.info(producedMessage) - - producedEvent = self.storeProducedEvent(connection=self.connection, event=producedEvent) - - except Exception as e: - """Produce failure here""" - logger.exception(e) - try: - producedEvent = MediaEvent( - metadata = EventMetadata( - referenceId=event.metadata.referenceId, - eventId=str(uuid.uuid4()), - derivedFromEventId=event.metadata.eventId, - status= "Failed", - created= datetime.now().isoformat(), - source="metadataApp" - ), - data=None, - eventType="MetadataSearchPerformed" - ) - self.storeProducedEvent(connection=self.connection, event=producedEvent) - except Exception as iex: - logger.error("Failed to push error to database..") - self.connection.close() - except mysql.connector.Error as err: - logger.error("Database error: %s", err) - - # Introduce a small sleep to reduce CPU usage - time.sleep(5) - if (self.shutdown.is_set()): - logger.info("Shutdown is set..") - logging.debug("End of puller function..") - - - def stop(self): - self.shutdown.set() - global should_stop - should_stop = True - -class MetadataEventHandler: - mediaEvent: MediaEvent | None = None - - def __init__(self, data: MediaEvent): - super().__init__() - self.mediaEvent = data - logger.info(self.mediaEvent) - - async def run(self) -> MediaEvent | None: - logger.info("Starting search") - if self.mediaEvent is None: - logger.error("Event does not contain anything...") - return None - - event: MediaEvent = self.mediaEvent - - unique_titles = set(event.data.searchTitles) - unique_titles.update([ - event.data.title, - event.data.sanitizedName - ]) - searchableTitles = list(unique_titles) - - joinedTitles = "\n".join(searchableTitles) - logger.info("Searching for:\n%s", joinedTitles) - - # Kjør den asynkrone søkemetoden - result: Metadata | None = await self.__getMetadata(searchableTitles) - - result_message: str | None = None - if result is None: - result_message = f"No result for {joinedTitles}" - logger.info(result_message) - - producedEvent = MediaEvent( - metadata=EventMetadata( - referenceId=event.metadata.referenceId, - eventId=str(uuid.uuid4()), - derivedFromEventId=event.metadata.eventId, - status="Failed" if result is None else "Success", - created=datetime.now().isoformat(), - source="metadataApp" - ), - data=result, - eventType="MetadataSearchPerformed" - ) - return producedEvent - - async def __getMetadata(self, titles: List[str]) -> Metadata | None: - mal = Mal(titles=titles) - anii = Anii(titles=titles) - imdb = Imdb(titles=titles) - - results: List[Metadata | None] = await asyncio.gather( - mal.search(), - anii.search(), - imdb.search() - ) - - filtered_results = [result for result in results if result is not None] - logger.info("\nSimple matcher") - simpleSelector = SimpleMatcher(titles=titles, metadata=filtered_results).getBestMatch() - logger.info("\nAdvanced matcher") - advancedSelector = AdvancedMatcher(titles=titles, metadata=filtered_results).getBestMatch() - logger.info("\nPrefix matcher") - prefixSelector = PrefixMatcher(titles=titles, metadata=filtered_results).getBestMatch() - - if advancedSelector is not None: - return advancedSelector - if simpleSelector is not None: - return simpleSelector - if prefixSelector is not None: - return prefixSelector - return None - -# Global variabel for å indikere om applikasjonen skal avsluttes -should_stop = False - -# Signalhåndteringsfunksjon -def signal_handler(sig, frame): - global should_stop - should_stop = True - -# Hovedprogrammet def main(): + # registrer signal handlers for graceful shutdown + signal.signal(signal.SIGINT, handle_shutdown) + signal.signal(signal.SIGTERM, handle_shutdown) + + logger.info("🚀 Starter worker-applikasjon") try: - # Angi signalhåndterer for å fange opp SIGINT (Ctrl+C) - signal.signal(signal.SIGINT, signal_handler) - - # Opprett og start consumer-tråden - consumer_thread = EventsPullerThread() - consumer_thread.start() - - logger.info("App started") - - # Vent til should_stop er satt til True for å avslutte applikasjonen - while not should_stop: - time.sleep(60) - - # Stopp consumer-tråden - consumer_thread.stop() - consumer_thread.join() - except: - logger.info("App crashed") + config: DatabaseConfig = DatabaseConfig.from_env() + db: Database = Database(config) + db.connect() + run_worker(db=db, shutdown_flag_ref=lambda: shutdown_flag) + except Exception as e: + logger.error(f"❌ Kritisk feil i app: {e}") sys.exit(1) - logger.info("App stopped") - sys.exit(0) -if __name__ == '__main__': + logger.info("👋 Worker avsluttet gracefully") + +if __name__ == "__main__": main() diff --git a/apps/pyMetadata/clazz/Metadata.py b/apps/pyMetadata/clazz/Metadata.py deleted file mode 100644 index ab0bdab2..00000000 --- a/apps/pyMetadata/clazz/Metadata.py +++ /dev/null @@ -1,33 +0,0 @@ -from dataclasses import asdict, dataclass -from typing import List, Optional - -@dataclass -class Summary: - summary: str - language: str - - def to_dict(self): - return asdict(self) - - -@dataclass -class Metadata: - title: str - altTitle: List[str] - cover: str - banner: Optional[str] - type: str # Serie/Movie - summary: List[Summary] - genres: List[str] - source: str - - def to_dict(self): - # Trimmer alle strenger før de konverteres til dict - def trim(item): - if isinstance(item, str): - return item.strip() - elif isinstance(item, list): - return [trim(sub_item) for sub_item in item] - return item - - return {key: trim(value) for key, value in asdict(self).items()} \ No newline at end of file diff --git a/apps/pyMetadata/clazz/shared.py b/apps/pyMetadata/clazz/shared.py deleted file mode 100644 index 2e27a9c7..00000000 --- a/apps/pyMetadata/clazz/shared.py +++ /dev/null @@ -1,73 +0,0 @@ -import json -from dataclasses import dataclass, asdict -from typing import Any, Dict, List, Optional -from datetime import datetime - -# Definer dataclassene for strukturen -@dataclass -class EventMetadata: - derivedFromEventId: str - eventId: str - referenceId: str - status: str - created: datetime - source: str - - def to_dict(self): - return asdict(self) - - -@dataclass -class EventData: - title: str - sanitizedName: str - searchTitles: List[str] - - def to_dict(self): - return asdict(self) - -@dataclass -class MediaEvent: - metadata: EventMetadata - eventType: str - data: Any| EventData - - def to_dict(self): - return asdict(self) - -# Funksjon for å parse datetime fra streng -def parse_datetime(datetime_str: str) -> datetime: - return datetime.fromisoformat(datetime_str) - -def event_data_to_json(event_data: EventData) -> str: - return json.dumps(event_data.to_dict()) - -# Funksjon for å konvertere JSON til klasser -def json_to_media_event(json_data: str) -> MediaEvent: - data_dict = json.loads(json_data) - - metadata_dict: Dict[str, str] = data_dict['metadata'] - event_data_dict = data_dict['data'] - - metadata = EventMetadata( - derivedFromEventId=metadata_dict['derivedFromEventId'], - eventId=metadata_dict['eventId'], - referenceId=metadata_dict['referenceId'], - status=metadata_dict['status'], - created=parse_datetime(metadata_dict['created']), - source= metadata_dict.get('source', None) - ) - - event_data = EventData( - title=event_data_dict['title'], - sanitizedName=event_data_dict['sanitizedName'], - searchTitles=event_data_dict['searchTitles'] - ) - - media_event = MediaEvent( - metadata=metadata, - eventType=data_dict['eventType'], - data=event_data - ) - - return media_event \ No newline at end of file diff --git a/apps/pyMetadata/requirements.txt b/apps/pyMetadata/requirements.txt index c4037f8d..2cf8ef7c 100644 --- a/apps/pyMetadata/requirements.txt +++ b/apps/pyMetadata/requirements.txt @@ -6,4 +6,5 @@ python-Levenshtein>=0.21.1 mal-api>=0.5.3 Unidecode>=1.3.8 tabulate>=0.9.0 -mysql-connector-python>=9.0.0 \ No newline at end of file +mysql-connector-python>=9.0.0 +pydantic>=2.12.5 \ No newline at end of file diff --git a/apps/pyMetadata/sources/anii.py b/apps/pyMetadata/sources/anii.py index 5b191946..81a0f59f 100644 --- a/apps/pyMetadata/sources/anii.py +++ b/apps/pyMetadata/sources/anii.py @@ -2,7 +2,8 @@ import logging, sys import hashlib from typing import List, Dict, Optional -from clazz.Metadata import Metadata, Summary +from models.enums import MediaType +from models.metadata import Metadata, Summary from .source import SourceBase from AnilistPython import Anilist @@ -90,5 +91,5 @@ class Anii(SourceBase): return hashlib.md5(text.encode()).hexdigest() return None - def getMediaType(self, type: str) -> str: - return 'movie' if type.lower() == 'movie' else 'serie' \ No newline at end of file + def getMediaType(self, type: str) -> MediaType: + return MediaType.MOVIE if type.lower() == 'movie' else MediaType.SERIE \ No newline at end of file diff --git a/apps/pyMetadata/sources/imdb.py b/apps/pyMetadata/sources/imdb.py index 1ba9e7d1..a35575e1 100644 --- a/apps/pyMetadata/sources/imdb.py +++ b/apps/pyMetadata/sources/imdb.py @@ -4,7 +4,8 @@ from imdb.Movie import Movie from typing import List, Dict, Optional -from clazz.Metadata import Metadata, Summary +from models.enums import MediaType +from models.metadata import Metadata, Summary from .source import SourceBase import asyncio @@ -74,5 +75,5 @@ class Imdb(SourceBase): log.exception(e) return None - def getMediaType(self, type: str) -> str: - return 'movie' if type.lower() == 'movie' else 'serie' \ No newline at end of file + def getMediaType(self, type: str) -> MediaType: + return MediaType.MOVIE if type.lower() == 'movie' else MediaType.SERIE \ No newline at end of file diff --git a/apps/pyMetadata/sources/mal.py b/apps/pyMetadata/sources/mal.py index 931529bb..5fa6d358 100644 --- a/apps/pyMetadata/sources/mal.py +++ b/apps/pyMetadata/sources/mal.py @@ -1,7 +1,8 @@ import logging, sys from typing import Dict, List, Optional -from clazz.Metadata import Metadata, Summary +from models.enums import MediaType +from models.metadata import Metadata, Summary from .source import SourceBase from mal import Anime, AnimeSearch, AnimeSearchResult @@ -69,5 +70,5 @@ class Mal(SourceBase): log.exception(e) return None - def getMediaType(self, type: str) -> str: - return 'movie' if type.lower() == 'movie' else 'serie' \ No newline at end of file + def getMediaType(self, type: str) -> MediaType: + return MediaType.MOVIE if type.lower() == 'movie' else MediaType.SERIE \ No newline at end of file diff --git a/apps/pyMetadata/sources/source.py b/apps/pyMetadata/sources/source.py index c14a2c97..653db480 100644 --- a/apps/pyMetadata/sources/source.py +++ b/apps/pyMetadata/sources/source.py @@ -5,7 +5,7 @@ from typing import List, Tuple from fuzzywuzzy import fuzz -from clazz.Metadata import Metadata +from models.metadata import Metadata import asyncio diff --git a/apps/pyMetadata/tests/test_result.py b/apps/pyMetadata/tests/test_result.py deleted file mode 100644 index 39e263fb..00000000 --- a/apps/pyMetadata/tests/test_result.py +++ /dev/null @@ -1,39 +0,0 @@ -import unittest -import json -from sources.result import Metadata, DataResult - -class SerializationTest(unittest.TestCase): - def test_metadata_to_json(self): - metadata = Metadata( - title='Sample Title', - altTitle='Alternate Title', - cover='path/to/cover.jpg', - type='Movie', - summary='Lorem ipsum dolor sit amet', - genres=['Action', 'Drama', 'Thriller'] - ) - - metadata_json = json.dumps(metadata.to_dict()) - self.assertIsInstance(metadata_json, str) - - def test_data_result_to_json(self): - metadata = Metadata( - title='Sample Title', - altTitle='Alternate Title', - cover='path/to/cover.jpg', - type='Movie', - summary='Lorem ipsum dolor sit amet', - genres=['Action', 'Drama', 'Thriller'] - ) - - data_result = DataResult( - statusType='SUCCESS', - errorMessage=None, - data=metadata - ) - - data_result_json = json.dumps(data_result.to_dict()) - self.assertIsInstance(data_result_json, str) - -if __name__ == '__main__': - unittest.main() diff --git a/apps/pyWatcher/.vscode/settings.json b/apps/pyWatcher/.vscode/settings.json new file mode 100644 index 00000000..4c272f62 --- /dev/null +++ b/apps/pyWatcher/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "python.defaultInterpreterPath": "venv/bin/python", + "python.terminal.activateEnvironment": true +} \ No newline at end of file diff --git a/apps/pyWatcher/.vscode/tasks.json b/apps/pyWatcher/.vscode/tasks.json new file mode 100644 index 00000000..a6d83822 --- /dev/null +++ b/apps/pyWatcher/.vscode/tasks.json @@ -0,0 +1,11 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Install requirements", + "type": "shell", + "command": "${workspaceFolder}/venv/bin/pip install -r requirements.txt", + "group": "build" + } + ] +} \ No newline at end of file diff --git a/apps/pyWatcher/app.py b/apps/pyWatcher/app.py new file mode 100644 index 00000000..e23a92d7 --- /dev/null +++ b/apps/pyWatcher/app.py @@ -0,0 +1,65 @@ +import asyncio +import signal +import sys +import uvicorn +from api.health_api import create_health_app +from config.database_config import DatabaseConfig +from db.database import Database +from db.repository import insert_event +from worker.file_watcher import start_observer +from utils.logger import logger + +# global flag for shutdown +shutdown_flag = False +observers = [] + +def handle_shutdown(signum, frame): + global shutdown_flag + logger.info("🛑 Shutdown signal mottatt, avslutter worker...") + shutdown_flag = True + +async def run_worker(db: Database, paths, extensions, shutdown_flag_ref): + global observers + observers = observers = [start_observer(db, [p], extensions, insert_event) for p in paths] + try: + while not shutdown_flag_ref(): + await asyncio.sleep(5) + finally: + logger.info("🛑 Stopper observer...") + for obs in observers: + obs.stop() + obs.join() + logger.info("👋 Alle observers stoppet") + + return observers + +def main(): + # registrer signal handlers for graceful shutdown + signal.signal(signal.SIGINT, handle_shutdown) + signal.signal(signal.SIGTERM, handle_shutdown) + + logger.info("🚀 Starter worker-applikasjon") + try: + config: DatabaseConfig = DatabaseConfig.from_env() + db: Database = Database(config) + db.connect() + + # paths og extensions fra PathsConfig + from config.paths_config import PathsConfig + paths_config = PathsConfig.from_env() + paths_config.validate() + + loop = asyncio.get_event_loop() + loop.create_task(run_worker(db, paths_config.watch_paths, paths_config.extensions, lambda: shutdown_flag)) + + # bruk health_api + app = create_health_app(lambda: observers) + uvicorn.run(app, host="0.0.0.0", port=8000) + except Exception as e: + logger.error(f"❌ Kritisk feil i app: {e}") + sys.exit(1) + + logger.info("👋 Worker avsluttet gracefully") + +if __name__ == "__main__": + main() diff --git a/apps/pyMetadata/clazz/__init__.py b/apps/pyWatcher/config/__init__.py similarity index 100% rename from apps/pyMetadata/clazz/__init__.py rename to apps/pyWatcher/config/__init__.py diff --git a/apps/pyWatcher/config/database_config.py b/apps/pyWatcher/config/database_config.py new file mode 100644 index 00000000..9bb43be4 --- /dev/null +++ b/apps/pyWatcher/config/database_config.py @@ -0,0 +1,29 @@ +import os +from dataclasses import dataclass + +@dataclass +class DatabaseConfig: + address: str + port: int + name: str + username: str + password: str + + @staticmethod + def from_env() -> "DatabaseConfig": + return DatabaseConfig( + address=os.environ.get("DATABASE_ADDRESS") or "192.168.2.250", + port=int(os.environ.get("DATABASE_PORT") or "3306"), + name=os.environ.get("DATABASE_NAME_E") or "eventsV3", + username=os.environ.get("DATABASE_USERNAME") or "root", + password=os.environ.get("DATABASE_PASSWORD") or "def", + ) + + def validate(self) -> None: + if not self.address: + raise ValueError("Database address mangler") + if not self.name: + raise ValueError("Database name mangler") + if not self.username: + raise ValueError("Database username mangler") + # du kan legge til flere regler her diff --git a/apps/pyWatcher/config/paths_config.py b/apps/pyWatcher/config/paths_config.py new file mode 100644 index 00000000..527129ef --- /dev/null +++ b/apps/pyWatcher/config/paths_config.py @@ -0,0 +1,31 @@ +import os +from dataclasses import dataclass +from typing import List + +@dataclass +class PathsConfig: + watch_paths: List[str] + extensions: List[str] + + @staticmethod + def from_env() -> "PathsConfig": + # Paths kan legges inn som kommaseparert liste i miljøvariabel + raw_paths = os.environ.get("WATCH_PATHS") + paths = [p.strip() for p in raw_paths.split(",") if p.strip()] + + # Extensions kan legges inn som kommaseparert liste + raw_ext = os.environ.get("WATCH_EXTENSIONS") or ".mkv,.mp4,.avi" + extensions = [e.strip() for e in raw_ext.split(",") if e.strip()] + + return PathsConfig(watch_paths=paths, extensions=extensions) + + def validate(self) -> None: + if not self.watch_paths: + raise ValueError("Ingen paths definert for overvåkning") + for path in self.watch_paths: + if not os.path.exists(path): + raise ValueError(f"Path finnes ikke: {path}") + if not os.path.isdir(path): + raise ValueError(f"Path er ikke en katalog: {path}") + if not self.extensions: + raise ValueError("Ingen filendelser definert for filtrering") diff --git a/apps/pyWatcher/db/__init__.py b/apps/pyWatcher/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/pyWatcher/db/database.py b/apps/pyWatcher/db/database.py new file mode 100644 index 00000000..89340260 --- /dev/null +++ b/apps/pyWatcher/db/database.py @@ -0,0 +1,53 @@ +from config.database_config import DatabaseConfig +from utils.logger import logger +import mysql.connector +from mysql.connector import Error +from utils.backoff import wait_with_backoff + +class Database: + def __init__(self, config: DatabaseConfig): + self.config = config + self.conn = None + + def connect(self): + """Koble til DB med backoff.""" + self.config.validate() + while True: + try: + self.conn = mysql.connector.connect( + host=self.config.address, + user=self.config.username, + password=self.config.password, + database=self.config.name + ) + if self.conn.is_connected(): + logger.info("✅ Tilkoblet til databasen") + return + except Error as e: + logger.error(f"❌ DB-tilkobling feilet: {e}") + for _ in wait_with_backoff(): + try: + self.conn = mysql.connector.connect( + host=self.config.address, + user=self.config.username, + password=self.config.password, + database=self.config.name + ) + if self.conn.is_connected(): + logger.info("✅ Tilkoblet til databasen") + return + except Error: + continue + + def validate(self): + """Sjekk at tilkoblingen er aktiv.""" + if not self.conn or not self.conn.is_connected(): + logger.warning("⚠️ Tilkobling mistet, prøver igjen...") + self.connect() + + def query(self, sql: str, params=None): + """Kjør en spørring med validering.""" + self.validate() + cursor = self.conn.cursor(dictionary=True) + cursor.execute(sql, params or ()) + return cursor.fetchall() diff --git a/apps/pyWatcher/db/repository.py b/apps/pyWatcher/db/repository.py new file mode 100644 index 00000000..29d261ce --- /dev/null +++ b/apps/pyWatcher/db/repository.py @@ -0,0 +1,55 @@ +from datetime import datetime +import json +from typing import List, Optional +from db.database import Database +from models.event import FileAddedEvent +from utils.logger import logger +from models.event import Event, FileAddedEvent + +def insert_event(db: Database, event: Event) -> None: + """Persistér et Event til Events-tabellen.""" + db.validate() + sql = """ + INSERT INTO Events(reference_id, event_id, event, data, persisted_at) + VALUES (%s, %s, %s, %s, NOW()) + """ + with db.conn.cursor() as cursor: + cursor.execute( + sql, + (event.referenceId, event.eventId, event.__class__.__name__, event.model_dump_json()) + ) + db.conn.commit() + logger.info(f"📦 Event persisted: {event.__class__.__name__} ({event.referenceId})") + +def get_open_added_events(db: Database) -> List[FileAddedEvent]: + """ + Hent alle FileAddedEvent som fortsatt er 'åpne', + dvs. ikke har en FileReadyEvent eller FileRemovedEvent. + Returnerer en liste med FileAddedEvent-objekter. + """ + db.validate() + sql = """ + SELECT e.reference_id, e.event_id, e.event, e.data + FROM Events e + WHERE e.event = 'FileAddedEvent' + AND NOT EXISTS ( + SELECT 1 FROM Events r + WHERE r.reference_id = e.reference_id + AND r.event IN ('FileReadyEvent', 'FileRemovedEvent') + ) + ORDER BY e.persisted_at ASC + """ + events: List[FileAddedEvent] = [] + with db.conn.cursor(dictionary=True) as cursor: + cursor.execute(sql) + rows = cursor.fetchall() + for row in rows: + # Bruk Pydantic v2 sin model_validate_json + event = FileAddedEvent.model_validate_json(row["data"]) + # Overstyr referenceId og eventId fra kolonnene (sannhetskilde) + event.referenceId = row["reference_id"] + event.eventId = row["event_id"] + events.append(event) + + logger.info(f"🔎 Fant {len(events)} åpne FileAddedEvent uten Ready/Removed") + return events \ No newline at end of file diff --git a/apps/pyWatcher/models/event.py b/apps/pyWatcher/models/event.py new file mode 100644 index 00000000..a3c3ab5b --- /dev/null +++ b/apps/pyWatcher/models/event.py @@ -0,0 +1,39 @@ +import uuid +from datetime import datetime +from typing import Optional, Set +from pydantic import BaseModel + +# --- Metadata --- +class Metadata(BaseModel): + created: str + derivedFromId: Optional[Set[str]] = None + +# --- FileInfo --- +class FileInfo(BaseModel): + fileName: str + fileUri: str + +# --- Base Event --- +class Event(BaseModel): + referenceId: str + eventId: str + metadata: Metadata + +# --- Spesifikke events --- +class FileAddedEvent(Event): + data: FileInfo + +class FileReadyEvent(Event): + data: FileInfo + +class FileRemovedEvent(Event): + data: FileInfo + +# --- Helper-funksjoner --- +def create_event(event_cls, file_name: str, file_uri: str, reference_id: Optional[str] = None) -> Event: + return event_cls( + referenceId=reference_id or str(uuid.uuid4()), + eventId=str(uuid.uuid4()), + metadata=Metadata(created=datetime.now().isoformat()), + data=FileInfo(fileName=file_name, fileUri=file_uri) + ) diff --git a/apps/pyWatcher/requirements.txt b/apps/pyWatcher/requirements.txt new file mode 100644 index 00000000..24bd6270 --- /dev/null +++ b/apps/pyWatcher/requirements.txt @@ -0,0 +1,13 @@ +cinemagoer>=2023.5.1 +AnilistPython>=0.1.3 +fuzzywuzzy>=0.18.0 +requests>=2.31.0 +python-Levenshtein>=0.21.1 +mal-api>=0.5.3 +Unidecode>=1.3.8 +tabulate>=0.9.0 +mysql-connector-python>=9.0.0 +pydantic>=2.12.5 +watchdog>=6.0.0 +fastapi==0.124.4 +uvicorn==0.38.0 \ No newline at end of file diff --git a/apps/pyWatcher/requirments-test.txt b/apps/pyWatcher/requirments-test.txt new file mode 100644 index 00000000..bf6276f4 --- /dev/null +++ b/apps/pyWatcher/requirments-test.txt @@ -0,0 +1,2 @@ +pytest==9.0.2 +pytest-asyncio==1.3.0 \ No newline at end of file diff --git a/apps/pyWatcher/tests/test_file_handler.py b/apps/pyWatcher/tests/test_file_handler.py new file mode 100644 index 00000000..ea02f198 --- /dev/null +++ b/apps/pyWatcher/tests/test_file_handler.py @@ -0,0 +1,23 @@ +import os +import pytest +from utils.file_handler import FileHandler +from models.event import FileAddedEvent, FileRemovedEvent + +def test_handle_created_returns_event_for_valid_extension(tmp_path): + handler = FileHandler(extensions={".csv"}) + file_path = tmp_path / "test.csv" + file_path.write_text("dummy") + + ev = handler.handle_created(str(file_path)) + assert isinstance(ev, FileAddedEvent) + assert ev.data.fileName == "test.csv" + assert ev.data.fileUri == str(file_path) + +def test_handle_deleted_returns_event_for_valid_extension(tmp_path): + handler = FileHandler(extensions={".csv"}) + file_path = tmp_path / "test.csv" + file_path.write_text("dummy") + + ev = handler.handle_deleted(str(file_path)) + assert isinstance(ev, FileRemovedEvent) + assert ev.data.fileName == "test.csv" diff --git a/apps/pyWatcher/tests/test_readiness.py b/apps/pyWatcher/tests/test_readiness.py new file mode 100644 index 00000000..a86b1a8a --- /dev/null +++ b/apps/pyWatcher/tests/test_readiness.py @@ -0,0 +1,23 @@ +import asyncio +import pytest +from utils.readiness import file_is_ready, check_ready +from models.event import FileReadyEvent + +@pytest.mark.asyncio +async def test_check_ready_creates_event(tmp_path): + file_path = tmp_path / "test.csv" + file_path.write_text("dummy") + + events = [] + def fake_insert(db, ev): + events.append(ev) + + ev = await check_ready(db=None, + ref_id="ref123", + file_name="test.csv", + file_uri=str(file_path), + insert_event=fake_insert) + + assert isinstance(ev, FileReadyEvent) + assert ev.referenceId == "ref123" + assert events[0] == ev diff --git a/apps/pyWatcher/tests/test_repository.py b/apps/pyWatcher/tests/test_repository.py new file mode 100644 index 00000000..890d83d6 --- /dev/null +++ b/apps/pyWatcher/tests/test_repository.py @@ -0,0 +1,32 @@ +import json +from models.event import create_event, FileAddedEvent +from db.repository import get_open_added_events + +class FakeCursor: + def __init__(self, rows): + self.rows = rows + def execute(self, sql, params=None): pass + def fetchall(self): return self.rows + def __enter__(self): return self + def __exit__(self, *a): pass + +class FakeConn: + def cursor(self, dictionary=True): return self.cursor_obj + def __init__(self, rows): self.cursor_obj = FakeCursor(rows) + +class FakeDB: + def __init__(self, rows): self.conn = FakeConn(rows) + def validate(self): pass + +def test_get_open_added_events_returns_typed_objects(): + ev = create_event(FileAddedEvent, "test.csv", "/tmp/test.csv", reference_id="ref123") + row = { + "reference_id": ev.referenceId, + "event_id": ev.eventId, + "event": "FileAddedEvent", + "data": ev.model_dump_json() + } + db = FakeDB([row]) + events = get_open_added_events(db) + assert isinstance(events[0], FileAddedEvent) + assert events[0].data.fileName == "test.csv" diff --git a/apps/pyWatcher/tests/test_shutdown.py b/apps/pyWatcher/tests/test_shutdown.py new file mode 100644 index 00000000..2cc065e2 --- /dev/null +++ b/apps/pyWatcher/tests/test_shutdown.py @@ -0,0 +1,20 @@ +import asyncio +import pytest +from app import run_worker + +class FakeObserver: + def __init__(self): self.stopped = False + def stop(self): self.stopped = True + def join(self, timeout=None): pass + +@pytest.mark.asyncio +async def test_run_worker_stops_on_shutdown(monkeypatch): + fake = FakeObserver() + + def shutdown_ref(): return True + + # monkeypatch start_observer as imported in app.py + monkeypatch.setattr("app.start_observer", lambda *a, **kw: fake) + + await run_worker(db=object(), paths=["/tmp"], extensions={".csv"}, shutdown_flag_ref=shutdown_ref) + assert fake.stopped # nå blir True diff --git a/apps/pyWatcher/utils/backoff.py b/apps/pyWatcher/utils/backoff.py new file mode 100644 index 00000000..61ec8d44 --- /dev/null +++ b/apps/pyWatcher/utils/backoff.py @@ -0,0 +1,11 @@ +from utils.logger import logger +import time + +def retry_delays(): + return [5, 15, 30, 60] + +def wait_with_backoff(): + for delay in retry_delays(): + logger.info(f"⏳ Venter {delay} sekunder...") + time.sleep(delay) + yield diff --git a/apps/pyWatcher/utils/file_handler.py b/apps/pyWatcher/utils/file_handler.py new file mode 100644 index 00000000..49fa498a --- /dev/null +++ b/apps/pyWatcher/utils/file_handler.py @@ -0,0 +1,26 @@ +import os +from models.event import Event, create_event, FileAddedEvent, FileRemovedEvent + +class FileHandler: + def __init__(self, extensions): + self.extensions = extensions + self.file_refs = {} + + def get_ref_id(self, path: str) -> str: + if path in self.file_refs: + return self.file_refs[path] + ref_id = os.path.basename(path) + "-" + os.urandom(4).hex() + self.file_refs[path] = ref_id + return ref_id + + def handle_created(self, path: str) -> FileAddedEvent: + if os.path.splitext(path)[1] not in self.extensions: + return None + ref_id = self.get_ref_id(path) + return create_event(FileAddedEvent, os.path.basename(path), path, reference_id=ref_id) + + def handle_deleted(self, path: str) -> FileRemovedEvent: + if os.path.splitext(path)[1] not in self.extensions: + return None + ref_id = self.get_ref_id(path) + return create_event(FileRemovedEvent, os.path.basename(path), path, reference_id=ref_id) diff --git a/apps/pyWatcher/utils/logger.py b/apps/pyWatcher/utils/logger.py new file mode 100644 index 00000000..561e0dd6 --- /dev/null +++ b/apps/pyWatcher/utils/logger.py @@ -0,0 +1,32 @@ +import logging +import sys + +# ANSI farger +COLORS = { + "INFO": "\033[94m", # blå + "DEBUG": "\033[92m", # grønn + "WARNING": "\033[93m", # gul + "ERROR": "\033[91m", # rød + "RESET": "\033[0m" +} + +class ColoredFormatter(logging.Formatter): + def format(self, record): + levelname = record.levelname + color = COLORS.get(levelname, COLORS["RESET"]) + prefix = f"[{levelname}]" + message = super().format(record) + return f"{color}{prefix}{COLORS['RESET']} {message}" + +def setup_logger(level=logging.INFO): + handler = logging.StreamHandler(sys.stdout) + formatter = ColoredFormatter("%(asctime)s - %(name)s - %(message)s") + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.setLevel(level) + logger.handlers = [handler] + return logger + +# Opprett global logger +logger: logging.Logger = setup_logger() diff --git a/apps/pyWatcher/utils/readiness.py b/apps/pyWatcher/utils/readiness.py new file mode 100644 index 00000000..bf779340 --- /dev/null +++ b/apps/pyWatcher/utils/readiness.py @@ -0,0 +1,21 @@ +import os +import asyncio +from models.event import create_event, FileReadyEvent + +async def file_is_ready(path: str, wait: float = 1.0) -> bool: + try: + size1 = os.path.getsize(path) + await asyncio.sleep(wait) + size2 = os.path.getsize(path) + return size1 == size2 + except Exception: + return False + +async def check_ready(db, ref_id: str, file_name: str, file_uri: str, insert_event): + for _ in range(5): + await asyncio.sleep(2) + if await file_is_ready(file_uri): + ev = create_event(FileReadyEvent, file_name, file_uri, reference_id=ref_id) + insert_event(db, ev) + return ev + return None diff --git a/apps/pyWatcher/worker/file_watcher.py b/apps/pyWatcher/worker/file_watcher.py new file mode 100644 index 00000000..b828e846 --- /dev/null +++ b/apps/pyWatcher/worker/file_watcher.py @@ -0,0 +1,38 @@ +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler +import asyncio +from utils.file_handler import FileHandler +from utils.readiness import check_ready +from utils.logger import logger + +class Handler(FileSystemEventHandler): + def __init__(self, db, extensions, insert_event): + self.db = db + self.file_handler = FileHandler(extensions) + self.insert_event = insert_event + + def on_created(self, event): + if event.is_directory: + return + ev = self.file_handler.handle_created(event.src_path) + if ev: + self.insert_event(self.db, ev) + asyncio.create_task(check_ready(self.db, ev.referenceId, ev.data.fileName, ev.data.fileUri, self.insert_event)) + logger.info(f"➕ FileAddedEvent persisted for {ev.data.fileName}") + + def on_deleted(self, event): + if event.is_directory: + return + ev = self.file_handler.handle_deleted(event.src_path) + if ev: + self.insert_event(self.db, ev) + logger.info(f"➖ FileRemovedEvent persisted for {ev.data.fileName}") + +def start_observer(db, path, extensions, insert_event): + observer = Observer() + handler = Handler(db, extensions, insert_event) + observer.schedule(handler, path, recursive=True) + observer.start() + logger.info(f"👀 Watching path: {path}") + return observer + diff --git a/shared/build.gradle.kts b/shared/build.gradle.kts index 6802aff9..ffa0cdd1 100644 --- a/shared/build.gradle.kts +++ b/shared/build.gradle.kts @@ -17,7 +17,7 @@ repositories { } } -val exposedVersion = "0.44.0" +val exposedVersion = "0.61.0" dependencies { implementation(kotlin("stdlib-jdk8")) diff --git a/shared/common/build.gradle.kts b/shared/common/build.gradle.kts index b8c39da6..db7c6240 100644 --- a/shared/common/build.gradle.kts +++ b/shared/common/build.gradle.kts @@ -38,6 +38,7 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.flywaydb:flyway-core") + implementation("org.flywaydb:flyway-mysql") implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.5.0") @@ -60,7 +61,7 @@ dependencies { implementation("com.zaxxer:HikariCP:7.0.2") implementation(project(":shared:ffmpeg")) - implementation("no.iktdev:eventi:1.0-rc13") + implementation("no.iktdev:eventi:1.0-rc16") testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") @@ -73,6 +74,7 @@ dependencies { testImplementation("io.kotest:kotest-assertions-core:5.7.2") testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0") + testImplementation("io.github.classgraph:classgraph:4.8.184") } @@ -82,4 +84,15 @@ tasks.test { kotlin { jvmToolchain(21) -} \ No newline at end of file +} + +configurations { create("testArtifacts") } + +tasks.register("testJar") { + from(sourceSets.test.get().output) + archiveClassifier.set("tests") +} + +artifacts { + add("testArtifacts", tasks.named("testJar")) +} diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseApplication.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseApplication.kt index 6d94e6a2..b6f9eef2 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseApplication.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseApplication.kt @@ -1,15 +1,35 @@ package no.iktdev.mediaprocessing.shared.common +import org.jetbrains.exposed.sql.Database +import org.springframework.boot.ApplicationArguments +import org.springframework.boot.ApplicationRunner import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.context.annotation.ComponentScan +import org.springframework.stereotype.Component +import javax.sql.DataSource -@SpringBootApplication -@ComponentScan("no.iktdev.mediaprocessing") // sikrer at common beans blir plukket opp abstract class DatabaseApplication { companion object { inline fun launch(args: Array) { runApplication(*args) } } -} \ No newline at end of file +} + +@Component +class ExposedInitializer( + private val dataSource: DataSource +) : ApplicationRunner { + + override fun run(args: ApplicationArguments?) { + Database.connect(dataSource) + } +} + + +@Target(AnnotationTarget.CLASS) +@Retention(AnnotationRetention.RUNTIME) +@SpringBootApplication +@ComponentScan("no.iktdev.mediaprocessing") // sikrer at common beans blir plukket opp +annotation class MediaProcessingApp diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DownloadClient.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DownloadClient.kt new file mode 100755 index 00000000..113a4f1e --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DownloadClient.kt @@ -0,0 +1,163 @@ +package no.iktdev.mediaprocessing.shared.common + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import mu.KotlinLogging +import no.iktdev.exfl.using +import java.io.File +import java.io.FileOutputStream +import java.net.HttpURLConnection +import java.net.URI +import java.net.URL +import java.util.UUID +import kotlin.apply +import kotlin.io.use +import kotlin.run +import kotlin.text.lastIndexOf +import kotlin.text.substring +import kotlin.to + +open class DownloadClient(val url: String, val outDir: File, val baseName: String) { + val log = KotlinLogging.logger {} + protected val http: HttpURLConnection = openConnection() + private val BUFFER_SIZE = 4096 + + private fun openConnection(): HttpURLConnection { + try { + return URI(url).toURL().openConnection() as HttpURLConnection + } catch (e: Exception) { + e.printStackTrace() + throw BadAddressException("Provided url is either not provided (null) or is not a valid http url") + } + } + + protected fun getLength(): Int = http.contentLength + + + protected fun getProgress(read: Int, total: Int = getLength()): Int { + return ((read * 100) / total) + } + + suspend fun download(): File? = withContext(Dispatchers.IO) { + val downloadFile = outDir.using(UUID.randomUUID().toString() + ".downloading") + + if (downloadFile.exists()) { + log.info { "${downloadFile.name} already exists. Download skipped!" } + return@withContext null + } + + val inputStream = http.inputStream + val mimeType: String? = http.contentType + if (mimeType == null) { + log.error { "Unable to determine mime type for $url" } + } else { + log.info { "Downloading file from $url with mime type $mimeType" } + } + + val fos = FileOutputStream(downloadFile, false) + + var totalBytesRead = 0 + val buffer = ByteArray(BUFFER_SIZE) + inputStream.apply { + fos.use { fout -> + run { + var bytesRead = read(buffer) + while (bytesRead >= 0) { + fout.write(buffer, 0, bytesRead) + totalBytesRead += bytesRead + bytesRead = read(buffer) + // System.out.println(getProgress(totalBytesRead)) + } + } + } + } + inputStream.close() + fos.close() + + val extension = getExtension(downloadFile, mimeType ?: "") + ?: throw UnsupportedFormatException("Downloaded file does not contain a supported file extension") + + val outFile = outDir.using("$baseName.$extension") + val renamed = downloadFile.renameTo(outFile) + if (!renamed) { + log.error { "Failed to rename ${downloadFile.name} to ${outFile.name}" } + throw InvalidFileException("Failed to rename downloaded file") + } + + return@withContext outFile + } + + open fun getExtension(outFile: File, mimeType: String): String? { + val extensionFormat = mimeToExtension(mimeType) ?: outFile.getFileType() + if (extensionFormat == null) { + val possiblyExtension = url.lastIndexOf(".") + 1 + if (possiblyExtension > 1) { + return url.substring(possiblyExtension) + } + } + return null + } + + fun mimeToExtension(mimeType: String): String? { + return when(mimeType) { + "image/png" -> "png" + "image/jpg", "image/jpeg" -> "jpg" + "image/webp" -> "webp" + "image/bmp" -> "bmp" + "image/tiff" -> "tiff" + else -> null + } + } + + fun File.getFileType(): String? { + val bytes = this.inputStream().use { it.readNBytes(12) } // les første 12 bytes + return when { + // JPEG: FF D8 FF + bytes.size >= 3 && bytes[0] == 0xFF.toByte() && bytes[1] == 0xD8.toByte() && bytes[2] == 0xFF.toByte() -> "jpg" + + // PNG: 89 50 4E 47 + bytes.size >= 4 && bytes[0] == 0x89.toByte() && bytes[1] == 0x50.toByte() && + bytes[2] == 0x4E.toByte() && bytes[3] == 0x47.toByte() -> "png" + + // GIF: "GIF8" + bytes.size >= 4 && bytes[0] == 'G'.code.toByte() && bytes[1] == 'I'.code.toByte() && + bytes[2] == 'F'.code.toByte() && bytes[3] == '8'.code.toByte() -> "gif" + + // WEBP: RIFF....WEBP + bytes.size >= 12 && bytes[0] == 'R'.code.toByte() && bytes[1] == 'I'.code.toByte() && + bytes[2] == 'F'.code.toByte() && bytes[3] == 'F'.code.toByte() && + bytes[8] == 'W'.code.toByte() && bytes[9] == 'E'.code.toByte() && + bytes[10] == 'B'.code.toByte() && bytes[11] == 'P'.code.toByte() -> "webp" + + // BMP: 42 4D ("BM") + bytes.size >= 2 && bytes[0] == 0x42.toByte() && bytes[1] == 0x4D.toByte() -> "bmp" + + // TIFF: enten "II*" eller "MM*" + bytes.size >= 4 && ( + (bytes[0] == 'I'.code.toByte() && bytes[1] == 'I'.code.toByte() && bytes[2] == 0x2A.toByte()) || + (bytes[0] == 'M'.code.toByte() && bytes[1] == 'M'.code.toByte() && bytes[2] == 0x2A.toByte()) + ) -> "tiff" + + else -> null + } + } + + + class BadAddressException : java.lang.Exception { + constructor() : super() {} + constructor(message: String?) : super(message) {} + constructor(message: String?, cause: Throwable?) : super(message, cause) {} + } + + class UnsupportedFormatException : Exception { + constructor() : super() {} + constructor(message: String?) : super(message) {} + constructor(message: String?, cause: Throwable?) : super(message, cause) {} + } + + class InvalidFileException : Exception { + constructor() : super() {} + constructor(message: String?) : super(message) {} + constructor(message: String?, cause: Throwable?) : super(message, cause) {} + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt index 1ca8dfec..c34c089d 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt @@ -1,18 +1,15 @@ package no.iktdev.mediaprocessing.shared.common -import com.google.gson.GsonBuilder import kotlinx.coroutines.delay import mu.KotlinLogging -import no.iktdev.eventi.ZDS +import no.iktdev.eventi.models.Event import org.springframework.messaging.simp.SimpMessagingTemplate import org.springframework.web.client.RestTemplate -import org.springframework.web.client.postForEntity import java.io.File import java.io.FileInputStream import java.io.RandomAccessFile import java.net.InetAddress import java.security.MessageDigest -import java.time.LocalDateTime import java.util.zip.CRC32 private val logger = KotlinLogging.logger {} @@ -184,4 +181,36 @@ fun SimpMessagingTemplate.trySend(destination: String, data: Any, onError: ((Exc } catch (e: Exception) { onError?.invoke(e) } -} \ No newline at end of file +} + + +inline fun List.getInstanceOf(): T? { + return this.firstOrNull { it is T } as? T +} + +// Extension-funksjon på List som returnerer alle instanser av T +inline fun List.getInstancesOf(): List { + return this.filterIsInstance() +} + +inline fun List.sizeEquals(other: List): Boolean { + return this.size == other.size +} + +fun File.resolveConflict(): File { + if (!exists()) return this + + val parent = parentFile + val name = nameWithoutExtension + val ext = extension + + var index = 1 + var candidate: File + + do { + candidate = File(parent, "$name ($index).$ext") + index++ + } while (candidate.exists()) + + return candidate +} diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfig.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfig.kt index 1fef2b5a..0822404a 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfig.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfig.kt @@ -1,51 +1,10 @@ package no.iktdev.mediaprocessing.shared.common.database -import com.zaxxer.hikari.HikariConfig -import com.zaxxer.hikari.HikariDataSource -import org.jetbrains.exposed.sql.Database -import javax.sql.DataSource - -object DatabaseConfig { - fun connect( - access: Access, - maxPoolSize: Int = 10 - ): Pair { - val jdbcUrl = when (access.dbType) { - DbType.MySQL -> "jdbc:mysql://${access.address}:${access.port}/${access.databaseName}?useSSL=false&serverTimezone=UTC" - DbType.PostgreSQL -> "jdbc:postgresql://${access.address}:${access.port}/${access.databaseName}" - DbType.SQLite -> "jdbc:sqlite:${access.databaseName}.db" - DbType.H2 -> "jdbc:h2:mem:${access.databaseName};DB_CLOSE_DELAY=-1" - } - - val driver = when (access.dbType) { - DbType.MySQL -> "com.mysql.cj.jdbc.Driver" - DbType.PostgreSQL -> "org.postgresql.Driver" - DbType.SQLite -> "org.sqlite.JDBC" - DbType.H2 -> "org.h2.Driver" - } - - val config = HikariConfig().apply { - this.jdbcUrl = jdbcUrl - this.driverClassName = driver - this.username = access.username - this.password = access.password - this.maximumPoolSize = maxPoolSize - this.isAutoCommit = false - this.transactionIsolation = "TRANSACTION_REPEATABLE_READ" - this.validate() - } - - val dataSource = HikariDataSource(config) - val db = Database.connect(dataSource) - return db to dataSource - } -} - data class Access( val username: String, val password: String, val address: String, val port: Int, val databaseName: String, - val dbType: DbType + val dbType: DatabaseTypes ) {} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfiguration.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfiguration.kt index c265793f..72e3449c 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfiguration.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfiguration.kt @@ -1,15 +1,15 @@ package no.iktdev.mediaprocessing.shared.common.database +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource import jakarta.annotation.PostConstruct import org.flywaydb.core.Flyway -import org.jetbrains.exposed.sql.Database import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration -import org.springframework.stereotype.Component import javax.sql.DataSource @Configuration @@ -17,15 +17,35 @@ open class DatabaseConfiguration { @Bean fun dataSource(): DataSource { - val access = Access( - username = "sa", - password = "", - address = "", // ikke brukt for H2 - port = 0, // ikke brukt for H2 - databaseName = "testdb", - dbType = DbType.H2 - ) - return DatabaseConfig.connect(access).second + val maxPoolSize: Int = 10 + val access = DatabaseEnv.toAccess() + + val jdbcUrl = when (access.dbType) { + DatabaseTypes.MySQL -> "jdbc:mysql://${access.address}:${access.port}/${access.databaseName}?allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC" + DatabaseTypes.PostgreSQL -> "jdbc:postgresql://${access.address}:${access.port}/${access.databaseName}" + DatabaseTypes.SQLite -> "jdbc:sqlite:${access.databaseName}.db" + DatabaseTypes.H2 -> "jdbc:h2:mem:${access.databaseName};MODE=MySQL;DB_CLOSE_DELAY=-1" + } + + val driver = when (access.dbType) { + DatabaseTypes.MySQL -> "com.mysql.cj.jdbc.Driver" + DatabaseTypes.PostgreSQL -> "org.postgresql.Driver" + DatabaseTypes.SQLite -> "org.sqlite.JDBC" + DatabaseTypes.H2 -> "org.h2.Driver" + } + + val config = HikariConfig().apply { + this.jdbcUrl = jdbcUrl + this.driverClassName = driver + this.username = access.username + this.password = access.password + this.maximumPoolSize = maxPoolSize + this.isAutoCommit = false + this.transactionIsolation = "TRANSACTION_REPEATABLE_READ" + this.validate() + } + + return HikariDataSource(config) } } @@ -52,7 +72,7 @@ class FlywayAutoConfig( val pending = flyway.info().pending() if (pending.isEmpty()) { - log.warn("⚠️ No pending Flyway migrations found in ${locations.joinToString()}") + log.info("⚠️ No pending Flyway migrations found in ${locations.joinToString()}") } else { log.info("📦 Pending migrations: ${pending.joinToString { it.script }}") } diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseEnv.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseEnv.kt index a500c913..83d821a9 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseEnv.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseEnv.kt @@ -1,31 +1,30 @@ package no.iktdev.mediaprocessing.shared.common.database object DatabaseEnv { - val address: String? = System.getenv("DATABASE_ADDRESS") - val port: String? = System.getenv("DATABASE_PORT") - val username: String? = System.getenv("DATABASE_USERNAME") - val password: String? = System.getenv("DATABASE_PASSWORD") - val database: String? = System.getenv("DATABASE_NAME") - val databaseType: DbType = DbType.valueOf(System.getenv("DATABASE_TYPE") ?: "MySQL") + + fun address() = System.getenv("DATABASE_ADDRESS") ?: "localhost" + fun port(): String? = System.getenv("DATABASE_PORT") + fun username() = System.getenv("DATABASE_USERNAME") ?: "root" + fun password() = System.getenv("DATABASE_PASSWORD") ?: "" + fun databaseName() = System.getenv("DATABASE_NAME") ?: "mediaprocessing" + fun databaseType() = DatabaseTypes.valueOf(System.getenv("DATABASE_TYPE") ?: "MySQL") + fun toAccess(): Access { + val databaseType = databaseType() return Access( - username = username ?: "root", - password = password ?: "", - address = address ?: "localhost", - port = port?.toIntOrNull() ?: when (databaseType) { - DbType.MySQL -> 3306 - DbType.PostgreSQL -> 5432 - DbType.SQLite -> 0 - DbType.H2 -> 0 + username = username(), + password = password(), + address = address(), + port = port()?.toIntOrNull() ?: when (databaseType) { + DatabaseTypes.MySQL -> 3306 + DatabaseTypes.PostgreSQL -> 5432 + DatabaseTypes.SQLite -> 0 + DatabaseTypes.H2 -> 0 }, - databaseName = database ?: "mediaprocessing", + databaseName = databaseName() ?: "streamit", dbType = databaseType ) } } - -enum class DbType { - MySQL, PostgreSQL, SQLite, H2 -} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseTypes.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseTypes.kt new file mode 100644 index 00000000..167a4404 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseTypes.kt @@ -0,0 +1,5 @@ +package no.iktdev.mediaprocessing.shared.common.database + +enum class DatabaseTypes { + MySQL, PostgreSQL, SQLite, H2 +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/EventRegistry.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/EventRegistry.kt index ea47e7fb..19a02760 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/EventRegistry.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/EventRegistry.kt @@ -5,41 +5,76 @@ import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.FileAd import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.FileReadyEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.FileRemovedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchTaskCreated -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchTaskPerformed -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeResultEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeTaskCreatedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractTaskCreatedEvent -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractEvent -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserReadTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksEncodeSelectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksExtractSelectedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodePerformedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractPerformedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent object EventRegistry { fun getEvents(): List> { return listOf( + CollectedEvent::class.java, + ConvertTaskCreatedEvent::class.java, ConvertTaskResultEvent::class.java, + CoordinatorReadStreamsResultEvent::class.java, + CoordinatorReadStreamsTaskCreatedEvent::class.java, + + CoverDownloadTaskCreatedEvent::class.java, + CoverDownloadResultEvent::class.java, + FileAddedEvent::class.java, FileReadyEvent::class.java, FileRemovedEvent::class.java, MediaParsedInfoEvent::class.java, + MediaStreamParsedEvent::class.java, + MediaTracksDetermineSubtitleTypeEvent::class.java, + MediaTracksEncodeSelectedEvent::class.java, + MediaTracksExtractSelectedEvent::class.java, - MetadataSearchTaskCreated::class.java, - MetadataSearchTaskPerformed::class.java, + MetadataSearchResultEvent::class.java, + MetadataSearchTaskCreatedEvent::class.java, + MigrateContentToStoreTaskCreatedEvent::class.java, + MigrateContentToStoreTaskResultEvent::class.java, + + ProcesserEncodePerformedEvent::class.java, + ProcesserEncodeResultEvent::class.java, + ProcesserEncodeTaskCreatedEvent::class.java, + + ProcesserExtractPerformedEvent::class.java, + ProcesserExtractResultEvent::class.java, ProcesserExtractTaskCreatedEvent::class.java, - ProcesserExtractEvent::class.java, ProcesserEncodeTaskCreatedEvent::class.java, - ProcesserEncodeEvent::class.java, + ProcesserEncodeResultEvent::class.java, - ProcesserReadTaskCreatedEvent::class.java, // Do i need this? + StartProcessingEvent::class.java, - StartProcessingEvent::class.java + StoreContentAndMetadataTaskCreatedEvent::class.java, + StoreContentAndMetadataTaskResultEvent::class.java ) } } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/TaskRegistry.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/TaskRegistry.kt index 4ddfb257..e592b31a 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/TaskRegistry.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/TaskRegistry.kt @@ -6,15 +6,24 @@ import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.Extract import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MetadataSearchTask import no.iktdev.eventi.models.Task +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask object TaskRegistry { fun getTasks(): List> { return listOf( ConvertTask::class.java, + CoverDownloadTask::class.java, + EncodeTask::class.java, ExtractSubtitleTask::class.java, + MediaReadTask::class.java, - MetadataSearchTask::class.java + MetadataSearchTask::class.java, + MigrateToContentStoreTask::class.java, + + StoreContentAndMetadataTask::class.java, ) } } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CollectedEvent.kt similarity index 62% rename from shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadTaskCreatedEvent.kt rename to shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CollectedEvent.kt index 2c60982c..f37c2b5d 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadTaskCreatedEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CollectedEvent.kt @@ -1,9 +1,9 @@ package no.iktdev.mediaprocessing.shared.common.event_task_contract.events import no.iktdev.eventi.models.Event +import java.util.UUID - -data class MediaStreamReadTaskCreatedEvent( - val fileUri: String +data class CollectedEvent( + val eventIds: Set ): Event() { } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskCreatedEvent.kt index 99ab3eec..02e0af8c 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskCreatedEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskCreatedEvent.kt @@ -1,6 +1,9 @@ package no.iktdev.mediaprocessing.shared.common.event_task_contract.events import no.iktdev.eventi.models.Event +import java.util.UUID -class ConvertTaskCreatedEvent: Event() { +data class ConvertTaskCreatedEvent( + val taskId: UUID +): Event() { } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskResultEvent.kt index e08af6f5..9ebcdb8a 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskResultEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConvertTaskResultEvent.kt @@ -7,10 +7,10 @@ data class ConvertTaskResultEvent( val data: ConvertedData?, val status: TaskStatus, ): Event() { + data class ConvertedData( + val language: String, + val baseName: String, + val outputFiles: List + ) } -data class ConvertedData( - val language: String, - val baseName: String, - val outputFiles: List -) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoordinatorReadStreamsResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoordinatorReadStreamsResultEvent.kt new file mode 100644 index 00000000..124c1b58 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoordinatorReadStreamsResultEvent.kt @@ -0,0 +1,11 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import com.google.gson.JsonObject +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + +data class CoordinatorReadStreamsResultEvent( + val data: JsonObject? = null, + val status: TaskStatus +): Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoordinatorReadStreamsTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoordinatorReadStreamsTaskCreatedEvent.kt new file mode 100644 index 00000000..c72c104b --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoordinatorReadStreamsTaskCreatedEvent.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import java.util.UUID + +data class CoordinatorReadStreamsTaskCreatedEvent( + val taskId: UUID +): Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadResultEvent.kt new file mode 100644 index 00000000..144147d3 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadResultEvent.kt @@ -0,0 +1,15 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + +data class CoverDownloadResultEvent( + val data: CoverDownloadedData? = null, + val status: TaskStatus +): Event() { + data class CoverDownloadedData( + val source: String, + val outputFile: String + ) +} + diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadTaskCreatedEvent.kt similarity index 58% rename from shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadEvent.kt rename to shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadTaskCreatedEvent.kt index e5db309c..a35e8c2f 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadTaskCreatedEvent.kt @@ -1,9 +1,9 @@ package no.iktdev.mediaprocessing.shared.common.event_task_contract.events -import com.google.gson.JsonObject import no.iktdev.eventi.models.Event +import java.util.UUID -data class MediaStreamReadEvent( - val data: JsonObject +data class CoverDownloadTaskCreatedEvent( + val taskIds: List ): Event() { } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadedEvent.kt deleted file mode 100644 index 08a99c02..00000000 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadedEvent.kt +++ /dev/null @@ -1,12 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common.event_task_contract.events - -import no.iktdev.eventi.models.Event - -data class CoverDownloadedEvent( - val data: CoverDownloadedData -): Event() { -} - -data class CoverDownloadedData( - val outputFile: String -) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaParsedInfoEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaParsedInfoEvent.kt index a4ea7d05..89310870 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaParsedInfoEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaParsedInfoEvent.kt @@ -12,8 +12,14 @@ class MediaParsedInfoEvent( val parsedCollection: String, val parsedFileName: String, val parsedSearchTitles: List, - val mediaType: MediaType + val mediaType: MediaType, + val episodeInfo: EpisodeInfo? = null ) { + data class EpisodeInfo( + val episodeNumber: Int, + val seasonNumber: Int, + val episodeTitle: String? = null, + ) } } diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataEvents.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataEvents.kt deleted file mode 100644 index 3860895d..00000000 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataEvents.kt +++ /dev/null @@ -1,30 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common.event_task_contract.events - -import no.iktdev.eventi.models.Event -import no.iktdev.eventi.models.store.TaskStatus - - -class MetadataSearchTaskCreated(): Event() {} - -class MetadataSearchTaskPerformed( - val data: pyMetadata? = null, - val taskStatus: TaskStatus -): Event() { - init { - assert(taskStatus in listOf(TaskStatus.Completed, TaskStatus.Failed), { "Task status is not of acceptable state $taskStatus" }) - } -} - -data class pyMetadata( - val title: String, - val altTitle: List = emptyList(), - val cover: String? = null, - val type: String, - val summary: List = emptyList(), - val genres: List = emptyList() -) - -data class pySummary( - val summary: String?, - val language: String = "eng" -) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataSearchResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataSearchResultEvent.kt new file mode 100644 index 00000000..1f2c6f0f --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataSearchResultEvent.kt @@ -0,0 +1,43 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Metadata +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import java.util.UUID + +data class MetadataSearchResultEvent( + val results: List = emptyList(), + val recommended: SearchResult? = null, + val status: TaskStatus +): Event() { + data class SearchResult( + val simpleScore: Int, + val prefixScore: Int, + val advancedScore: Int, + val sourceWeight: Float, + val data: MetadataResult + ) { + + data class MetadataResult( + val source: String, + val title: String, + val alternateTitles: List = emptyList(), + val cover: String, + val bannerImage: String? = null, + val type: MediaType, + val summary: List, + val genres: List + ) { + data class Summary(val language: String, val description: String) + } + } + + fun setFailed(derivedIds: List) { + assert(status == TaskStatus.Failed) + val metadata = Metadata().apply { + this.derivedFromEventId(derivedIds.toSet()) + } + this.metadata = metadata + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataSearchTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataSearchTaskCreatedEvent.kt new file mode 100644 index 00000000..a819415e --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataSearchTaskCreatedEvent.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import java.util.UUID + +data class MetadataSearchTaskCreatedEvent( + val taskId: UUID +): Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MigrateContentToStoreTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MigrateContentToStoreTaskCreatedEvent.kt new file mode 100644 index 00000000..50bda1a4 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MigrateContentToStoreTaskCreatedEvent.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import java.util.UUID + +data class MigrateContentToStoreTaskCreatedEvent( + val taskId: UUID +): Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MigrateContentToStoreTaskResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MigrateContentToStoreTaskResultEvent.kt new file mode 100644 index 00000000..35daac2d --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MigrateContentToStoreTaskResultEvent.kt @@ -0,0 +1,30 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus + +data class MigrateContentToStoreTaskResultEvent( + val status: TaskStatus, + val collection: String, + val videoMigrate: FileMigration, + val subtitleMigrate: List, + val coverMigrate: List +) : Event() { + data class FileMigration( + val storedUri: String?, + val status: MigrateStatus + ) + + data class SubtitleMigration( + val language: String?, + val storedUri: String?, + val status: MigrateStatus + ) { + init { + if (status == MigrateStatus.Completed && language == null) + throw IllegalStateException("SubtitleMigration: language cannot be null when status is COMPLETED") + + } + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEncodeResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEncodeResultEvent.kt new file mode 100644 index 00000000..792d4bb2 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEncodeResultEvent.kt @@ -0,0 +1,13 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + +data class ProcesserEncodeResultEvent( + val data: EncodeResult? = null, + val status: TaskStatus, +): Event() { + data class EncodeResult( + val cachedOutputFile: String? = null + ) +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEncodeTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEncodeTaskCreatedEvent.kt index a9840741..d0111e30 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEncodeTaskCreatedEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEncodeTaskCreatedEvent.kt @@ -1,6 +1,8 @@ package no.iktdev.mediaprocessing.shared.common.event_task_contract.events import no.iktdev.eventi.models.Event +import java.util.UUID -class ProcesserEncodeTaskCreatedEvent: Event() { -} \ No newline at end of file +data class ProcesserEncodeTaskCreatedEvent( + val taskCreated: UUID +): Event() \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEvents.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEvents.kt deleted file mode 100644 index 584b4452..00000000 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEvents.kt +++ /dev/null @@ -1,35 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common.event_task_contract.events - -import no.iktdev.eventi.models.Event -import no.iktdev.eventi.models.store.TaskStatus - -// Placeholder event, so that the listener does not continue to create tasks -class ProcesserExtractTaskCreatedEvent: Event() { -} - - -// Placeholder event, so that the listener does not continue to create tasks -class ProcesserReadTaskCreatedEvent: Event() { -} - - -data class ProcesserEncodeEvent( - val data: EncodeResult -): Event() { - -} - -data class EncodeResult( - val status: TaskStatus, - val cachedOutputFile: String? = null -) - - -data class ProcesserExtractEvent( - val data: ExtractResult -): Event() - -data class ExtractResult( - val status: TaskStatus, - val cachedOutputFile: String? = null -) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaReadyEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractPerformedEvent.kt similarity index 63% rename from shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaReadyEvent.kt rename to shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractPerformedEvent.kt index 0249bca9..3cc871b3 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaReadyEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractPerformedEvent.kt @@ -2,7 +2,5 @@ package no.iktdev.mediaprocessing.shared.common.event_task_contract.events import no.iktdev.eventi.models.Event -data class MediaReadyEvent( - val fileUri: String -): Event() { +class ProcesserExtractPerformedEvent: Event() { } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractResultEvent.kt new file mode 100644 index 00000000..d6eb1528 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractResultEvent.kt @@ -0,0 +1,14 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + +data class ProcesserExtractResultEvent( + val status: TaskStatus, + val data: ExtractResult? = null +): Event() { + data class ExtractResult( + val language: String, + val cachedOutputFile: String + ) +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractTaskCreatedEvent.kt new file mode 100644 index 00000000..ab6d6b7d --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserExtractTaskCreatedEvent.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import java.util.UUID + +data class ProcesserExtractTaskCreatedEvent( + val tasksCreated: MutableList = mutableListOf() +): Event() \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StartProcessingEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StartProcessingEvent.kt index b2c06039..00f57b73 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StartProcessingEvent.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StartProcessingEvent.kt @@ -10,11 +10,11 @@ data class StartProcessingEvent( data class StartData( val operation: Set, - val flow: ProcessFlow = ProcessFlow.Auto, + val flow: StartFlow = StartFlow.Auto, val fileUri: String, ) -enum class ProcessFlow { +enum class StartFlow { Auto, Manual } diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StoreContentAndMetadataTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StoreContentAndMetadataTaskCreatedEvent.kt new file mode 100644 index 00000000..4f81c85c --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StoreContentAndMetadataTaskCreatedEvent.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import java.util.UUID + +data class StoreContentAndMetadataTaskCreatedEvent( + val taskId: UUID +): Event() {} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StoreContentAndMetadataTaskResultEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StoreContentAndMetadataTaskResultEvent.kt new file mode 100644 index 00000000..c3de0f27 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StoreContentAndMetadataTaskResultEvent.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + +data class StoreContentAndMetadataTaskResultEvent( + val taskStatus: TaskStatus, +) : Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ConvertTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ConvertTask.kt index 1b2fe746..648fa298 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ConvertTask.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ConvertTask.kt @@ -1,26 +1,18 @@ package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks import no.iktdev.eventi.models.Task +import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat data class ConvertTask( val data: Data ): Task() { + data class Data( + val inputFile: String, + val language: String, + val outputDirectory: String, + val outputFileName: String, + val formats: List = emptyList(), + val allowOverwrite: Boolean + ) } -data class Data( - val inputFile: String, - val language: String, - val outputDirectory: String, - val outputFileName: String, - val storeFileName: String, - val formats: List = emptyList(), - val allowOverwrite: Boolean -) - - -enum class SubtitleFormats { - ASS, - SRT, - VTT, - SMI -} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/CoverDownloadTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/CoverDownloadTask.kt index b499999d..d2d3da9e 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/CoverDownloadTask.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/CoverDownloadTask.kt @@ -5,6 +5,10 @@ import no.iktdev.eventi.models.Task data class CoverDownloadTask( val data: CoverDownloadData ): Task() { + data class CoverDownloadData( + val url: String, + val source: String, + val outputFileName: String + ) } -data class CoverDownloadData(val url: String, val outputFile: String) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ExtractSubtitleTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ExtractSubtitleTask.kt index ed8760c6..ea0714db 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ExtractSubtitleTask.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ExtractSubtitleTask.kt @@ -8,10 +8,10 @@ data class ExtractSubtitleTask( } data class ExtractSubtitleData( + val inputFile: String, val arguments: List, val outputFileName: String, val language: String, - val inputFile: String ) { } diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MetadataSearchTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MetadataSearchTask.kt index 4ec96bc6..b839ce23 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MetadataSearchTask.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MetadataSearchTask.kt @@ -3,11 +3,11 @@ package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks import no.iktdev.eventi.models.Task data class MetadataSearchTask( - val data: MetadataSearchData -): Task() {} + val data: SearchData +): Task() { + data class SearchData( + val searchTitles: List, + val collection: String, + ) +} -data class MetadataSearchData( - val searchString: String, - val maxResults: Int = 10, - val offset: Int = 0, -) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MigrateToContentStoreTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MigrateToContentStoreTask.kt new file mode 100644 index 00000000..3a226912 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MigrateToContentStoreTask.kt @@ -0,0 +1,24 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task + +data class MigrateToContentStoreTask( + val data: Data +): Task() { + data class Data( + val collection: String, + val videoContent: SingleContent? = null, + val subtitleContent: List? = null, // Both extracted and converted + val coverContent: List? = null + ) { + data class SingleContent( + val cachedUri: String, + val storeUri: String + ) + data class SingleSubtitle( + val language: String, + val cachedUri: String, + val storeUri: String + ) + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/StoreContentAndMetadataTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/StoreContentAndMetadataTask.kt new file mode 100644 index 00000000..f9786ceb --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/StoreContentAndMetadataTask.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task +import no.iktdev.mediaprocessing.shared.common.model.ContentExport + +data class StoreContentAndMetadataTask( + val data: ContentExport +): Task() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/ContentExport.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/ContentExport.kt new file mode 100644 index 00000000..8e057d8b --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/ContentExport.kt @@ -0,0 +1,36 @@ +package no.iktdev.mediaprocessing.shared.common.model + +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent + +data class ContentExport( + val collection: String, + val episodeInfo: EpisodeInfo? = null, + val media: MediaExport? = null, + val metadata: MetadataExport +) { + + data class MetadataExport( + // Vi tar ikke med collection fra metadata, da det bestemmes i Migrate + val title: String, + val genres: List = emptyList(), + val cover: String? = null, + val summary: List = emptyList(), + val mediaType: MediaType, + val source: String? = null + ) + + data class MediaExport( + val videoFile: String?, + val subtitles: List, + ) { + data class Subtitle( + val subtitleFile: String, + val language: String + ) + } + data class EpisodeInfo( + val episodeNumber: Int, + val seasonNumber: Int, + val episodeTitle: String? = null, + ) +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaType.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaType.kt index 049e4aaf..ba0b5178 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaType.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaType.kt @@ -2,5 +2,6 @@ package no.iktdev.mediaprocessing.shared.common.model enum class MediaType { Movie, - Serie + Serie, + Subtitle } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MigrateStatus.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MigrateStatus.kt new file mode 100644 index 00000000..2820f9ea --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MigrateStatus.kt @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing.shared.common.model + +enum class MigrateStatus { + NotPresent, + Completed, + Failed +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/SubtitleFormat.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/SubtitleFormat.kt new file mode 100644 index 00000000..20344b48 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/SubtitleFormat.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.common.model + +enum class SubtitleFormat { + ASS, + SRT, + VTT, + SMI +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/CollectProjection.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/CollectProjection.kt new file mode 100644 index 00000000..dc43c8e0 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/CollectProjection.kt @@ -0,0 +1,193 @@ +package no.iktdev.mediaprocessing.shared.common.projection + +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartFlow +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import java.io.File + +class CollectProjection(val events: List) { + + val startedWith: StartProjection by lazy { projectStartedWith() } + var metadataTaskStatus: TaskStatus = TaskStatus.NotInitiated + private set + var encodeTaskStatus: TaskStatus = TaskStatus.NotInitiated + private set + var extreactTaskStatus: TaskStatus = TaskStatus.NotInitiated + private set + var convertTaskStatus: TaskStatus = TaskStatus.NotInitiated + private set + var coverDownloadTaskStatus: TaskStatus = TaskStatus.NotInitiated + private set + val metadata: MetadataProjection? by lazy { projectMetadata() } + val processedMedia: ProcessedMediaProjection? by lazy { projectProcessedMedia() } + val parsedFileInfo: ParsedFileInfoProjection? by lazy { projectParsedFileInfo() } + + init { + val taskProjection = TaskProjection(events) + metadataTaskStatus = taskProjection.projectMetadataSearchStatus() + encodeTaskStatus = taskProjection.projectEncodingPerformedStatus() + extreactTaskStatus = taskProjection.projectExtractSubtitleStatus() + convertTaskStatus = taskProjection.projectConvertStatus() + coverDownloadTaskStatus = taskProjection.projectCoverDownloadStatus() + + } + + fun getTaskStatus(): List = listOf( + metadataTaskStatus, + encodeTaskStatus, + extreactTaskStatus, + convertTaskStatus, + coverDownloadTaskStatus + ) + + private fun projectStartedWith(): StartProjection { + val startEvent = events.filterIsInstance().first() + return StartProjection( + inputFile = startEvent.data.fileUri.let { File(it) }, + mode = startEvent.data.flow, + tasks = startEvent.data.operation + ) + } + + + + private fun projectMetadata(): MetadataProjection? { + val metadataEvent = events.filterIsInstance().lastOrNull() + ?: return null + val coverDownloadResultEvents = events.filterIsInstance().filter { it.status == no.iktdev.eventi.models.store.TaskStatus.Completed } + val coverFile = coverDownloadResultEvents.find { it -> it.data?.source == metadataEvent.recommended?.data?.source }?.data?.outputFile + ?.let { File(it) } + val result = metadataEvent.recommended ?: return null + return MetadataProjection( + title = result.data.title, + summary = result.data.summary, + mediaType = result.data.type, + genres = result.data.genres, + cover = coverFile, + source = result.data.source + ) + } + + private fun projectProcessedMedia(): ProcessedMediaProjection? { + val encodeEvent = events.filterIsInstance().lastOrNull { it.status == no.iktdev.eventi.models.store.TaskStatus.Completed } + ?: return null + + val extreactEvents = events.filterIsInstance() + val extractedFiles = if (extreactEvents.all { it.status == no.iktdev.eventi.models.store.TaskStatus.Completed }) { + extreactEvents.mapNotNull { it.data?.cachedOutputFile?.let { filePath -> File(filePath) } } + } else { + emptyList() + } + + val convertedEvents = events.filterIsInstance() + val convertedFiles = if (convertedEvents.all { it.status == no.iktdev.eventi.models.store.TaskStatus.Completed }) { + convertedEvents.flatMap { it.data?.outputFiles?.map { filePath -> File(filePath) } ?: emptyList() } + } else { + emptyList() + } + + val encodedFile = encodeEvent.data?.cachedOutputFile?.let { File(it) } + + return ProcessedMediaProjection( + encodedFile = encodedFile, + extractedFiles = extractedFiles, + convertedFiles = convertedFiles + ) + } + + private fun projectParsedFileInfo(): ParsedFileInfoProjection? { + val result = events.filterIsInstance().lastOrNull() ?: return null + return ParsedFileInfoProjection( + name = result.data.parsedFileName, + collection = result.data.parsedCollection, + mediaType = result.data.mediaType + ) + } + + + data class StartProjection( + val inputFile: File, + val mode: StartFlow, + val tasks: Set + ) + + + data class MetadataProjection( + val title: String, + val summary: List, + val mediaType: MediaType, + val genres: List, + val cover: File?, + val source: String + ) + + data class ParsedFileInfoProjection( + val name: String, + val collection: String, + val mediaType: MediaType + ) + + data class ProcessedMediaProjection( + val encodedFile: File?, + val extractedFiles: List, + val convertedFiles: List + ) + + + + + enum class TaskStatus { + NotInitiated, + Pending, + Completed, + Failed + } + + fun prettyPrint(): String = buildString { + appendLine("📦 Project snapshot") + appendLine("Started with: ${startedWith.inputFile.name} [mode=${startedWith.mode}, tasks=${startedWith.tasks}]") + appendLine("Task statuses:") + appendLine(" - Metadata: ${metadataTaskStatus.colored()}") + appendLine(" - Encode: ${encodeTaskStatus.colored()}") + appendLine(" - Extract: ${extreactTaskStatus.colored()}") + appendLine(" - Convert: ${convertTaskStatus.colored()}") + appendLine(" - Cover: ${coverDownloadTaskStatus.colored()}") + + metadata?.let { + appendLine("Metadata:") + appendLine(" • Title: ${it.title}") + appendLine(" • Genres: ${it.genres.joinToString()}") + appendLine(" • Source: ${it.source}") + appendLine(" • Cover: ${it.cover?.path ?: "none"}") + } + + parsedFileInfo?.let { + appendLine("Parsed file info:") + appendLine(" • Name: ${it.name}") + appendLine(" • Collection: ${it.collection}") + appendLine(" • Type: ${it.mediaType}") + } + + processedMedia?.let { + appendLine("Processed media:") + appendLine(" • Encoded: ${it.encodedFile?.path ?: "none"}") + appendLine(" • Extracted: ${it.extractedFiles.joinToString { f -> f.name }}") + appendLine(" • Converted: ${it.convertedFiles.joinToString { f -> f.name }}") + } + } + + private fun TaskStatus.colored(): String = when (this) { + TaskStatus.NotInitiated -> "\u001B[90m$this\u001B[0m" // grå + TaskStatus.Pending -> "\u001B[33m$this\u001B[0m" // gul + TaskStatus.Completed -> "\u001B[32m$this\u001B[0m" // grønn + TaskStatus.Failed -> "\u001B[31m$this\u001B[0m" // rød + } + +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/MigrateContentProject.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/MigrateContentProject.kt new file mode 100644 index 00000000..944fced3 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/MigrateContentProject.kt @@ -0,0 +1,141 @@ +package no.iktdev.mediaprocessing.shared.common.projection + +import no.iktdev.eventi.models.Event +import no.iktdev.exfl.using +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.* +import no.iktdev.mediaprocessing.shared.common.resolveConflict +import java.io.File + +open class MigrateContentProject(val events: List, val storageArea: File) { + val useStore: File? = getDesiredStoreFolder() + + open fun getFoldersInStore(): List { + return storageArea.listFiles { file -> file.isDirectory }?.map { it.name }?.toList() ?: emptyList() + } + + internal fun getFileName(): String? { + val parsedInfo = events.filterIsInstance().lastOrNull() ?: return null + return parsedInfo.data.parsedFileName + } + + internal fun getDesiredStoreFolder(): File? { + val assuredCollection = getDesiredCollection() ?: return null + val assuredStore = storageArea.using(assuredCollection) + val existingCollectionNames = getFoldersInStore().ifEmpty { + return assuredStore + } + val titles = getMetadataTitles() + val existingCollection = titles.filter { it in existingCollectionNames }.firstOrNull()?.let { + File(it) + } ?: assuredStore + return existingCollection + } + + internal fun getMetadataTitles(): List { + val metadataEvent = events.filterIsInstance().lastOrNull()?.recommended?.data + ?: return emptyList() + return (metadataEvent.alternateTitles + listOf(metadataEvent.title)) + } + + internal fun getDesiredCollection(): String? { + val metadataEvent = events.filterIsInstance().lastOrNull()?.data + return metadataEvent?.parsedCollection + } + + + fun getVideoStoreFile(): CachedToStore? { + val encoded = events.filterIsInstance().lastOrNull() + val cached = encoded?.data?.cachedOutputFile?.let { File(it) } ?: return null + val useFilename = getFileName()?.let { "$it.${cached.extension}" } ?: return null + val store = useStore?.using(useFilename) ?: return null + return CachedToStore( + cachedFile = cached, + storeFile = store + ) + } + + fun getSubtitleStoreFiles(): List? { + val extractedFiles: Map> = events + .filterIsInstance() + .mapNotNull { it.data } + .map { data -> data.language to File(data.cachedOutputFile) } + .groupBy({ it.first }, { it.second }) + + val convertedFilesGrouped: Map> = events + .filterIsInstance() + .mapNotNull { it.data } + .map { x -> x.outputFiles.map { x.language to File(it) } } + .flatten() + .groupBy({ it.first }, { it.second }) + + + val byLanguage = mutableMapOf>() + extractedFiles.forEach { (lang, files) -> + byLanguage.getOrPut(lang) { mutableListOf() }.addAll(files) + } + convertedFilesGrouped.forEach { (lang, files) -> + byLanguage.getOrPut(lang) { mutableListOf() }.addAll(files) + } + + val useFilename = getFileName() ?: return null + + return byLanguage.flatMap { (language, files) -> + files.mapNotNull { cached -> + val useFilename = "$useFilename.${cached.extension}" + val store = useStore?.using(language, useFilename) ?: return@mapNotNull null + CachedToStoreLanguage( + CachedToStore( + cachedFile = cached, + storeFile = store + ), + language = language + ) + } + } + } + + fun getCoverStoreFiles(): List? { + val downloaded = events.filterIsInstance() + .mapNotNull { event -> + val file = event.data?.outputFile?.let(::File) ?: return@mapNotNull null + event to file + } + + val baseName = getDesiredCollection() ?: return null + val store = useStore ?: return null + + val multiple = downloaded.size > 1 + + return downloaded.mapNotNull { (event, cached) -> + val ext = cached.extension + val source = event.data?.source ?: "unknown" + + // Bestem om vi skal bruke source i navnet + val useSource = multiple || store.using("$baseName.$ext").exists() + + val filename = if (useSource) { + "$baseName-$source.$ext" + } else { + "$baseName.$ext" + } + + val storeFile = store.using(filename).resolveConflict() + + CachedToStore( + cachedFile = cached, + storeFile = storeFile + ) + } + } + + + data class CachedToStore( + val cachedFile: File, + val storeFile: File + ) + + data class CachedToStoreLanguage( + val cts: CachedToStore, + val language: String + ) +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/Project.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/Project.kt deleted file mode 100644 index d4dcf83d..00000000 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/Project.kt +++ /dev/null @@ -1,47 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common.projection - -import no.iktdev.eventi.models.Event -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType -import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcessFlow -import java.io.File - -class Project(val events: List) { - lateinit var startedWith: StartProjection - private set - var metadataTaskStatus: TaskStatus = TaskStatus.NotInitiated - private set - var encodeTaskStatus: TaskStatus = TaskStatus.NotInitiated - private set - var extreactTaskStatus: TaskStatus = TaskStatus.NotInitiated - private set - var convertTaskStatus: TaskStatus = TaskStatus.NotInitiated - private set - var coverDownloadTaskStatus: TaskStatus = TaskStatus.NotInitiated - private set - - - init { - - } - - - - data class StartProjection( - val inputFile: String, - val mode: ProcessFlow, - val tasks: Set - ) - - - - - enum class TaskStatus { - NotInitiated, - NotAvailable, - Pending, - Skipped, - Completed, - Failed - } - -} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/StoreProjection.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/StoreProjection.kt new file mode 100644 index 00000000..fc60f1a7 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/StoreProjection.kt @@ -0,0 +1,66 @@ +package no.iktdev.mediaprocessing.shared.common.projection + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent +import no.iktdev.mediaprocessing.shared.common.model.ContentExport +import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus +import java.io.File + +class StoreProjection(val events: List) { + + + fun projectMetadata(): ContentExport.MetadataExport? { + val metadata = CollectProjection(events).metadata + if (metadata != null) { + val useCover = if (metadata.cover != null) { + val migrated = events.filterIsInstance().lastOrNull { it.status == TaskStatus.Completed }?.coverMigrate ?: emptyList() + migrated.filter { it.status == MigrateStatus.Completed && it.storedUri != null } + .map { File(it.storedUri!!).name } + .find { it == metadata.cover.name } + } else null + + return ContentExport.MetadataExport( + title = metadata.title, + genres = metadata.genres, + cover = useCover, + summary = metadata.summary, + mediaType = metadata.mediaType, + source = metadata.source + ) + } else { + val parsedInfo = events.filterIsInstance().lastOrNull() ?: return null + return ContentExport.MetadataExport( + title = parsedInfo.data.parsedCollection, + mediaType = parsedInfo.data.mediaType, + ) + } + } + + fun projectEpisodeInfo(): ContentExport.EpisodeInfo? { + val episodeInfo = events.filterIsInstance().lastOrNull()?.data?.episodeInfo ?: return null + return ContentExport.EpisodeInfo( + episodeNumber = episodeInfo.episodeNumber, + seasonNumber = episodeInfo.seasonNumber, + episodeTitle = episodeInfo.episodeTitle, + ) + } + + fun projectMediaFiles(): ContentExport.MediaExport? { + val migrated = events.filterIsInstance().lastOrNull { it.status == TaskStatus.Completed } + return ContentExport.MediaExport( + videoFile = migrated?.videoMigrate?.let { video -> + if (video.status == MigrateStatus.Completed) File(video.storedUri!!).name else null + }, + subtitles = migrated?.subtitleMigrate?.filter { it.status == MigrateStatus.Completed } + ?.map { ContentExport.MediaExport.Subtitle(subtitleFile = File(it.storedUri!!).name, language = it.language!!) } ?: emptyList() + ) + } + + fun getCollection(): String? { + val migrated = events.filterIsInstance().lastOrNull { it.status == TaskStatus.Completed } ?: return null + return if (migrated.status == TaskStatus.Completed) migrated.collection else null + } + +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/TaskProjection.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/TaskProjection.kt new file mode 100644 index 00000000..104518eb --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/projection/TaskProjection.kt @@ -0,0 +1,96 @@ +package no.iktdev.mediaprocessing.shared.common.projection + +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.* +import no.iktdev.mediaprocessing.shared.common.getInstancesOf +import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection.TaskStatus +import java.util.* + +class TaskProjection(val events: List) { + + private inline fun projectStatus( + crossinline createdIds: (List) -> List, + crossinline resultStatus: (R) -> no.iktdev.eventi.models.store.TaskStatus, + crossinline resultIds: (List) -> List + ): TaskStatus { + val createdEvent = events.getInstancesOf().ifEmpty { return TaskStatus.NotInitiated } + val resultEvent = events.getInstancesOf().ifEmpty { return TaskStatus.Pending } + + if (resultEvent.size != createdEvent.size) return TaskStatus.Pending + + val created = createdIds(createdEvent) + val results = resultIds(resultEvent) + + if (!created.all { it in results }) return TaskStatus.Pending + if (resultEvent.any { resultStatus(it) == no.iktdev.eventi.models.store.TaskStatus.Failed }) return TaskStatus.Failed + + return TaskStatus.Completed + } + + + fun projectStreamReadStatus() = projectStatus( + createdIds = { it.map { e -> e.taskId } }, + resultStatus = { it.status }, + resultIds = { it.flatMap { e -> e.metadata.derivedFromId?.toList() ?: emptyList() } } + ) + + fun projectCoverDownloadStatus() = projectStatus( + createdIds = { it.flatMap { e -> e.taskIds } }, + resultStatus = { it.status }, + resultIds = { it.flatMap { e -> e.metadata.derivedFromId?.toList() ?: emptyList() } } + ) + + fun projectMetadataSearchStatus() = projectStatus( + createdIds = { it.map { e -> e.taskId } }, + resultStatus = { it.status }, + resultIds = { it.flatMap { e -> e.metadata.derivedFromId?.toList() ?: emptyList() } } + ) + + fun projectEncodingPerformedStatus() = projectStatus( + createdIds = { it.map { e -> e.taskCreated } }, + resultStatus = { it.status }, + resultIds = { it.flatMap { e -> e.metadata.derivedFromId?.toList() ?: emptyList() } } + ) + + fun projectExtractSubtitleStatus(): TaskStatus { + return projectStatus( + createdIds = { it.flatMap { e -> e.tasksCreated } }, + resultStatus = { it.status }, + resultIds = { it.flatMap { e -> e.metadata.derivedFromId?.toList() ?: emptyList() } } + ) + } + + fun projectConvertStatus(): TaskStatus { + val baseStatus = projectStatus( + createdIds = { it.map { e -> e.taskId } }, + resultStatus = { it.status }, + resultIds = { it.flatMap { e -> e.metadata.derivedFromId?.toList() ?: emptyList() } } + ) + + val operations = events + .filterIsInstance() + .lastOrNull() + ?.data?.operation + ?: emptySet() + + val hasExtractAndConvert = operations.contains(OperationType.Extract) && + operations.contains(OperationType.Convert) + + val hasCreatedConvert = events.filterIsInstance().isNotEmpty() + + return when { + // Convert ikke en del av operasjonene → bruk baseStatus direkte + !operations.contains(OperationType.Convert) -> baseStatus + + // Sekvensregel: både Extract og Convert er planlagt, + // men ingen ConvertCreated finnes → Pending + hasExtractAndConvert && !hasCreatedConvert -> TaskStatus.Pending + + // Ellers → baseStatus (Completed, Failed, Pending, NotInitiated) + else -> baseStatus + } + } + + + +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/TaskStore.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/TaskStore.kt index d0f9110f..d6caf8de 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/TaskStore.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/TaskStore.kt @@ -121,10 +121,11 @@ object TaskStore: TaskStore { } } - override fun markConsumed(taskId: UUID) { + override fun markConsumed(taskId: UUID, status: TaskStatus) { withTransaction { TasksTable.update({ TasksTable.taskId eq taskId }) { it[consumed] = true + it[TasksTable.status] = status } } } diff --git a/shared/common/src/main/resources/application.yml b/shared/common/src/main/resources/application.yml index 7ea54f57..bc93c80d 100644 --- a/shared/common/src/main/resources/application.yml +++ b/shared/common/src/main/resources/application.yml @@ -2,7 +2,7 @@ spring: flyway: enabled: true locations: classpath:flyway - baseline-on-migrate: true + baseline-on-migrate: false datasource: url: jdbc:h2:mem:testdb;DB_CLOSE_DELAY=-1 driver-class-name: org.h2.Driver diff --git a/shared/common/src/main/resources/flyway/V1__create_events_table.sql b/shared/common/src/main/resources/flyway/V1__create_events_table.sql index d6e12101..024619d3 100644 --- a/shared/common/src/main/resources/flyway/V1__create_events_table.sql +++ b/shared/common/src/main/resources/flyway/V1__create_events_table.sql @@ -1,9 +1,12 @@ -CREATE TABLE Events ( - id SERIAL PRIMARY KEY, - REFERENCE_ID UUID NOT NULL, - EVENT_ID UUID NOT NULL, - EVENT VARCHAR(100) NOT NULL, - DATA TEXT NOT NULL, - PERSISTED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, +CREATE TABLE EVENTS +( + ID BIGINT NOT NULL AUTO_INCREMENT, + REFERENCE_ID CHAR(36) NOT NULL, + EVENT_ID CHAR(36) NOT NULL, + EVENT VARCHAR(100) NOT NULL, + DATA TEXT NOT NULL, + PERSISTED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (ID), + CONSTRAINT events_unique UNIQUE (REFERENCE_ID, EVENT_ID, EVENT) ); diff --git a/shared/common/src/main/resources/flyway/V2__create_tasks_table.sql b/shared/common/src/main/resources/flyway/V2__create_tasks_table.sql index f1e73b04..fcf3dfc8 100644 --- a/shared/common/src/main/resources/flyway/V2__create_tasks_table.sql +++ b/shared/common/src/main/resources/flyway/V2__create_tasks_table.sql @@ -1,13 +1,15 @@ -CREATE TABLE Tasks ( - id SERIAL PRIMARY KEY, - REFERENCE_ID UUID NOT NULL, - TASK_ID UUID NOT NULL, - TASK VARCHAR(100) NOT NULL, - STATUS VARCHAR(50) NOT NULL, - DATA TEXT NOT NULL, - CLAIMED BOOLEAN NOT NULL DEFAULT FALSE, - CLAIMED_BY VARCHAR(100), - CONSUMED BOOLEAN NOT NULL DEFAULT FALSE, +CREATE TABLE TASKS +( + ID BIGINT NOT NULL AUTO_INCREMENT, + REFERENCE_ID CHAR(36) NOT NULL, + TASK_ID CHAR(36) NOT NULL, + TASK VARCHAR(100) NOT NULL, + STATUS VARCHAR(50) NOT NULL, + DATA TEXT NOT NULL, + CLAIMED BOOLEAN NOT NULL DEFAULT FALSE, + CLAIMED_BY VARCHAR(100), + CONSUMED BOOLEAN NOT NULL DEFAULT FALSE, LAST_CHECK_IN TIMESTAMP, - PERSISTED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + PERSISTED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (ID) ); diff --git a/shared/common/src/main/resources/flyway/V3__create_file_registry_table.sql b/shared/common/src/main/resources/flyway/V3__create_file_registry_table.sql new file mode 100644 index 00000000..8d0d10c2 --- /dev/null +++ b/shared/common/src/main/resources/flyway/V3__create_file_registry_table.sql @@ -0,0 +1,9 @@ +CREATE TABLE FILES +( + ID BIGINT NOT NULL AUTO_INCREMENT, + NAME VARCHAR(255) NOT NULL COMMENT 'Base name of the file including extension', + URI TEXT NOT NULL, + CHECKSUM CHAR(64) UNIQUE NOT NULL COMMENT 'SHA-256 checksum of the file', + IDENTIFIED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (ID) +); \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/FlywayMigrationTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/FlywayMigrationTest.kt deleted file mode 100644 index d5c85224..00000000 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/FlywayMigrationTest.kt +++ /dev/null @@ -1,59 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common - -import mu.KotlinLogging -import no.iktdev.mediaprocessing.shared.common.database.Access -import no.iktdev.mediaprocessing.shared.common.database.DatabaseConfig -import no.iktdev.mediaprocessing.shared.common.database.DbType -import no.iktdev.mediaprocessing.shared.common.database.withTransaction -import org.flywaydb.core.Flyway -import org.jetbrains.exposed.sql.statements.jdbc.JdbcConnectionImpl -import org.jetbrains.exposed.sql.transactions.TransactionManager -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import org.springframework.test.context.junit.jupiter.SpringExtension - -@ExtendWith(SpringExtension::class) -class FlywayMigrationTest { - - private val log = KotlinLogging.logger {} - - - @Test - fun `should run flyway migrations and create expected tables`() { - val access = Access( - username = "sa", - password = "", - address = "", // ikke brukt for H2 - port = 0, // ikke brukt for H2 - databaseName = "testdb", - dbType = DbType.H2 - ) - - val connection = DatabaseConfig.connect(access) - - val flyway = Flyway.configure() - .dataSource(connection.second) - .locations("classpath:flyway") - .baselineOnMigrate(true) - .load() - - // Flyway migrering - flyway.migrate() - - // Verifiser at tabellene finnes - - withTransaction { - val jdbc = (TransactionManager.current().connection as JdbcConnectionImpl).connection - - val meta = jdbc.metaData - val eventsExists = meta.getTables(null, null, "EVENTS", null).next() - val tasksExists = meta.getTables(null, null, "TASKS", null).next() - - assertTrue(eventsExists, "Events table should exist") - assertTrue(tasksExists, "Tasks table should exist") - - log.info { "Found migrations: ${flyway.info().all().map { it.script }}" } - } - } -} diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/TestBase.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/TestBase.kt new file mode 100644 index 00000000..9281df6a --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/TestBase.kt @@ -0,0 +1,35 @@ +package no.iktdev.mediaprocessing.shared.common + +import no.iktdev.mediaprocessing.shared.common.config.DatasourceConfiguration +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.web.client.TestRestTemplate +import org.springframework.boot.test.web.server.LocalServerPort +import org.springframework.boot.web.client.RestTemplateBuilder +import org.springframework.context.annotation.Bean +import org.springframework.test.context.junit.jupiter.SpringExtension +import java.net.URI + +@SpringBootTest( + classes = [DatabaseApplication::class, + DatasourceConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +@ExtendWith(SpringExtension::class) +abstract class TestBase { + + @LocalServerPort + var port: Int = 0 + + @Bean + fun testRestTemplate(): TestRestTemplate { + val baseUrl = URI("http://localhost:$port") + return TestRestTemplate(RestTemplateBuilder().rootUri(baseUrl.toString())) + } + + init { + + } + + +} \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/TestBaseWithDatabase.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/TestBaseWithDatabase.kt new file mode 100644 index 00000000..19e1f29e --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/TestBaseWithDatabase.kt @@ -0,0 +1,81 @@ +package no.iktdev.mediaprocessing.shared.common + +import com.fasterxml.jackson.databind.ObjectMapper +import mu.KotlinLogging +import no.iktdev.mediaprocessing.shared.common.database.Access +import no.iktdev.mediaprocessing.shared.common.database.DatabaseTypes +import no.iktdev.mediaprocessing.shared.common.database.withTransaction +import org.flywaydb.core.Flyway +import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.statements.jdbc.JdbcConnectionImpl +import org.jetbrains.exposed.sql.transactions.TransactionManager +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import org.springframework.beans.factory.annotation.Autowired +import javax.sql.DataSource + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class TestBaseWithDatabase: TestBase() { + val log = KotlinLogging.logger {} + + var validToken: String? = null + val mapper = ObjectMapper() + + + @Autowired + lateinit var dataSource: DataSource + + lateinit var database: Database + private lateinit var flyway: Flyway + + + @BeforeAll + fun setupDatabase() { + val access = Access( + username = "sa", + password = "", + address = "", // ikke brukt for H2 + port = 0, // ikke brukt for H2 + databaseName = "testdb", + dbType = DatabaseTypes.H2 + ) + database = Database.connect(dataSource) + flyway = Flyway.configure() + .dataSource(dataSource) + .locations("classpath:flyway") + .cleanDisabled(false) + .load() + + flyway.clean() + flyway.migrate() + + + withTransaction { + val jdbc = (TransactionManager.current().connection as JdbcConnectionImpl).connection + + val meta = jdbc.metaData + val tableNames = listOf( + "EVENTS", + "TASKS" + ) + val existingTables = tableNames.map { + it to meta.getTables(null, null, it.uppercase(), null).next() + } + + existingTables.forEach { (tableName, exists) -> + assertTrue(exists, "Table $tableName should exist after migration") + } + + log.info { "Found migrations: ${flyway.info().all().map { it.script }}" } + } + } + + @AfterAll + fun clearDatabase() { + flyway.clean() + TransactionManager.closeAndUnregister(database) + } + +} diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/config/DatasourceConfiguration.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/config/DatasourceConfiguration.kt new file mode 100644 index 00000000..44704117 --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/config/DatasourceConfiguration.kt @@ -0,0 +1,40 @@ +package no.iktdev.mediaprocessing.shared.common.config + +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import no.iktdev.mediaprocessing.shared.common.database.Access +import no.iktdev.mediaprocessing.shared.common.database.DatabaseTypes +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Primary +import javax.sql.DataSource + +@TestConfiguration +class DatasourceConfiguration { + @Bean + @Primary + fun dataSource(): DataSource { + val access = Access( + username = "sa", + password = "", + address = "", // ikke brukt for H2 + port = 0, // ikke brukt for H2 + databaseName = "testdb", + dbType = DatabaseTypes.H2 + ) + + val maxPoolSize: Int = 10 + val config = HikariConfig().apply { + this.jdbcUrl = "jdbc:h2:mem:${access.databaseName};MODE=MySQL;DB_CLOSE_DELAY=-1" + this.driverClassName = "org.h2.Driver" + this.username = access.username + this.password = access.password + this.maximumPoolSize = maxPoolSize + this.isAutoCommit = false + this.transactionIsolation = "TRANSACTION_REPEATABLE_READ" + this.validate() + } + + return HikariDataSource(config) + } +} \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ValidateEventsRegistered.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ValidateEventsRegistered.kt new file mode 100644 index 00000000..a604328b --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ValidateEventsRegistered.kt @@ -0,0 +1,44 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import io.github.classgraph.ClassGraph +import io.kotest.assertions.fail +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class ValidateEventsRegistered { + + @Test + fun validateEventsAreRegistered() { + val eventsPackage = "no.iktdev.mediaprocessing.shared.common.event_task_contract.events" + val scanResult = ClassGraph() + .acceptPackages(eventsPackage) + .scan() + + val classesFound = scanResult.allClasses + .map { it.loadClass() } + .filter { Event::class.java.isAssignableFrom(it) } + .toSet() + + val registered = EventRegistry.getEvents().toSet() + + val missing = classesFound - registered + val extra = registered - classesFound + + if (missing.isNotEmpty() || extra.isNotEmpty()) { + fail(buildString { + if (missing.isNotEmpty()) { + appendLine("Mangler i EventRegistry:") + missing.forEach { appendLine(" - ${it.name}") } + } + if (extra.isNotEmpty()) { + appendLine("Registrert men finnes ikke i pakken:") + extra.forEach { appendLine(" - ${it.name}") } + } + }) + } + } + + +} \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ValidateTasksRegistered.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ValidateTasksRegistered.kt new file mode 100644 index 00000000..5916754f --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ValidateTasksRegistered.kt @@ -0,0 +1,46 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import io.github.classgraph.ClassGraph +import io.kotest.assertions.fail +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Task +import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry +import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class ValidateTasksRegistered { + + @Test + fun validateTasksAreRegistered() { + val tasksPackage = "no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks" + val scanResult = ClassGraph() + .acceptPackages(tasksPackage) + .scan() + + val classesFound = scanResult.allClasses + .map { it.loadClass() } + .filter { Task::class.java.isAssignableFrom(it) } + .toSet() + + val registered = TaskRegistry.getTasks().toSet() + + val missing = classesFound - registered + val extra = registered - classesFound + + if (missing.isNotEmpty() || extra.isNotEmpty()) { + fail(buildString { + if (missing.isNotEmpty()) { + appendLine("Mangler i TaskRegistry:") + missing.forEach { appendLine(" - ${it.name}") } + } + if (extra.isNotEmpty()) { + appendLine("Registrert men finnes ikke i pakken:") + extra.forEach { appendLine(" - ${it.name}") } + } + }) + } + } + + +} \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt index 392870d4..66b6284b 100644 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt @@ -1,7 +1,7 @@ package no.iktdev.mediaprocessing.shared.common.parsing import no.iktdev.mediaprocessing.shared.common.model.EpisodeInfo -import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.AssertionsForInterfaceTypes.assertThat import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParserTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParserTest.kt index 2c2e2ad6..75854359 100644 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParserTest.kt +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParserTest.kt @@ -1,6 +1,6 @@ package no.iktdev.mediaprocessing.shared.common.parsing -import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.AssertionsForInterfaceTypes.assertThat import org.junit.jupiter.api.Test import java.io.File diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/projection/ProjectContentStoreTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/projection/ProjectContentStoreTest.kt new file mode 100644 index 00000000..e1491140 --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/projection/ProjectContentStoreTest.kt @@ -0,0 +1,662 @@ +package no.iktdev.mediaprocessing.shared.common.projection + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.exfl.using +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.* +import no.iktdev.mediaprocessing.shared.common.model.MediaType +import org.assertj.core.util.Files +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import java.io.File + +class ProjectContentStoreTest { + + class MockMigrateContentProject(events: List, val folders: List?) : + MigrateContentProject(events, Files.newTemporaryFolder()) { + override fun getFoldersInStore(): List { + return folders ?: emptyList() + } + } + + private fun store(events: List, folders: List) = MockMigrateContentProject(events, folders) + + fun getTempFolder(): File { + return File("/tmp") + } + + + @DisplayName( + """ + Hvis encode-resultatet inneholder cachedOutputFile + Når getVideoStoreFile kalles + Så: + skal video lagres under // + """ + ) + @Test + fun videoPath_isCorrect() { + // Arrange + val temp = getTempFolder().using("store") + + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "MyShow", + parsedFileName = "episode1", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Serie + ) + ) + + val encode = ProcesserEncodeResultEvent( + data = ProcesserEncodeResultEvent.EncodeResult( + cachedOutputFile = "/tmp/cache/videoEncoded.mp4" + ), + status = TaskStatus.Completed + ) + + val events = listOf(parsed, encode) + val store = MigrateContentProject(events, temp) + + // Act + val result = store.getVideoStoreFile() + + // Assert + assertNotNull(result) + assertEquals("episode1.mp4", result!!.storeFile.name) + assertEquals("MyShow", result.storeFile.parentFile.name) + assertEquals(temp, result.storeFile.parentFile.parentFile) + } + + @DisplayName( + """ + Hvis extract- og convert-events inneholder undertekstfiler + Når getSubtitleStoreFiles kalles + Så: + skal filer lagres under /// + """ + ) + @Test + fun subtitlePaths_areCorrect() { + // Arrange + val temp = getTempFolder().using("store") + + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "MyShow", + parsedFileName = "episode1.mkv", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Serie + ) + ) + + val extract = ProcesserExtractResultEvent( + status = TaskStatus.Completed, + data = ProcesserExtractResultEvent.ExtractResult( + language = "eng", + cachedOutputFile = "/tmp/cache/sub1.srt" + ) + ) + + val convert = ConvertTaskResultEvent( + data = ConvertTaskResultEvent.ConvertedData( + language = "eng", + "sub1", + outputFiles = listOf("/tmp/cache/sub1.vtt") + ), + status = TaskStatus.Completed + ) + + val events = listOf(parsed, extract, convert) + val store = MigrateContentProject(events, temp) + + // Act + val results = store.getSubtitleStoreFiles() + + // Assert + assertEquals(2, results?.size) + + results?.forEach { entry -> + assertEquals("eng", entry.cts.storeFile.parentFile.name) + assertEquals("MyShow", entry.cts.storeFile.parentFile.parentFile.name) + assertEquals(temp, entry.cts.storeFile.parentFile.parentFile.parentFile) + } + } + + @DisplayName( + """ + Hvis cover-download-event inneholder en coverfil + Når getCoverStoreFiles kalles + Så: + skal cover lagres under // + """ + ) + @Test + fun coverPath_isCorrect() { + // Arrange + val temp = getTempFolder().using("store") + + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "MyShow", + parsedFileName = "episode1", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Serie + ) + ) + + val cover = CoverDownloadResultEvent( + data = CoverDownloadResultEvent.CoverDownloadedData( + source = "test", + outputFile = "/tmp/cache/cover.jpg" + ), + status = TaskStatus.Completed + ) + + val events = listOf(parsed, cover) + val store = MigrateContentProject(events, temp) + + // Act + val results = store.getCoverStoreFiles() + + // Assert + assertEquals(1, results?.size) + val entry = results?.first() + assertNotNull(entry) + + assertEquals("MyShow.jpg", entry!!.storeFile.name) + assertEquals("MyShow", entry!!.storeFile.parentFile.name) + assertEquals(temp, entry!!.storeFile.parentFile.parentFile) + } + + + // --------------------------------------------------------- + // getDesiredCollection() + // --------------------------------------------------------- + + @DisplayName( + """ + Hvis ingen MediaParsedInfoEvent finnes + Når getDesiredCollection kalles + Så: + returneres null + """ + ) + @Test + fun desiredCollection_none() { + // Arrange + val store = store(events = emptyList(), folders = emptyList()) + + // Act + val result = store.getDesiredCollection() + + // Assert + assertNull(result) + } + + @DisplayName( + """ + Hvis MediaParsedInfoEvent finnes + Når getDesiredCollection kalles + Så: + returneres parsedCollection + """ + ) + @Test + fun desiredCollection_found() { + // Arrange + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "MyCollection", + parsedFileName = "file.mkv", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Movie + ) + ) + + val store = store(events = listOf(parsed), folders = emptyList()) + + // Act + val result = store.getDesiredCollection() + + // Assert + assertEquals("MyCollection", result) + } + + // --------------------------------------------------------- + // getMetadataTitles() + // --------------------------------------------------------- + + @DisplayName( + """ + Hvis ingen MetadataSearchResultEvent finnes + Når getMetadataTitles kalles + Så: + returneres tom liste + """ + ) + @Test + fun metadataTitles_none() { + // Arrange + val store = store(events = emptyList(), folders = emptyList()) + + // Act + val result = store.getMetadataTitles() + + // Assert + assertTrue(result.isEmpty()) + } + + @DisplayName( + """ + Hvis MetadataSearchResultEvent finnes med recommended + Når getMetadataTitles kalles + Så: + returneres alternateTitles + title + """ + ) + @Test + fun metadataTitles_found() { + // Arrange + val metadata = MetadataSearchResultEvent( + results = emptyList(), + recommended = MetadataSearchResultEvent.SearchResult( + simpleScore = 0, + prefixScore = 0, + advancedScore = 0, + sourceWeight = 1f, + data = MetadataSearchResultEvent.SearchResult.MetadataResult( + source = "test", + title = "MainTitle", + alternateTitles = listOf("Alt1", "Alt2"), + cover = "cover.jpg", + bannerImage = null, + type = MediaType.Movie, + summary = emptyList(), + genres = emptyList() + ) + ), + status = TaskStatus.Completed + ) + + val store = store(events = listOf(metadata), folders = emptyList()) + + // Act + val result = store.getMetadataTitles() + + // Assert + assertEquals(listOf("Alt1", "Alt2", "MainTitle"), result) + } + + // --------------------------------------------------------- + // getDesiredStoreFolder() + // --------------------------------------------------------- + + @DisplayName( + """ + Hvis parsedCollection mangler + Når getDesiredStoreFolder kalles + Så: + returneres null + """ + ) + @Test + fun desiredStore_noCollection() { + // Arrange + val store = store(events = emptyList(), folders = emptyList()) + + // Act + val result = store.getDesiredStoreFolder() + + // Assert + assertNull(result) + } + + @DisplayName( + """ + Hvis parsedCollection finnes + Og ingen mapper finnes i store + Når getDesiredStoreFolder kalles + Så: + returneres assuredStore (/) + """ + ) + @Test + fun desiredStore_noFolders() { + // Arrange + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "MyCollection", + parsedFileName = "file.mkv", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Movie + ) + ) + + val store = store(events = listOf(parsed), folders = emptyList()) + + // Act + val result = store.getDesiredStoreFolder() + + // Assert + assertNotNull(result) + assertEquals("MyCollection", result!!.name) + } + + @DisplayName( + """ + Hvis parsedCollection finnes + Og mapper finnes i store + Og metadata-titler matcher en eksisterende mappe + Når getDesiredStoreFolder kalles + Så: + returneres mappen som matcher metadata + """ + ) + @Test + fun desiredStore_matchMetadata() { + // Arrange + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "FallbackCollection", + parsedFileName = "file.mkv", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Movie + ) + ) + + val metadata = MetadataSearchResultEvent( + results = emptyList(), + recommended = MetadataSearchResultEvent.SearchResult( + simpleScore = 0, + prefixScore = 0, + advancedScore = 0, + sourceWeight = 1f, + data = MetadataSearchResultEvent.SearchResult.MetadataResult( + source = "test", + title = "MatchMe", + alternateTitles = listOf("Alt1"), + cover = "cover.jpg", + bannerImage = null, + type = MediaType.Movie, + summary = emptyList(), + genres = emptyList() + ) + ), + status = TaskStatus.Completed + ) + + val store = store( + events = listOf(parsed, metadata), + folders = listOf("MatchMe", "Other") + ) + + // Act + val result = store.getDesiredStoreFolder() + + // Assert + assertNotNull(result) + assertEquals("MatchMe", result!!.name) + } + + @DisplayName( + """ + Hvis parsedCollection finnes + Og mapper finnes i store + Og ingen metadata-titler matcher + Når getDesiredStoreFolder kalles + Så: + returneres assuredStore (parsedCollection) + """ + ) + @Test + fun desiredStore_noMatch() { + // Arrange + val parsed = MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = "FallbackCollection", + parsedFileName = "file.mkv", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Movie + ) + ) + + val metadata = MetadataSearchResultEvent( + results = emptyList(), + recommended = MetadataSearchResultEvent.SearchResult( + simpleScore = 0, + prefixScore = 0, + advancedScore = 0, + sourceWeight = 1f, + data = MetadataSearchResultEvent.SearchResult.MetadataResult( + source = "test", + title = "Unrelated", + alternateTitles = listOf("Alt1"), + cover = "cover.jpg", + bannerImage = null, + type = MediaType.Movie, + summary = emptyList(), + genres = emptyList() + ) + ), + status = TaskStatus.Completed + ) + + val store = store( + events = listOf(parsed, metadata), + folders = listOf("FolderA", "FolderB") + ) + + // Act + val result = store.getDesiredStoreFolder() + + // Assert + assertNotNull(result) + assertEquals("FallbackCollection", result!!.name) + } + + + data class DesiredStoreCase( + val name: String, + val parsedCollection: String?, + val metadataTitles: List, + val existingFolders: List, + val expectedFolder: String? + ) + + @DisplayName(""" + Hvis parsedCollection varierer + Når getDesiredStoreFolder kalles + Så: + skal resultatet følge parsedCollection-reglene + """) + @ParameterizedTest() + @MethodSource("desiredStoreCases") + fun desiredStore_parsedCollectionLogic(case: DesiredStoreCase) { + if (case.parsedCollection == null) { + val store = MockMigrateContentProject(events = emptyList(), folders = case.existingFolders) + assertNull(store.getDesiredStoreFolder()) + return + } + } + + @DisplayName(""" + Hvis metadata-titler finnes + Når getDesiredStoreFolder kalles + Så: + skal riktig matchende mappe velges + """) + @ParameterizedTest() + @MethodSource("desiredStoreCases") + fun desiredStore_metadataMatching(case: DesiredStoreCase) { + if (case.metadataTitles.isEmpty()) return + if (case.parsedCollection == null) return + + val events = mutableListOf() + + events += MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = case.parsedCollection, + parsedFileName = "file.mkv", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Movie + ) + ) + + events += MetadataSearchResultEvent( + results = emptyList(), + recommended = MetadataSearchResultEvent.SearchResult( + simpleScore = 0, + prefixScore = 0, + advancedScore = 0, + sourceWeight = 1f, + data = MetadataSearchResultEvent.SearchResult.MetadataResult( + source = "test", + title = case.metadataTitles.last(), + alternateTitles = case.metadataTitles.dropLast(1), + cover = "cover.jpg", + bannerImage = null, + type = MediaType.Movie, + summary = emptyList(), + genres = emptyList() + ) + ), + status = TaskStatus.Completed + ) + + val store = MockMigrateContentProject(events, case.existingFolders) + + val result = store.getDesiredStoreFolder() + + assertEquals(case.expectedFolder, result?.name) + } + + @DisplayName(""" + Hvis metadata finnes men ingen mapper matcher + Når getDesiredStoreFolder kalles + Så: + skal fallback (parsedCollection) brukes +""") + @ParameterizedTest() + @MethodSource("desiredStoreCases") + fun desiredStore_fallbackLogic(case: DesiredStoreCase) { + if (case.parsedCollection == null) return + if (case.metadataTitles.isEmpty()) return + if (case.expectedFolder == case.metadataTitles.firstOrNull()) return + + val events = mutableListOf() + + events += MediaParsedInfoEvent( + data = MediaParsedInfoEvent.ParsedData( + parsedCollection = case.parsedCollection, + parsedFileName = "file.mkv", + parsedSearchTitles = emptyList(), + mediaType = MediaType.Movie + ) + ) + + events += MetadataSearchResultEvent( + results = emptyList(), + recommended = MetadataSearchResultEvent.SearchResult( + simpleScore = 0, + prefixScore = 0, + advancedScore = 0, + sourceWeight = 1f, + data = MetadataSearchResultEvent.SearchResult.MetadataResult( + source = "test", + title = case.metadataTitles.last(), + alternateTitles = case.metadataTitles.dropLast(1), + cover = "cover.jpg", + bannerImage = null, + type = MediaType.Movie, + summary = emptyList(), + genres = emptyList() + ) + ), + status = TaskStatus.Completed + ) + + val store = MockMigrateContentProject(events, case.existingFolders) + + val result = store.getDesiredStoreFolder() + + assertEquals(case.expectedFolder, result?.name) + } + + + companion object { + + @JvmStatic + fun desiredStoreCases() = listOf( + DesiredStoreCase( + name = "No parsed collection → null", + parsedCollection = null, + metadataTitles = emptyList(), + existingFolders = emptyList(), + expectedFolder = null + ), + DesiredStoreCase( + name = "Parsed collection, no folders → assuredStore", + parsedCollection = "MyShow", + metadataTitles = emptyList(), + existingFolders = emptyList(), + expectedFolder = "MyShow" + ), + DesiredStoreCase( + name = "Metadata matches existing folder", + parsedCollection = "Fallback", + metadataTitles = listOf("MatchMe"), + existingFolders = listOf("MatchMe", "Other"), + expectedFolder = "MatchMe" + ), + DesiredStoreCase( + name = "Metadata alternate title matches", + parsedCollection = "Fallback", + metadataTitles = listOf("Alt1", "MainTitle"), + existingFolders = listOf("Alt1"), + expectedFolder = "Alt1" + ), + DesiredStoreCase( + name = "Multiple metadata titles, first match wins", + parsedCollection = "Fallback", + metadataTitles = listOf("Nope", "YesMatch", "Another"), + existingFolders = listOf("YesMatch", "Another"), + expectedFolder = "YesMatch" + ), + DesiredStoreCase( + name = "Metadata titles exist but none match → fallback", + parsedCollection = "Fallback", + metadataTitles = listOf("A", "B", "C"), + existingFolders = listOf("X", "Y", "Z"), + expectedFolder = "Fallback" + ), + DesiredStoreCase( + name = "Weird folder names (spaces, unicode)", + parsedCollection = "Fallback", + metadataTitles = listOf("ÆØÅ Show"), + existingFolders = listOf("ÆØÅ Show"), + expectedFolder = "ÆØÅ Show" + ), + DesiredStoreCase( + name = "Case-insensitive mismatch → fallback", + parsedCollection = "Fallback", + metadataTitles = listOf("matchme"), + existingFolders = listOf("MatchMe"), + expectedFolder = "Fallback" // system is case-sensitive + ), + DesiredStoreCase( + name = "Existing folders but metadata empty → fallback", + parsedCollection = "Fallback", + metadataTitles = emptyList(), + existingFolders = listOf("A", "B"), + expectedFolder = "Fallback" + ) + ) + } + + + +} \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/projection/TaskProjectionTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/projection/TaskProjectionTest.kt new file mode 100644 index 00000000..268cb1a4 --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/projection/TaskProjectionTest.kt @@ -0,0 +1,90 @@ +package no.iktdev.mediaprocessing.shared.common.projection + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask +import org.assertj.core.api.AssertionsForInterfaceTypes.assertThat +import org.junit.jupiter.api.Test + +class TaskProjectionTest { + + @Test + fun testChainAcknowledgementNotInitiated1() { + val events: MutableList = mutableListOf() + + val projection = TaskProjection(events) + assertThat(projection.projectStreamReadStatus()).isEqualTo(CollectProjection.TaskStatus.NotInitiated) + } + + + @Test + fun testChainAcknowledgementSuccess1() { + val events: MutableList = mutableListOf() + + val readTask = MediaReadTask( + fileUri = "" + ).newReferenceId() + + CoordinatorReadStreamsTaskCreatedEvent( + taskId = readTask.taskId + ).usingReferenceId(readTask.referenceId).also { + events.add(it) + } + + CoordinatorReadStreamsResultEvent( + status = TaskStatus.Completed + ).producedFrom(readTask).also { + events.add(it) + } + + val projection = TaskProjection(events) + assertThat(projection.projectStreamReadStatus()).isEqualTo(CollectProjection.TaskStatus.Completed) + } + + @Test + fun testChainAcknowledgementPending1() { + val events: MutableList = mutableListOf() + + val readTask = MediaReadTask( + fileUri = "" + ).newReferenceId() + + CoordinatorReadStreamsTaskCreatedEvent( + taskId = readTask.taskId + ).usingReferenceId(readTask.referenceId).also { + events.add(it) + } + + val projection = TaskProjection(events) + assertThat(projection.projectStreamReadStatus()).isEqualTo(CollectProjection.TaskStatus.Pending) + } + + @Test + fun testChainAcknowledgementFailure1() { + val events: MutableList = mutableListOf() + + val readTask = MediaReadTask( + fileUri = "" + ).newReferenceId() + + CoordinatorReadStreamsTaskCreatedEvent( + taskId = readTask.taskId + ).usingReferenceId(readTask.referenceId).also { + events.add(it) + } + + CoordinatorReadStreamsResultEvent( + status = TaskStatus.Failed + ).producedFrom(readTask).also { + events.add(it) + } + + val projection = TaskProjection(events) + assertThat(projection.projectStreamReadStatus()).isEqualTo(CollectProjection.TaskStatus.Failed) + } + + + +} \ No newline at end of file diff --git a/shared/common/src/test/resources/application.yml b/shared/common/src/test/resources/application.yml new file mode 100644 index 00000000..3df1331c --- /dev/null +++ b/shared/common/src/test/resources/application.yml @@ -0,0 +1,28 @@ +spring: + main: + allow-bean-definition-overriding: true + flyway: + enabled: false + locations: classpath:flyway + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration + + output: + ansi: + enabled: always + +springdoc: + swagger-ui: + path: /open/swagger-ui + +logging: + level: + org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN + org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping: DEBUG + +management: + endpoints: + web: + exposure: + include: mappings diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFprobe.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFprobe.kt index 02045163..599d6017 100644 --- a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFprobe.kt +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFprobe.kt @@ -7,9 +7,8 @@ import com.google.gson.Gson import com.google.gson.JsonObject import no.iktdev.mediaprocessing.ffmpeg.data.FFinfoOutput -abstract class FFprobe { +abstract class FFprobe(val executable: String) { open val defaultArguments: List = listOf("-v", "quiet") - abstract val executable: String open suspend fun readJsonStreams(inputFile: String): FFinfoOutput { var error: String? = null diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlan.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlan.kt index e126a12f..f3b88cf8 100644 --- a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlan.kt +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlan.kt @@ -58,6 +58,23 @@ data class MediaPlan( return args } + fun toContainer(): String { + val videoCodec = videoTrack.codec + val audioCodecs = audioTracks.map { it.codec } + + return when { + // MP4: H.264/HEVC + AAC/MP3 + (videoCodec is VideoCodec.H264 || videoCodec is VideoCodec.Hevc) && + audioCodecs.all { it is AudioCodec.Aac || it is AudioCodec.Mp3 } -> "mp4" + + // WEBM: VP8/VP9/AV1 + Opus/Vorbis + (videoCodec is VideoCodec.Vp8 || videoCodec is VideoCodec.Vp9 || videoCodec is VideoCodec.Av1) && + audioCodecs.all { it is AudioCodec.Opus || it is AudioCodec.Vorbis } -> "webm" + + // Fallback: MKV (støtter nesten alt) + else -> "mkv" + } + } } // Video target: index + codec diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/SubtitleCodec.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/SubtitleCodec.kt new file mode 100644 index 00000000..8f259c3a --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/SubtitleCodec.kt @@ -0,0 +1,44 @@ +package no.iktdev.mediaprocessing.ffmpeg.dsl + +import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream + +sealed class SubtitleCodec(val codec: String) { + class Srt(): SubtitleCodec("srt") { + override fun getExtension(): String = "srt" + } + + class Vtt(): SubtitleCodec("vtt") { + override fun getExtension(): String = "vtt" + } + class Ass(): SubtitleCodec("ass") { + override fun getExtension(): String = "ass" + } + class Smi(): SubtitleCodec("smi") { + override fun getExtension(): String = "smi" + } + + open fun buildFfmpegArgs(stream: SubtitleStream): List { + return mutableListOf("-c:s", "copy") + } + abstract fun getExtension(): String + + companion object { + /** + * @return null if not supported + */ + fun getCodec(codecName: String): SubtitleCodec? { + return when (codecName) { + // ffmpeg bruker "subrip" for SRT + "srt", "subrip" -> Srt() + // webvtt + "vtt", "webvtt" -> Vtt() + // ass/ssa + "ass", "ssa" -> Ass() + // smi/sami + "smi", "sami" -> Smi() + else -> null + } + } + } +} + diff --git a/shared/ffmpeg/src/test/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlanTest.kt b/shared/ffmpeg/src/test/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlanTest.kt index 81d0d9ae..69c5d461 100644 --- a/shared/ffmpeg/src/test/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlanTest.kt +++ b/shared/ffmpeg/src/test/kotlin/no/iktdev/mediaprocessing/ffmpeg/dsl/MediaPlanTest.kt @@ -262,6 +262,98 @@ class MediaPlanTest { assertEquals(expected, args) } + @Test + @DisplayName(""" + Hvis video=H264 og audio=AAC + Når toContainer kalles + Så: + Returneres "mp4" + """) + fun testChooseContainerMp4() { + val plan = MediaPlan( + videoTrack = VideoTarget(0, VideoCodec.H264()), + audioTracks = mutableListOf(AudioTarget(0, AudioCodec.Aac(channels = 2))) + ) + assertEquals("mp4", plan.toContainer()) + } + + @Test + @DisplayName(""" + Hvis video=VP9 og audio=Opus + Når toContainer kalles + Så: + Returneres "webm" + """) + fun testChooseContainerWebm() { + val plan = MediaPlan( + videoTrack = VideoTarget(0, VideoCodec.Vp9()), + audioTracks = mutableListOf(AudioTarget(0, AudioCodec.Opus())) + ) + assertEquals("webm", plan.toContainer()) + } + + + @Test + @DisplayName(""" + Hvis video=AV1 og audio=FLAC + Når toContainer kalles + Så: + Returneres "mkv" (fallback) + """) + fun testChooseContainerMkv() { + val plan = MediaPlan( + videoTrack = VideoTarget(0, VideoCodec.Av1()), + audioTracks = mutableListOf(AudioTarget(0, AudioCodec.Flac())) + ) + assertEquals("mkv", plan.toContainer()) + } + + + @Test + @DisplayName(""" + Hvis video=HEVC og audio=AAC + Når chooseContainer kalles + Så: + Returneres "mp4" + """) + fun testHevcWithAacGivesMp4() { + val plan = MediaPlan( + videoTrack = VideoTarget(0, VideoCodec.Hevc()), + audioTracks = mutableListOf(AudioTarget(0, AudioCodec.Aac(channels = 2))) + ) + assertEquals("mp4", plan.toContainer()) + } + + @Test + @DisplayName(""" + Hvis video=HEVC og audio=AC3 + Når chooseContainer kalles + Så: + Returneres "mkv" (fallback, siden AC3 ikke støttes i MP4) + """) + fun testHevcWithAc3GivesMkv() { + val plan = MediaPlan( + videoTrack = VideoTarget(0, VideoCodec.Hevc()), + audioTracks = mutableListOf(AudioTarget(0, AudioCodec.Ac3())) + ) + assertEquals("mkv", plan.toContainer()) + } + + @Test + @DisplayName(""" + Hvis video=HEVC og audio=DTS + Når chooseContainer kalles + Så: + Returneres "mkv" (fallback, siden DTS ikke støttes i MP4) + """) + fun testHevcWithDtsGivesMkv() { + val plan = MediaPlan( + videoTrack = VideoTarget(0, VideoCodec.Hevc()), + audioTracks = mutableListOf(AudioTarget(0, AudioCodec.Dts())) + ) + assertEquals("mkv", plan.toContainer()) + } + fun mockVideoStream( index: Int = 0,