diff --git a/.idea/copilot.data.migration.agent.xml b/.idea/copilot.data.migration.agent.xml new file mode 100644 index 00000000..4ea72a91 --- /dev/null +++ b/.idea/copilot.data.migration.agent.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/copilot.data.migration.ask.xml b/.idea/copilot.data.migration.ask.xml new file mode 100644 index 00000000..7ef04e2e --- /dev/null +++ b/.idea/copilot.data.migration.ask.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/copilot.data.migration.ask2agent.xml b/.idea/copilot.data.migration.ask2agent.xml new file mode 100644 index 00000000..1f2ea11e --- /dev/null +++ b/.idea/copilot.data.migration.ask2agent.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/copilot.data.migration.edit.xml b/.idea/copilot.data.migration.edit.xml new file mode 100644 index 00000000..8648f940 --- /dev/null +++ b/.idea/copilot.data.migration.edit.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml index 9b706bb3..b459ecf3 100644 --- a/.idea/gradle.xml +++ b/.idea/gradle.xml @@ -16,7 +16,8 @@ diff --git a/.idea/misc.xml b/.idea/misc.xml index 5cbe6eaf..336886a4 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,5 +1,5 @@ - + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 2f67f95d..8df4529b 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -4,194 +4,86 @@ - @@ -958,7 +869,8 @@ - @@ -1028,16 +940,6 @@ 31 diff --git a/apps/build.gradle.kts b/apps/build.gradle.kts index 6daca501..379611eb 100644 --- a/apps/build.gradle.kts +++ b/apps/build.gradle.kts @@ -20,5 +20,5 @@ tasks.test { useJUnitPlatform() } kotlin { - jvmToolchain(17) + jvmToolchain(21) } \ No newline at end of file diff --git a/apps/converter/build.gradle.kts b/apps/converter/build.gradle.kts index 51e36f39..10ecd66e 100644 --- a/apps/converter/build.gradle.kts +++ b/apps/converter/build.gradle.kts @@ -1,9 +1,8 @@ plugins { id("java") kotlin("jvm") - kotlin("plugin.spring") version "1.5.31" - id("org.springframework.boot") version "2.5.5" - id("io.spring.dependency-management") version "1.0.11.RELEASE" + id("org.springframework.boot") + id("io.spring.dependency-management") } group = "no.iktdev.mediaprocessing.apps" @@ -27,40 +26,32 @@ repositories { } } -val exposedVersion = "0.44.0" dependencies { - /*Spring boot*/ implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter:2.7.0") - // implementation("org.springframework.kafka:spring-kafka:3.0.1") implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") - implementation("org.springframework.kafka:spring-kafka:2.8.5") - implementation("org.jetbrains.exposed:exposed-core:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion") - implementation ("mysql:mysql-connector-java:8.0.29") - implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") implementation("com.google.code.gson:gson:2.8.9") implementation("org.json:json:20210307") implementation("no.iktdev:exfl:0.0.16-SNAPSHOT") implementation("no.iktdev.library:subtitle:1.8.1-SNAPSHOT") + implementation("no.iktdev:eventi:1.0-rc13") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") implementation("com.github.pgreze:kotlin-process:1.4.1") - implementation(project(mapOf("path" to ":shared:eventi"))) implementation(project(mapOf("path" to ":shared:common"))) - implementation(kotlin("stdlib-jdk8")) + + testImplementation("io.mockk:mockk:1.12.0") + testImplementation("org.springframework.boot:spring-boot-starter-test") } tasks.test { @@ -78,5 +69,5 @@ tasks.jar { } kotlin { - jvmToolchain(17) + jvmToolchain(21) } \ No newline at end of file diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt new file mode 100644 index 00000000..29693777 --- /dev/null +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt @@ -0,0 +1,57 @@ +package no.iktdev.mediaprocessing.converter + +import mu.KotlinLogging +import no.iktdev.eventi.events.EventTypeRegistry +import no.iktdev.eventi.tasks.TaskTypeRegistry +import no.iktdev.exfl.coroutines.CoroutinesDefault +import no.iktdev.exfl.coroutines.CoroutinesIO +import no.iktdev.exfl.observable.Observables +import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry +import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry +import no.iktdev.mediaprocessing.shared.common.DatabaseApplication +import no.iktdev.mediaprocessing.shared.common.getAppVersion +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.runApplication +import org.springframework.context.annotation.Configuration + +@SpringBootApplication(scanBasePackages = [ + "no.iktdev.converter", + "no.iktdev.mediaprocessing.shared.common" +]) +open class ConverterApplication: DatabaseApplication() { +} + +val ioCoroutine = CoroutinesIO() +val defaultCoroutine = CoroutinesDefault() + +private val log = KotlinLogging.logger {} + +fun main(args: Array) { + ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener { + override fun onUpdated(value: Throwable) { + value.printStackTrace() + } + }) + defaultCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener { + override fun onUpdated(value: Throwable) { + value.printStackTrace() + } + }) + + + runApplication(*args) + log.info { "App Version: ${getAppVersion()}" } +} +//private val logger = KotlinLogging.logger {} + +@Configuration +open class ApplicationConfiguration() { + init { + EventRegistry.getEvents().let { + EventTypeRegistry.register(it) + } + TaskRegistry.getTasks().let { + TaskTypeRegistry.register(it) + } + } +} diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt new file mode 100755 index 00000000..8cf84efb --- /dev/null +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt @@ -0,0 +1,15 @@ +package no.iktdev.mediaprocessing.converter + +import no.iktdev.exfl.using +import java.io.File + +class ConverterEnv { + companion object { + val allowOverwrite = System.getenv("ALLOW_OVERWRITE").toBoolean() ?: false + val syncDialogs = System.getenv("SYNC_DIALOGS").toBoolean() + val outFormats: List = System.getenv("OUT_FORMATS")?.split(",")?.toList() ?: emptyList() + + val logDirectory = if (!System.getenv("LOG_DIR").isNullOrBlank()) File(System.getenv("LOG_DIR")) else + File("data").using("logs", "convert") + } +} \ No newline at end of file diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt new file mode 100755 index 00000000..8ee527a3 --- /dev/null +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing.converter.convert + +interface ConvertListener { + fun onStarted(inputFile: String) + fun onCompleted(inputFile: String, outputFiles: List) + fun onError(inputFile: String, message: String) {} +} \ No newline at end of file diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt new file mode 100644 index 00000000..5d732cf9 --- /dev/null +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt @@ -0,0 +1,87 @@ +package no.iktdev.mediaprocessing.converter.convert + +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.Data +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.SubtitleFormats +import no.iktdev.library.subtitle.Configuration +import no.iktdev.library.subtitle.Syncro +import no.iktdev.library.subtitle.classes.Dialog +import no.iktdev.library.subtitle.classes.DialogType +import no.iktdev.library.subtitle.export.Export +import no.iktdev.library.subtitle.reader.BaseReader +import no.iktdev.library.subtitle.reader.Reader +import no.iktdev.mediaprocessing.converter.ConverterEnv +import java.io.File +import kotlin.jvm.Throws + +class Converter2(val data: Data, + private val listener: ConvertListener) { + + @Throws(FileUnavailableException::class) + private fun getReader(): BaseReader? { + val file = File(data.inputFile) + if (!file.canRead()) + throw FileUnavailableException("Can't open file for reading..") + return Reader(file).getSubtitleReader() + } + + private fun syncDialogs(input: List): List { + return if (ConverterEnv.syncDialogs) Syncro().sync(input) else input + } + + fun canRead(): Boolean { + try { + val reader = getReader() + return reader != null + } catch (e: FileUnavailableException) { + return false + } + } + + @Throws(FileUnavailableException::class, FileIsNullOrEmpty::class) + fun execute() { + val file = File(data.inputFile) + listener.onStarted(file.absolutePath) + try { + Configuration.exportJson = true + val read = getReader()?.read() ?: throw FileIsNullOrEmpty() + if (read.isEmpty()) + throw FileIsNullOrEmpty() + val filtered = read.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) } + val syncOrNotSync = syncDialogs(filtered) + + val exporter = Export(file, File(data.outputDirectory), data.outputFileName) + + val outFiles = if (data.formats.isEmpty()) { + exporter.write(syncOrNotSync) + } else { + val exported = mutableListOf() + if (data.formats.contains(SubtitleFormats.SRT)) { + exported.add(exporter.writeSrt(syncOrNotSync)) + } + if (data.formats.contains(SubtitleFormats.SMI)) { + exported.add(exporter.writeSmi(syncOrNotSync)) + } + if (data.formats.contains(SubtitleFormats.VTT)) { + exported.add(exporter.writeVtt(syncOrNotSync)) + } + exported + } + result = outFiles.map { it.absolutePath } + listener.onCompleted(file.absolutePath, result!!) + } catch (e: Exception) { + listener.onError(file.absolutePath, e.message ?: e.localizedMessage) + } + } + + private var result: List? = null + fun getResult(): List { + if (result == null) { + throw IllegalStateException("Execute must be called before getting result") + } + return result!! + } + + + class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException() + class FileUnavailableException(override val message: String): RuntimeException() +} \ No newline at end of file diff --git a/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListener.kt b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListener.kt new file mode 100644 index 00000000..5dc45d9c --- /dev/null +++ b/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/listeners/ConvertTaskListener.kt @@ -0,0 +1,68 @@ +package no.iktdev.mediaprocessing.converter.listeners + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.tasks.TaskListener +import no.iktdev.eventi.tasks.TaskType +import no.iktdev.mediaprocessing.converter.convert.ConvertListener +import no.iktdev.mediaprocessing.converter.convert.Converter2 +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskPerformedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertedData +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask +import org.springframework.stereotype.Component +import java.util.* + +@Component +class ConvertTaskListener: TaskListener(TaskType.CPU_INTENSIVE) { + + override fun getWorkerId(): String { + return "${this::class.java.simpleName}-${TaskType.CPU_INTENSIVE}-${UUID.randomUUID()}" + } + + override fun supports(task: Task): Boolean { + return task is ConvertTask + } + + override suspend fun onTask(task: Task): Event? { + if (task !is ConvertTask) { + throw IllegalArgumentException("Invalid task type: ${task::class.java.name}") + } + val converter = Converter2(task.data, object: ConvertListener { + override fun onStarted(inputFile: String) { + } + override fun onCompleted(inputFile: String, outputFiles: List) { + } + }) + + withHeartbeatRunner { + reporter?.updateLastSeen(task.taskId) + } + + converter.execute() + + return try { + val result = converter.getResult() + val newEvent = ConvertTaskPerformedEvent( + data = ConvertedData( + language = task.data.language, + outputFiles = result, + baseName = task.data.storeFileName + ), + status = TaskStatus.Completed + ).producedFrom(task) + newEvent + } catch (e: Exception) { + e.printStackTrace() + val newEvent = ConvertTaskPerformedEvent( + data = null, + status = TaskStatus.Failed + ).producedFrom(task) + newEvent + } + + + } + + +} \ No newline at end of file diff --git a/apps/converter/src/main/resources/application.properties b/apps/converter/src/main/resources/application.properties deleted file mode 100644 index 775a2d6c..00000000 --- a/apps/converter/src/main/resources/application.properties +++ /dev/null @@ -1,5 +0,0 @@ -spring.output.ansi.enabled=always -logging.level.org.apache.kafka=INFO -logging.level.root=INFO -logging.level.Exposed=OFF -logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats = WARN \ No newline at end of file diff --git a/apps/converter/src/main/resources/application.yml b/apps/converter/src/main/resources/application.yml new file mode 100644 index 00000000..fc3d5ccc --- /dev/null +++ b/apps/converter/src/main/resources/application.yml @@ -0,0 +1,21 @@ +spring: + output: + ansi: + enabled: always + flyway: + enabled: true + locations: classpath:flyway + baseline-on-migrate: true + datasource: + url: jdbc:h2:mem:testdb;DB_CLOSE_DELAY=-1 + driver-class-name: org.h2.Driver + username: sa + password: + + +logging: + level: + root: INFO + org.apache.kafka: INFO + Exposed: OFF + org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN diff --git a/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplicationTest.kt b/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplicationTest.kt new file mode 100644 index 00000000..6ab0a024 --- /dev/null +++ b/apps/converter/src/test/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplicationTest.kt @@ -0,0 +1,47 @@ +package no.iktdev.mediaprocessing.converter + +import io.mockk.junit5.MockKExtension +import mu.KotlinLogging +import no.iktdev.eventi.models.Task +import no.iktdev.mediaprocessing.shared.common.stores.TaskStore +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.junit.jupiter.SpringExtension + +@SpringBootTest(classes = [ConverterApplication::class]) +@ExtendWith(SpringExtension::class) +class ConverterApplicationTest { + private val log = KotlinLogging.logger {} + + data class TestTask( + val success: Boolean + ) : Task() + + + @Test + fun `context loads and common configuration is available`() { + // Hvis du har beans du vil verifisere, kan du autowire dem her + // @Autowired lateinit var database: Database + + // Dummy assertion for å verifisere at konteksten starter + assertNotNull(Unit) + } + + + @Test + fun `Verify that we can access TaskStore`() { + val tasks = TaskStore.getPendingTasks() + assertNotNull(tasks) + assert(tasks.isEmpty()) + + TaskStore.persist(TestTask(success = true).newReferenceId()) + + val tasksAfter = TaskStore.getPendingTasks() + assertNotNull(tasksAfter) + assert(tasksAfter.isNotEmpty()) + } +} diff --git a/apps/coordinator/build.gradle.kts b/apps/coordinator/build.gradle.kts index e5008c44..e7f2f17a 100644 --- a/apps/coordinator/build.gradle.kts +++ b/apps/coordinator/build.gradle.kts @@ -1,10 +1,10 @@ plugins { id("java") kotlin("jvm") - kotlin("plugin.spring") version "1.5.31" - id("org.springframework.boot") version "3.2.0" - id("io.spring.dependency-management") version "1.1.4" - id("org.jetbrains.kotlin.plugin.serialization") version "1.5.0" // Legg til Kotlin Serialization-plugin + kotlin("plugin.spring") + id("org.springframework.boot") + id("io.spring.dependency-management") + id("org.jetbrains.kotlin.plugin.serialization") } group = "no.iktdev.mediaprocessing" @@ -44,9 +44,8 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") - //implementation(project(mapOf("path" to ":shared"))) - implementation(project(mapOf("path" to ":shared:eventi"))) + implementation(project(mapOf("path" to ":shared:ffmpeg"))) implementation(project(mapOf("path" to ":shared:common"))) implementation("org.jetbrains.exposed:exposed-core:$exposedVersion") @@ -85,7 +84,7 @@ tasks.withType { } kotlin { - jvmToolchain(17) + jvmToolchain(21) } tasks.bootJar { diff --git a/apps/processer/build.gradle.kts b/apps/processer/build.gradle.kts index 96b7b2a4..d3bcc8d8 100644 --- a/apps/processer/build.gradle.kts +++ b/apps/processer/build.gradle.kts @@ -1,9 +1,9 @@ plugins { id("java") kotlin("jvm") - kotlin("plugin.spring") version "1.5.31" - id("org.springframework.boot") version "2.5.5" - id("io.spring.dependency-management") version "1.0.11.RELEASE" + kotlin("plugin.spring") + id("org.springframework.boot") + id("io.spring.dependency-management") } group = "no.iktdev.mediaprocessing.apps" @@ -49,7 +49,7 @@ dependencies { implementation("com.github.pgreze:kotlin-process:1.4.1") //implementation(project(mapOf("path" to ":shared"))) - implementation(project(mapOf("path" to ":shared:eventi"))) + implementation(project(mapOf("path" to ":shared:ffmpeg"))) implementation(project(mapOf("path" to ":shared:common"))) @@ -83,5 +83,5 @@ tasks.jar { } kotlin { - jvmToolchain(17) + jvmToolchain(21) } \ No newline at end of file diff --git a/apps/ui/build.gradle.kts b/apps/ui/build.gradle.kts index ef0df784..fd9dc04e 100644 --- a/apps/ui/build.gradle.kts +++ b/apps/ui/build.gradle.kts @@ -1,9 +1,9 @@ plugins { id("java") kotlin("jvm") - kotlin("plugin.spring") version "1.5.31" - id("org.springframework.boot") version "2.5.5" - id("io.spring.dependency-management") version "1.0.11.RELEASE" + kotlin("plugin.spring") + id("org.springframework.boot") + id("io.spring.dependency-management") } group = "no.iktdev.mediaprocessing" @@ -44,7 +44,6 @@ dependencies { implementation ("mysql:mysql-connector-java:8.0.29") implementation("no.iktdev:exfl:0.0.16-SNAPSHOT") - implementation(project(mapOf("path" to ":shared:eventi"))) implementation(project(mapOf("path" to ":shared:common"))) @@ -56,7 +55,7 @@ tasks.test { useJUnitPlatform() } kotlin { - jvmToolchain(17) + jvmToolchain(21) } tasks.bootJar { diff --git a/build.gradle.kts b/build.gradle.kts index eb570c89..093a4d40 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,10 @@ plugins { id("java") - kotlin("plugin.spring") version "1.5.31" kotlin("jvm") version "2.1.0" + kotlin("plugin.spring") version "2.1.0" + id("org.jetbrains.kotlin.plugin.serialization") version "2.1.0" + id("org.springframework.boot") version "3.2.2" + id("io.spring.dependency-management") version "1.1.4" } group = "no.iktdev.mediaprocessing" @@ -21,5 +24,5 @@ tasks.test { useJUnitPlatform() } kotlin { - jvmToolchain(17) + jvmToolchain(21) } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index a0e1f018..fd83091e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,7 +10,7 @@ findProject(":apps:processer")?.name = "processer" findProject(":shared")?.name = "shared" -findProject(":shared:eventi")?.name = "eventi" +findProject(":shared:ffmpeg")?.name = "ffmpeg" findProject(":shared:common")?.name = "common" include("apps") @@ -20,5 +20,7 @@ include("apps:converter") include("apps:processer") include("shared") -include("shared:eventi") include("shared:common") + +include("shared:ffmpeg") +include("shared:event-task-contract") \ No newline at end of file diff --git a/shared/build.gradle.kts b/shared/build.gradle.kts index 4347c885..6802aff9 100644 --- a/shared/build.gradle.kts +++ b/shared/build.gradle.kts @@ -45,5 +45,5 @@ tasks.test { useJUnitPlatform() } kotlin { - jvmToolchain(17) + jvmToolchain(21) } \ No newline at end of file diff --git a/shared/common/build.gradle.kts b/shared/common/build.gradle.kts index b8a6220b..0b38247e 100644 --- a/shared/common/build.gradle.kts +++ b/shared/common/build.gradle.kts @@ -1,7 +1,10 @@ plugins { id("java") kotlin("jvm") - id("org.jetbrains.kotlin.plugin.serialization") version "1.5.0" // Legg til Kotlin Serialization-plugin + kotlin("plugin.spring") + id("org.jetbrains.kotlin.plugin.serialization") + id("org.springframework.boot") + id("io.spring.dependency-management") } group = "no.iktdev.mediaprocessing.shared" @@ -11,14 +14,16 @@ repositories { mavenCentral() maven("https://jitpack.io") maven { + name = "ReposiliteReleases" url = uri("https://reposilite.iktdev.no/releases") } maven { + name = "ReposiliteSnapshot" url = uri("https://reposilite.iktdev.no/snapshots") } } -val exposedVersion = "0.44.0" +val exposedVersion = "0.61.0" dependencies { @@ -28,9 +33,11 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") implementation("com.google.code.gson:gson:2.8.9") - implementation("org.json:json:20230227") + implementation("org.json:json:20231013") implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") - implementation("org.springframework.kafka:spring-kafka:2.8.5") + + implementation("org.springframework.boot:spring-boot-starter-data-jpa") + implementation("org.flywaydb:flyway-core") implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.5.0") @@ -38,22 +45,33 @@ dependencies { implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion") implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion") - implementation ("mysql:mysql-connector-java:8.0.29") + + + + + + implementation ("mysql:mysql-connector-java:8.0.33") + implementation("org.postgresql:postgresql:42.7.7") + implementation("org.xerial:sqlite-jdbc:3.43.2.0") + + implementation("org.apache.commons:commons-lang3:3.12.0") + implementation("com.zaxxer:HikariCP:7.0.2") - implementation(project(mapOf("path" to ":shared:eventi"))) - testImplementation(platform("org.junit:junit-bom:5.9.1")) + implementation("no.iktdev:eventi:1.0-rc13") + + testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation("io.mockk:mockk:1.12.0") - testImplementation("com.h2database:h2:1.4.200") - testImplementation("org.assertj:assertj-core:3.4.1") + testImplementation("org.springframework.boot:spring-boot-starter-test") - testImplementation("org.junit.jupiter:junit-jupiter-api:5.7.2") - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.7.2") - testImplementation("io.kotlintest:kotlintest-assertions:3.3.2") + testImplementation("io.mockk:mockk:1.12.0") + implementation("com.h2database:h2:2.2.220") + testImplementation("org.assertj:assertj-core:3.24.2") + + testImplementation("io.kotest:kotest-assertions-core:5.7.2") testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0") } @@ -63,5 +81,5 @@ tasks.test { } kotlin { - jvmToolchain(17) + jvmToolchain(21) } \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseApplication.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseApplication.kt new file mode 100644 index 00000000..6d94e6a2 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseApplication.kt @@ -0,0 +1,15 @@ +package no.iktdev.mediaprocessing.shared.common + +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.runApplication +import org.springframework.context.annotation.ComponentScan + +@SpringBootApplication +@ComponentScan("no.iktdev.mediaprocessing") // sikrer at common beans blir plukket opp +abstract class DatabaseApplication { + companion object { + inline fun launch(args: Array) { + runApplication(*args) + } + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt index d3ef1a3c..1ca8dfec 100644 --- a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Utils.kt @@ -1,7 +1,9 @@ package no.iktdev.mediaprocessing.shared.common +import com.google.gson.GsonBuilder import kotlinx.coroutines.delay import mu.KotlinLogging +import no.iktdev.eventi.ZDS import org.springframework.messaging.simp.SimpMessagingTemplate import org.springframework.web.client.RestTemplate import org.springframework.web.client.postForEntity @@ -10,6 +12,7 @@ import java.io.FileInputStream import java.io.RandomAccessFile import java.net.InetAddress import java.security.MessageDigest +import java.time.LocalDateTime import java.util.zip.CRC32 private val logger = KotlinLogging.logger {} diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfig.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfig.kt new file mode 100644 index 00000000..1fef2b5a --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfig.kt @@ -0,0 +1,51 @@ +package no.iktdev.mediaprocessing.shared.common.database + +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import org.jetbrains.exposed.sql.Database +import javax.sql.DataSource + +object DatabaseConfig { + fun connect( + access: Access, + maxPoolSize: Int = 10 + ): Pair { + val jdbcUrl = when (access.dbType) { + DbType.MySQL -> "jdbc:mysql://${access.address}:${access.port}/${access.databaseName}?useSSL=false&serverTimezone=UTC" + DbType.PostgreSQL -> "jdbc:postgresql://${access.address}:${access.port}/${access.databaseName}" + DbType.SQLite -> "jdbc:sqlite:${access.databaseName}.db" + DbType.H2 -> "jdbc:h2:mem:${access.databaseName};DB_CLOSE_DELAY=-1" + } + + val driver = when (access.dbType) { + DbType.MySQL -> "com.mysql.cj.jdbc.Driver" + DbType.PostgreSQL -> "org.postgresql.Driver" + DbType.SQLite -> "org.sqlite.JDBC" + DbType.H2 -> "org.h2.Driver" + } + + val config = HikariConfig().apply { + this.jdbcUrl = jdbcUrl + this.driverClassName = driver + this.username = access.username + this.password = access.password + this.maximumPoolSize = maxPoolSize + this.isAutoCommit = false + this.transactionIsolation = "TRANSACTION_REPEATABLE_READ" + this.validate() + } + + val dataSource = HikariDataSource(config) + val db = Database.connect(dataSource) + return db to dataSource + } +} + +data class Access( + val username: String, + val password: String, + val address: String, + val port: Int, + val databaseName: String, + val dbType: DbType +) {} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfiguration.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfiguration.kt new file mode 100644 index 00000000..c265793f --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseConfiguration.kt @@ -0,0 +1,68 @@ +package no.iktdev.mediaprocessing.shared.common.database + +import jakarta.annotation.PostConstruct +import org.flywaydb.core.Flyway +import org.jetbrains.exposed.sql.Database +import org.slf4j.LoggerFactory +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.stereotype.Component +import javax.sql.DataSource + +@Configuration +open class DatabaseConfiguration { + + @Bean + fun dataSource(): DataSource { + val access = Access( + username = "sa", + password = "", + address = "", // ikke brukt for H2 + port = 0, // ikke brukt for H2 + databaseName = "testdb", + dbType = DbType.H2 + ) + return DatabaseConfig.connect(access).second + } + +} + +@Configuration +@EnableConfigurationProperties(FlywayProperties::class) +@ConditionalOnProperty(name = ["spring.flyway.enabled"], havingValue = "true", matchIfMissing = true) +class FlywayAutoConfig( + private val dataSource: DataSource, + private val props: FlywayProperties +) { + + private val log = LoggerFactory.getLogger(FlywayAutoConfig::class.java) + + @PostConstruct + fun migrate() { + val locations = props.locations.ifEmpty { listOf("classpath:flyway") } + + val flyway = Flyway.configure() + .dataSource(dataSource) + .locations(*locations.toTypedArray()) + .baselineOnMigrate(true) + .load() + + val pending = flyway.info().pending() + if (pending.isEmpty()) { + log.warn("⚠️ No pending Flyway migrations found in ${locations.joinToString()}") + } else { + log.info("📦 Pending migrations: ${pending.joinToString { it.script }}") + } + + flyway.migrate() + log.info("✅ Flyway migration complete.") + } +} + +@ConfigurationProperties(prefix = "spring.flyway") +data class FlywayProperties( + var locations: List = emptyList() +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseEnv.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseEnv.kt new file mode 100644 index 00000000..a500c913 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseEnv.kt @@ -0,0 +1,31 @@ +package no.iktdev.mediaprocessing.shared.common.database + +object DatabaseEnv { + val address: String? = System.getenv("DATABASE_ADDRESS") + val port: String? = System.getenv("DATABASE_PORT") + val username: String? = System.getenv("DATABASE_USERNAME") + val password: String? = System.getenv("DATABASE_PASSWORD") + val database: String? = System.getenv("DATABASE_NAME") + val databaseType: DbType = DbType.valueOf(System.getenv("DATABASE_TYPE") ?: "MySQL") + + fun toAccess(): Access { + return Access( + username = username ?: "root", + password = password ?: "", + address = address ?: "localhost", + port = port?.toIntOrNull() ?: when (databaseType) { + DbType.MySQL -> 3306 + DbType.PostgreSQL -> 5432 + DbType.SQLite -> 0 + DbType.H2 -> 0 + }, + databaseName = database ?: "mediaprocessing", + dbType = databaseType + ) + } + +} + +enum class DbType { + MySQL, PostgreSQL, SQLite, H2 +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseUtil.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseUtil.kt new file mode 100644 index 00000000..c3e145ea --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/DatabaseUtil.kt @@ -0,0 +1,23 @@ +package no.iktdev.mediaprocessing.shared.common.database + +import org.jetbrains.exposed.sql.transactions.transaction + +fun withTransaction( + rollbackOnFailure: Boolean = false, + run: () -> T +): Result { + return try { + val result = transaction { + try { + run() + } catch (e: Exception) { + if (rollbackOnFailure) rollback() + throw e + } + } + Result.success(result) + } catch (e: Exception) { + e.printStackTrace() + Result.failure(e) + } +} diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/EventsTable.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/EventsTable.kt new file mode 100644 index 00000000..5461d4f6 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/EventsTable.kt @@ -0,0 +1,20 @@ +package no.iktdev.mediaprocessing.shared.common.database.tables + +import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.Column +import org.jetbrains.exposed.sql.javatime.CurrentDateTime +import org.jetbrains.exposed.sql.javatime.datetime +import java.time.LocalDateTime +import java.util.UUID + +object EventsTable: IntIdTable(name = "EVENTS") { + val referenceId: Column = uuid("REFERENCE_ID") + val eventId: Column = uuid("EVENT_ID") + val event: Column = varchar("EVENT",100) + val data: Column = text("DATA") + val persistedAt: Column = datetime("PERSISTED_AT").defaultExpression(CurrentDateTime) + + init { + uniqueIndex(referenceId, eventId, event) + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/TasksTable.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/TasksTable.kt new file mode 100644 index 00000000..2468d325 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/TasksTable.kt @@ -0,0 +1,21 @@ +package no.iktdev.mediaprocessing.shared.common.database.tables + +import no.iktdev.eventi.models.store.TaskStatus +import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.Column +import org.jetbrains.exposed.sql.javatime.CurrentDateTime +import org.jetbrains.exposed.sql.javatime.datetime +import java.util.UUID + +object TasksTable: IntIdTable(name = "TASKS") { + val referenceId: Column = uuid("REFERENCE_ID") + val taskId: Column = uuid("TASK_ID") + val task: Column = varchar("TASK",100) + val status: Column = enumerationByName("STATUS", 50, TaskStatus::class).default(TaskStatus.Pending) + val data: Column = text("DATA") + val claimed: Column = bool("CLAIMED").default(false) + val claimedBy: Column = varchar("CLAIMED_BY",100).nullable() + val consumed: Column = bool("CONSUMED").default(false) + val lastCheckIn: Column = datetime("LAST_CHECK_IN").nullable() + val persistedAt: Column = datetime("PERSISTED_AT").defaultExpression(CurrentDateTime) +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/EventRegistry.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/EventRegistry.kt new file mode 100644 index 00000000..b6363471 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/EventRegistry.kt @@ -0,0 +1,45 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract + +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskCreatedEvents +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskPerformedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.FileAddedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.FileReadyEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.FileRemovedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchTaskCreated +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchTaskPerformed +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodePerformedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserEncodeTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractedPerformedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserReadTaskCreatedEvent +import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent +import no.iktdev.eventi.models.Event + +object EventRegistry { + fun getEvents(): List> { + return listOf( + ConvertTaskCreatedEvents::class.java, + ConvertTaskPerformedEvent::class.java, + + FileAddedEvent::class.java, + FileReadyEvent::class.java, + FileRemovedEvent::class.java, + + MediaParsedInfoEvent::class.java, + + MetadataSearchTaskCreated::class.java, + MetadataSearchTaskPerformed::class.java, + + ProcesserExtractTaskCreatedEvent::class.java, + ProcesserExtractedPerformedEvent::class.java, + + ProcesserEncodeTaskCreatedEvent::class.java, + ProcesserEncodePerformedEvent::class.java, + + ProcesserReadTaskCreatedEvent::class.java, // Do i need this? + + StartProcessingEvent::class.java + ) + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/TaskRegistry.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/TaskRegistry.kt new file mode 100644 index 00000000..8fd07a3c --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/TaskRegistry.kt @@ -0,0 +1,20 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract + +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask +import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MetadataSearchTask +import no.iktdev.eventi.models.Task + +object TaskRegistry { + fun getTasks(): List> { + return listOf( + ConvertTask::class.java, + EncodeTask::class.java, + ExtractTask::class.java, + MediaReadTask::class.java, + MetadataSearchTask::class.java + ) + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/Util.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/Util.kt new file mode 100644 index 00000000..273045b9 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/Util.kt @@ -0,0 +1,11 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract + +import no.iktdev.eventi.models.Event + + +inline fun Event.az(): T? { + return if (this !is T) { + System.err.println("${this::class.java.name} is not a type of ${T::class.java.name}") + null + } else this +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConverterEvents.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConverterEvents.kt new file mode 100644 index 00000000..520494af --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ConverterEvents.kt @@ -0,0 +1,21 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + + +// Placeholder event, so that the listener does not continue to create tasks +class ConvertTaskCreatedEvents: Event() { +} + +data class ConvertTaskPerformedEvent( + val data: ConvertedData?, + val status: TaskStatus, +): Event() { +} + +data class ConvertedData( + val language: String, + val baseName: String, + val outputFiles: List +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadedEvent.kt new file mode 100644 index 00000000..08a99c02 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/CoverDownloadedEvent.kt @@ -0,0 +1,12 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event + +data class CoverDownloadedEvent( + val data: CoverDownloadedData +): Event() { +} + +data class CoverDownloadedData( + val outputFile: String +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/FileEvents.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/FileEvents.kt new file mode 100644 index 00000000..74672523 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/FileEvents.kt @@ -0,0 +1,23 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.DeleteEvent +import no.iktdev.eventi.models.Event + +data class FileAddedEvent( + val data: FileInfo +): Event() { +} + +data class FileReadyEvent( + val data: FileInfo +): Event() {} + +class FileRemovedEvent( + val data: FileInfo +): DeleteEvent() { +} + +data class FileInfo( + val fileName: String, + val fileUri: String, +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaParsedInfoEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaParsedInfoEvent.kt new file mode 100644 index 00000000..bd698991 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaParsedInfoEvent.kt @@ -0,0 +1,16 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event + + +class MediaParsedInfoEvent( + val data: ParsedData +): Event() { +} + +data class ParsedData( + val parsedTitle: String, + val parsedFileName: String, + val parsedSearchTitles: List +) { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaReadyEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaReadyEvent.kt new file mode 100644 index 00000000..0249bca9 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaReadyEvent.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event + +data class MediaReadyEvent( + val fileUri: String +): Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamParsedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamParsedEvent.kt new file mode 100644 index 00000000..a9981418 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamParsedEvent.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.mediaprocessing.shared.common.model.ParsedMediaStreams + +data class MediaStreamParsedEvent( + val data: ParsedMediaStreams +): Event() { +} diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadEvent.kt new file mode 100644 index 00000000..e5db309c --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadEvent.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import com.google.gson.JsonObject +import no.iktdev.eventi.models.Event + +data class MediaStreamReadEvent( + val data: JsonObject +): Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadTaskCreatedEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadTaskCreatedEvent.kt new file mode 100644 index 00000000..df2a3f0b --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MediaStreamReadTaskCreatedEvent.kt @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event + + +class MediaStreamReadTaskCreatedEvent(): Event() { +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataEvents.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataEvents.kt new file mode 100644 index 00000000..3860895d --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/MetadataEvents.kt @@ -0,0 +1,30 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + + +class MetadataSearchTaskCreated(): Event() {} + +class MetadataSearchTaskPerformed( + val data: pyMetadata? = null, + val taskStatus: TaskStatus +): Event() { + init { + assert(taskStatus in listOf(TaskStatus.Completed, TaskStatus.Failed), { "Task status is not of acceptable state $taskStatus" }) + } +} + +data class pyMetadata( + val title: String, + val altTitle: List = emptyList(), + val cover: String? = null, + val type: String, + val summary: List = emptyList(), + val genres: List = emptyList() +) + +data class pySummary( + val summary: String?, + val language: String = "eng" +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEvents.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEvents.kt new file mode 100644 index 00000000..83483747 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/ProcesserEvents.kt @@ -0,0 +1,38 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.TaskStatus + +// Placeholder event, so that the listener does not continue to create tasks +class ProcesserExtractTaskCreatedEvent: Event() { +} + +// Placeholder event, so that the listener does not continue to create tasks +class ProcesserEncodeTaskCreatedEvent: Event() { +} + +// Placeholder event, so that the listener does not continue to create tasks +class ProcesserReadTaskCreatedEvent: Event() { +} + + +data class ProcesserEncodePerformedEvent( + val data: EncodeResult +): Event() { + +} + +data class EncodeResult( + val status: TaskStatus, + val cachedOutputFile: String? = null +) + + +data class ProcesserExtractedPerformedEvent( + val data: ExtractResult +): Event() + +data class ExtractResult( + val status: TaskStatus, + val cachedOutputFile: String? = null +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StartProcessingEvent.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StartProcessingEvent.kt new file mode 100644 index 00000000..b2c06039 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/events/StartProcessingEvent.kt @@ -0,0 +1,26 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.events + +import no.iktdev.eventi.models.Event + +data class StartProcessingEvent( + val data: StartData +): Event() { +} + + +data class StartData( + val operation: Set, + val flow: ProcessFlow = ProcessFlow.Auto, + val fileUri: String, +) + +enum class ProcessFlow { + Auto, + Manual +} + +enum class OperationType { + Extract, + Encode, + Convert +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ConvertTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ConvertTask.kt new file mode 100644 index 00000000..1b2fe746 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ConvertTask.kt @@ -0,0 +1,26 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task + +data class ConvertTask( + val data: Data +): Task() { +} + +data class Data( + val inputFile: String, + val language: String, + val outputDirectory: String, + val outputFileName: String, + val storeFileName: String, + val formats: List = emptyList(), + val allowOverwrite: Boolean +) + + +enum class SubtitleFormats { + ASS, + SRT, + VTT, + SMI +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/CoverDownloadTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/CoverDownloadTask.kt new file mode 100644 index 00000000..b499999d --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/CoverDownloadTask.kt @@ -0,0 +1,10 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task + +data class CoverDownloadTask( + val data: CoverDownloadData +): Task() { +} + +data class CoverDownloadData(val url: String, val outputFile: String) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/EncodeTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/EncodeTask.kt new file mode 100644 index 00000000..34fc1145 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/EncodeTask.kt @@ -0,0 +1,14 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task + +data class EncodeTask( + val data: EncodeData +): Task() { +} + +data class EncodeData( + val arguments: List, + val outputFile: String, + val inputFile: String +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ExtractTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ExtractTask.kt new file mode 100644 index 00000000..3a0187d2 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/ExtractTask.kt @@ -0,0 +1,20 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task + +data class ExtractTask( + val data: ExtractData +): Task() { +} + +data class ExtractData( + val arguments: List, + val outputFileName: String, + val language: String, + val inputFile: String +) { + +} + + + diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MediaReadTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MediaReadTask.kt new file mode 100644 index 00000000..eb46664a --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MediaReadTask.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task + +// Task for reading and parsing media files +class MediaReadTask(val fileUri: String): Task() { +} + diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MetadataSearchTask.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MetadataSearchTask.kt new file mode 100644 index 00000000..4ec96bc6 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/event_task_contract/tasks/MetadataSearchTask.kt @@ -0,0 +1,13 @@ +package no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks + +import no.iktdev.eventi.models.Task + +data class MetadataSearchTask( + val data: MetadataSearchData +): Task() {} + +data class MetadataSearchData( + val searchString: String, + val maxResults: Int = 10, + val offset: Int = 0, +) \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaInfo.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaInfo.kt new file mode 100644 index 00000000..bc5ba986 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaInfo.kt @@ -0,0 +1,35 @@ +package no.iktdev.mediaprocessing.shared.common.model + +import com.google.gson.Gson +import com.google.gson.JsonObject + +data class EpisodeInfo( + override val type: String = "serie", + override val title: String, + val episode: Int, + val season: Int, + val episodeTitle: String?, + override val fullName: String +): MediaInfo(type, title, fullName) + +data class MovieInfo( + override val type: String = "movie", + override val title: String, + override val fullName: String +) : MediaInfo(type, title, fullName) + +data class SubtitleInfo( + val inputFile: String, + val collection: String, + val language: String +) + +open class MediaInfo( + @Transient open val type: String, + @Transient open val title: String, + @Transient open val fullName: String +) { + fun toJsonObject(): JsonObject { + return Gson().toJsonTree(this).asJsonObject + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaStreams.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaStreams.kt new file mode 100755 index 00000000..6b1b504b --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/model/MediaStreams.kt @@ -0,0 +1,191 @@ +package no.iktdev.mediaprocessing.shared.common.model + +data class ParsedMediaStreams( + val videoStream: List = listOf(), + val audioStream: List = listOf(), + val subtitleStream: List = listOf() +) + +data class MediaStreams( + val streams: List +) + +sealed class Stream( + @Transient open val index: Int, + @Transient open val codec_name: String, + @Transient open val codec_long_name: String, + @Transient open val codec_type: String, + @Transient open val codec_tag_string: String, + @Transient open val codec_tag: String, + @Transient open val r_frame_rate: String, + @Transient open val avg_frame_rate: String, + @Transient open val time_base: String, + @Transient open val start_pts: Long, + @Transient open val start_time: String, + @Transient open val duration_ts: Long? = null, + @Transient open val duration: String? = null, + @Transient open val disposition: Disposition? = null, + @Transient open val tags: Tags +) + +data class VideoStream( + override val index: Int, + override val codec_name: String, + override val codec_long_name: String, + override val codec_type: String, + override val codec_tag_string: String, + override val codec_tag: String, + override val r_frame_rate: String, + override val avg_frame_rate: String, + override val time_base: String, + override val start_pts: Long, + override val start_time: String, + override val disposition: Disposition, + override val tags: Tags, + override val duration: String?, + override val duration_ts: Long?, + val profile: String, + val width: Int, + val height: Int, + val coded_width: Int, + val coded_height: Int, + val closed_captions: Int, + val has_b_frames: Int, + val sample_aspect_ratio: String, + val display_aspect_ratio: String, + val pix_fmt: String, + val level: Int, + val color_range: String, + val color_space: String, + val color_transfer: String, + val color_primaries: String, + val chroma_location: String, + val refs: Int +) : Stream( + index, + codec_name, + codec_long_name, + codec_type, + codec_tag_string, + codec_tag, + r_frame_rate, + avg_frame_rate, + time_base, + start_pts, + start_time, + duration_ts, + duration, + disposition, + tags +) + +data class AudioStream( + override val index: Int, + override val codec_name: String, + override val codec_long_name: String, + override val codec_type: String, + override val codec_tag_string: String, + override val codec_tag: String, + override val r_frame_rate: String, + override val avg_frame_rate: String, + override val time_base: String, + override val start_pts: Long, + override val start_time: String, + override val duration: String?, + override val duration_ts: Long?, + override val disposition: Disposition, + override val tags: Tags, + val profile: String, + val sample_fmt: String, + val sample_rate: String, + val channels: Int, + val channel_layout: String, + val bits_per_sample: Int +) : Stream( + index, + codec_name, + codec_long_name, + codec_type, + codec_tag_string, + codec_tag, + r_frame_rate, + avg_frame_rate, + time_base, + start_pts, + start_time, + duration_ts, + duration, + disposition, + tags +) + +data class SubtitleStream( + override val index: Int, + override val codec_name: String, + override val codec_long_name: String, + override val codec_type: String, + override val codec_tag_string: String, + override val codec_tag: String, + override val r_frame_rate: String, + override val avg_frame_rate: String, + override val time_base: String, + override val start_pts: Long, + override val start_time: String, + override val duration: String?, + override val duration_ts: Long?, + override val disposition: Disposition? = null, + override val tags: Tags, + val subtitle_tags: SubtitleTags +) : Stream( + index, + codec_name, + codec_long_name, + codec_type, + codec_tag_string, + codec_tag, + r_frame_rate, + avg_frame_rate, + time_base, + start_pts, + start_time, + duration_ts, + duration, + disposition, + tags +) + +data class Disposition( + val default: Int, + val dub: Int, + val original: Int, + val comment: Int, + val lyrics: Int, + val karaoke: Int, + val forced: Int, + val hearing_impaired: Int, + val captions: Int, + val visual_impaired: Int, + val clean_effects: Int, + val attached_pic: Int, + val timed_thumbnails: Int +) + +data class Tags( + val title: String?, + val BPS: String?, + val DURATION: String?, + val NUMBER_OF_FRAMES: Int? = 0, + val NUMBER_OF_BYTES: String?, + val _STATISTICS_WRITING_APP: String?, + val _STATISTICS_WRITING_DATE_UTC: String?, + val _STATISTICS_TAGS: String?, + val language: String?, + val filename: String?, + val mimetype: String? +) + +data class SubtitleTags( + val language: String?, + val filename: String?, + val mimetype: String? +) diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminate.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminate.kt new file mode 100755 index 00000000..84514695 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminate.kt @@ -0,0 +1,177 @@ +package no.iktdev.mediaprocessing.shared.common.parsing + +import no.iktdev.mediaprocessing.shared.common.model.EpisodeInfo +import no.iktdev.mediaprocessing.shared.common.model.MediaInfo +import no.iktdev.mediaprocessing.shared.common.model.MovieInfo + + +class FileNameDeterminate(val title: String, val sanitizedName: String, val ctype: ContentType = ContentType.UNDEFINED) { + + enum class ContentType { + MOVIE, + SERIE, + UNDEFINED + } + + fun getDeterminedVideoInfo(): MediaInfo? { + return when (ctype) { + ContentType.MOVIE -> determineMovieFileName() + ContentType.SERIE -> determineSerieFileName() + ContentType.UNDEFINED -> determineUndefinedFileName() + } + } + + private fun determineMovieFileName(): MovieInfo? { + val movieEx = MovieEx(title, sanitizedName) + val stripped = when { + movieEx.isDefinedWithYear() -> sanitizedName.replace(movieEx.yearRegex(), "").trim() + movieEx.doesContainMovieKeywords() -> sanitizedName.replace(Regex("(?i)\\s*\\(\\s*movie\\s*\\)\\s*"), "").trim() + else -> sanitizedName + } + val nonResolutioned = movieEx.removeResolutionAndBeyond(stripped) ?: stripped + return MovieInfo(title = cleanup(nonResolutioned), fullName = cleanup(nonResolutioned)) + } + + private fun determineSerieFileName(): EpisodeInfo? { + val serieEx = SerieEx(title, sanitizedName) + + val (season, episode) = serieEx.findSeasonAndEpisode(sanitizedName) + val episodeNumberSingle = serieEx.findEpisodeNumber() + + val seasonNumber = season ?: "1" + val episodeNumber = episode ?: (episodeNumberSingle ?: return null) + val seasonEpisodeCombined = serieEx.getSeasonEpisodeCombined(seasonNumber, episodeNumber) + val episodeTitle = serieEx.findEpisodeTitle() + + val useTitle = if (title == sanitizedName) { + if (title.contains(" - ")) { + title.split(" - ").firstOrNull() ?: title + } else { + val seasonNumberIndex = if (title.indexOf(seasonNumber) < 0) title.length -1 else title.indexOf(seasonNumber) + val episodeNumberIndex = if (title.indexOf(episodeNumber) < 0) title.length -1 else title.indexOf(episodeNumber) + val closest = listOf(seasonNumberIndex, episodeNumberIndex).min() + val shrunkenTitle = title.substring(0, closest) + if (closest - shrunkenTitle.lastIndexOf(" ") < 3) { + title.substring(0, shrunkenTitle.lastIndexOf(" ")) + } else title.substring(0, closest) + + } + } else title + val fullName = "${useTitle.trim()} - $seasonEpisodeCombined ${if (episodeTitle.isNullOrEmpty()) "" else " - $episodeTitle"}".trim() + return EpisodeInfo( + title = title, + episode = episodeNumber.toInt(), + season = seasonNumber.toInt(), + episodeTitle = episodeTitle, + fullName = cleanup(fullName) + ) + } + + private fun determineUndefinedFileName(): MediaInfo? { + val serieEx = SerieEx(title, sanitizedName) + val (season, episode) = serieEx.findSeasonAndEpisode(sanitizedName) + val episodeNumber = serieEx.findEpisodeNumber() + return if ((sanitizedName.contains(" - ") && episodeNumber != null) || season != null || episode != null) { + determineSerieFileName() + } else { + determineMovieFileName() + } + } + + private fun cleanup(input: String): String { + var cleaned = Regex("(?<=\\w)[_.](?=\\w)").replace(input, " ") + cleaned = Regexes.illegalCharacters.replace(cleaned, " - ") + cleaned = Regexes.trimWhiteSpaces.replace(cleaned, " ") + return NameHelper.normalize(cleaned) + } + + open internal class Base(val title: String, val sanitizedName: String) { + fun getMatch(regex: String): String? { + return Regex(regex, RegexOption.IGNORE_CASE).find(sanitizedName)?.value + } + + fun removeResolutionAndBeyond(input: String): String? { + val removalValue = Regex("(i?)([0-9].*[pk]|[ ._-]+[UHD]+[ ._-])").find(input)?.value ?: return null + return input.substring(0, input.indexOf(removalValue)) + } + + fun yearRegex(): Regex { + return Regex("[ .(][0-9]{4}[ .)]") + } + } + + internal class MovieEx(title: String, sanitizedName: String) : Base(title, sanitizedName) { + /** + * @return not null if matches " 2020 " or ".2020." + */ + fun isDefinedWithYear(): Boolean { + return getMatch(yearRegex().pattern)?.isNotBlank() ?: false + } + + /** + * Checks whether the filename contains the keyword movie, if so, default to movie + */ + fun doesContainMovieKeywords(): Boolean { + return getMatch("[(](?<=\\()movie(?=\\))[)]")?.isNotBlank() ?: false + } + } + + internal class SerieEx(title: String, sanitizedName: String) : Base(title, sanitizedName) { + + fun getSeasonEpisodeCombined(season: String, episode: String): String { + return StringBuilder() + .append("S") + .append(if (season.length < 2) season.padStart(2, '0') else season) + .append("E") + .append(if (episode.length < 2) episode.padStart(2, '0') else episode) + .toString().trim() + } + + + /** + * Sjekken matcher tekst som dette: + * Cool - Season 1 Episode 13 + * Cool - s1e13 + * Cool - S1E13 + * Cool - S1 13 + */ + fun findSeasonAndEpisode(inputText: String): Pair { + val matchResult = Regexes.SeasonEpisodeBlock.find(inputText) + val season = matchResult?.groups?.get(1)?.value + val episode = matchResult?.groups?.get(2)?.value + return season to episode + } + + fun findEpisodeNumber(): String? { + val regex = Regex("\\b(\\d+)\\b") + val matchResult = regex.findAll(sanitizedName) + val usabeNumber = if (matchResult.toList().size > 1) { + Regex("[-_] \\b(\\d+)\\b").find(sanitizedName)?.groups?.lastOrNull()?.value + } else { + matchResult.lastOrNull()?.value + } + return usabeNumber?.trim() + } + + fun findEpisodeTitle(): String? { + var startPosition: Int = 0 + startPosition = Regexes.SeasonEpisodeBlock.find(sanitizedName)?.value?.let { block -> + sanitizedName.indexOf(block) + block.length + } ?: 0 + + val seCombo = findSeasonAndEpisode(sanitizedName) + val episodeNumber = findEpisodeNumber() + + startPosition = if (startPosition != 0) startPosition else if (seCombo.second != null) sanitizedName.indexOf(seCombo.second!!)+ seCombo.second!!.length + else if (episodeNumber != null) sanitizedName.indexOf(episodeNumber) + episodeNumber.length else 0 + val availableText = sanitizedName.substring(startPosition) + + val cleanedEpisodeTitle = availableText.replace(Regex("""(?i)\b(?:season|episode|ep)\b"""), "") + .replace(Regex("""^\s*-\s*"""), "") + .replace(Regex("""\s+"""), " ") + .trim() + + return cleanedEpisodeTitle + } + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParser.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParser.kt new file mode 100755 index 00000000..2dd5c5da --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParser.kt @@ -0,0 +1,146 @@ +package no.iktdev.mediaprocessing.shared.common.parsing + + +class FileNameParser(val fileName: String) { + var cleanedFileName: String + private set + + init { + cleanedFileName = removeBracketedText(fileName) + cleanedFileName = removeParenthesizedText(cleanedFileName) + cleanedFileName = removeResolutionAndTrailing(cleanedFileName) + cleanedFileName = removeResolutionAndTags(cleanedFileName) + cleanedFileName = removeParenthesizedText(cleanedFileName) + cleanedFileName = removeYear(cleanedFileName) + cleanedFileName = removeDot(cleanedFileName) + cleanedFileName = removeExtraWhiteSpace(cleanedFileName) + cleanedFileName = removeTrailingAndLeadingCharacters(cleanedFileName).trim() + } + + fun guessDesiredFileName(): String { + val parts = cleanedFileName.split(" - ") + return when { + parts.size == 2 && parts[1].matches(Regex("\\d{4}")) -> { + val title = parts[0] + val year = parts[1] + "$title ($year)" + } + + parts.size >= 3 && parts[1].matches(Regex("S\\d+")) && parts[2].matches(Regex("\\d+[vV]\\d+")) -> { + val title = parts[0] + val episodeWithRevision = parts[2] + val episodeParts = episodeWithRevision.split("v", "V") + val episodeNumber = episodeParts[0].toInt() + val revisionNumber = episodeParts[1].toInt() + val seasonEpisode = + "S${episodeNumber.toString().padStart(2, '0')}E${revisionNumber.toString().padStart(2, '0')}" + val episodeTitle = if (parts.size > 3) parts[3] else "" + "$title - $seasonEpisode - $episodeTitle" + } + + else -> cleanedFileName + }.trim() + .replace(Regex("[-\\s]+$"), "") // fjern trailing "-" og whitespace + } + + fun guessDesiredTitle(): String { + val desiredFileName = guessDesiredFileName() + return if (Regexes.season.containsMatchIn(desiredFileName)) { + Regexes.season.split(desiredFileName).firstOrNull()?.trim() ?: desiredFileName + } else { + val result = if (desiredFileName.contains(" - ")) { + desiredFileName.split(" - ").firstOrNull() ?: desiredFileName + } else desiredFileName + result.trim() + }.trim('.', '-').trim() + .replace(Regex("[-\\s]+$"), "") // fjern trailing "-" og whitespace + } + + fun guessSearchableTitle(): MutableList { + var cleaned = removeBracketedText(fileName) + cleaned = keepParanthesesWithYear(cleaned) + cleaned = removeResolutionAndTrailing(cleaned) + cleaned = removeResolutionAndTags(cleaned) + cleaned = removeDot(cleaned) + + val titles = mutableListOf() + var ch = cleaned.split('-').firstOrNull() ?: cleaned + ch = removeExtraWhiteSpace(ch) + ch = ch.trim('.', ',', ' ') + titles.add(ch) + + return titles + } + + + /** + * Modifies the input value and removes "[Text]" + * @param text "[TEST] Dummy - 01 [AZ 1080p] " + */ + fun removeBracketedText(text: String): String { + return Regex("\\[.*?]").replace(text, " ") + } + + /** + * + */ + fun removeParenthesizedText(text: String): String { + return Regex("\\(.*?\\)").replace(text, " ") + } + + fun removeResolutionAndTrailing(text: String): String { + return Regex("[0-9]+[pP].*").replace(text, "") + } + + fun removeTrailingAndLeadingCharacters(text: String): String { + return Regex("^[^a-zA-Z0-9!,]+|[^a-zA-Z0-9!~,]+\$").replace(text, " ") + } + + /** + * + */ + fun removeResolutionAndTags(input: String): String { + var text = Regex("(?i)(\\d+[pk]\\b|(hd|uhd))", RegexOption.IGNORE_CASE).replace(input, " ") + text = Regex("(?i)(bluray|laserdisc|dvd|web)", RegexOption.IGNORE_CASE).replace(text, " ") + return text + } + + + fun keepParanthesesWithYear(text: String): String { + val regex = "\\((?!\\d{4}\\))(?>[^()]+|\\b)\\)" + return Regex(regex).replace(text, "") + } + + fun removeYear(text: String): String { + val match = Regex("\\b\\d{4}\\W").find(text, 0)?.value + if (match == null || text.indexOf(match) > 0) { + //return Regex("\\b\\d{4}\\b(.*)").replace(text, " ") + return Regex("\\b\\d{4}\\b").replace(text, "") + } + return text + } + + fun removeDot(input: String): String { + //var text = Regex("(?<=\\s)\\.|\\.(?=\\s)").replace(input, "") + //return Regex("\\.(?|!`()\[\]]""") + val trimWhiteSpaces = Regex("\\s{2,}") + val SeasonEpisodeBlock = Regex("""(?i)\b(?:S|Season)\s*(\d+).*?(?:E|Episode)?\s*(\d+)\b""", RegexOption.IGNORE_CASE) + val season = Regex("""(?i)\b(?:S|Season)\s*-?\s*(\d+)""", RegexOption.IGNORE_CASE) +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/EventStore.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/EventStore.kt new file mode 100644 index 00000000..6acf72b4 --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/EventStore.kt @@ -0,0 +1,67 @@ +package no.iktdev.mediaprocessing.shared.common.stores + +import com.google.gson.Gson +import no.iktdev.eventi.ZDS.toPersisted +import no.iktdev.eventi.models.Event +import no.iktdev.eventi.models.store.PersistedEvent +import no.iktdev.eventi.stores.EventStore +import no.iktdev.mediaprocessing.shared.common.database.tables.EventsTable +import no.iktdev.mediaprocessing.shared.common.database.withTransaction +import org.jetbrains.exposed.sql.insert +import org.jetbrains.exposed.sql.selectAll +import java.time.LocalDateTime +import java.util.UUID + +object EventStore: EventStore { + override fun getPersistedEventsAfter(timestamp: LocalDateTime): List { + val result = withTransaction { + EventsTable.selectAll() + .where { EventsTable.persistedAt greater timestamp } + .map { + PersistedEvent( + id = it[EventsTable.id].value.toLong(), + referenceId = it[EventsTable.referenceId], + eventId = it[EventsTable.eventId], + event = "", // You might want to store the event type as well + data = it[EventsTable.data], + persistedAt = it[EventsTable.persistedAt] + ) + } + } + return result.getOrDefault(emptyList()) + } + + override fun getPersistedEventsFor(referenceId: UUID): List { + val result = withTransaction { + EventsTable.selectAll() + .where { EventsTable.referenceId eq referenceId} + .map { + PersistedEvent( + id = it[EventsTable.id].value.toLong(), + referenceId = it[EventsTable.referenceId], + eventId = it[EventsTable.eventId], + event = "", // You might want to store the event type as well + data = it[EventsTable.data], + persistedAt = it[EventsTable.persistedAt] + ) + } + } + return result.getOrDefault(emptyList()) + } + + override fun persist(event: Event) { + val asData = Gson().toJson(event) + val eventName = event::class.simpleName ?: run { + throw RuntimeException("Missing class name for event: $event") + } + withTransaction { + EventsTable.insert { + it[EventsTable.referenceId] = event.referenceId + it[EventsTable.eventId] = event.eventId + it[EventsTable.event] = eventName + it[EventsTable.data] = asData + it[EventsTable.persistedAt] = LocalDateTime.now() + } + } + } +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/TaskStore.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/TaskStore.kt new file mode 100644 index 00000000..d0f9110f --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/stores/TaskStore.kt @@ -0,0 +1,170 @@ +package no.iktdev.mediaprocessing.shared.common.stores + +import com.google.gson.Gson +import no.iktdev.eventi.ZDS +import no.iktdev.eventi.models.Task +import no.iktdev.eventi.models.store.PersistedTask +import no.iktdev.eventi.models.store.TaskStatus +import no.iktdev.eventi.stores.TaskStore +import no.iktdev.mediaprocessing.shared.common.database.tables.TasksTable +import no.iktdev.mediaprocessing.shared.common.database.withTransaction +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.insert +import org.jetbrains.exposed.sql.selectAll +import org.jetbrains.exposed.sql.update +import java.time.Duration +import java.time.LocalDateTime +import java.util.UUID + +object TaskStore: TaskStore { + override fun persist(task: Task) { + val asData = ZDS.WGson.toJson(task) + val taskName = task::class.simpleName ?: run { + throw RuntimeException("Missing class name for task: $task") + } + withTransaction { + TasksTable.insert { + it[referenceId] = task.referenceId + it[taskId] = task.taskId + it[TasksTable.task] = taskName + it[status] = TaskStatus.Pending + it[data] = asData + it[persistedAt] = LocalDateTime.now() + } + } + } + + override fun findByTaskId(taskId: UUID): PersistedTask? { + return withTransaction { + TasksTable.selectAll() + .where { TasksTable.taskId eq taskId } + .singleOrNull()?.let { + PersistedTask( + id = it[TasksTable.id].value.toLong(), + referenceId = it[TasksTable.referenceId], + status = it[TasksTable.status], + taskId = it[TasksTable.taskId], + task = it[TasksTable.task], + data = it[TasksTable.data], + claimed = it[TasksTable.claimed], + claimedBy = it[TasksTable.claimedBy], + consumed = it[TasksTable.consumed], + lastCheckIn = it[TasksTable.lastCheckIn], + persistedAt = it[TasksTable.persistedAt] + ) + } + }.getOrNull() + } + + override fun findByReferenceId(referenceId: UUID): List { + return withTransaction { + TasksTable.selectAll() + .where { TasksTable.referenceId eq referenceId } + .map { + PersistedTask( + id = it[TasksTable.id].value.toLong(), + referenceId = it[TasksTable.referenceId], + status = it[TasksTable.status], + taskId = it[TasksTable.taskId], + task = it[TasksTable.task], + data = it[TasksTable.data], + claimed = it[TasksTable.claimed], + claimedBy = it[TasksTable.claimedBy], + consumed = it[TasksTable.consumed], + lastCheckIn = it[TasksTable.lastCheckIn], + persistedAt = it[TasksTable.persistedAt] + ) + } + }.getOrDefault(emptyList()) + } + + override fun findUnclaimed(referenceId: UUID): List { + return withTransaction { + TasksTable.selectAll() + .where { (TasksTable.referenceId eq referenceId) and (TasksTable.claimed eq false) and (TasksTable.consumed eq false) } + .map { + PersistedTask( + id = it[TasksTable.id].value.toLong(), + referenceId = it[TasksTable.referenceId], + status = it[TasksTable.status], + taskId = it[TasksTable.taskId], + task = it[TasksTable.task], + data = it[TasksTable.data], + claimed = it[TasksTable.claimed], + claimedBy = it[TasksTable.claimedBy], + consumed = it[TasksTable.consumed], + lastCheckIn = it[TasksTable.lastCheckIn], + persistedAt = it[TasksTable.persistedAt] + ) + } + }.getOrDefault(emptyList()) + } + + override fun claim(taskId: UUID, workerId: String): Boolean { + return withTransaction { + TasksTable.update({ + (TasksTable.taskId eq taskId) and + (TasksTable.claimed eq false) + }) { + it[claimed] = true + it[claimedBy] = workerId + it[lastCheckIn] = LocalDateTime.now() + } + }.isSuccess + } + + override fun heartbeat(taskId: UUID) { + withTransaction { + TasksTable.update({ TasksTable.taskId eq taskId }) { + it[lastCheckIn] = LocalDateTime.now() + } + } + } + + override fun markConsumed(taskId: UUID) { + withTransaction { + TasksTable.update({ TasksTable.taskId eq taskId }) { + it[consumed] = true + } + } + } + + override fun releaseExpiredTasks(timeout: Duration) { + val now = LocalDateTime.now() + val expirationTime = now.minus(timeout) + withTransaction { + TasksTable.update({ + (TasksTable.claimed eq true) and + (TasksTable.consumed eq false) and + (TasksTable.lastCheckIn.isNotNull()) and + (TasksTable.lastCheckIn less expirationTime) + }) { + it[claimed] = false + it[claimedBy] = null + it[lastCheckIn] = null + } + } + } + + override fun getPendingTasks(): List { + return withTransaction { + TasksTable.selectAll() + .where { (TasksTable.consumed eq false) and (TasksTable.claimed eq false) } + .map { + PersistedTask( + id = it[TasksTable.id].value.toLong(), + referenceId = it[TasksTable.referenceId], + status = it[TasksTable.status], + taskId = it[TasksTable.taskId], + task = it[TasksTable.task], + data = it[TasksTable.data], + claimed = it[TasksTable.claimed], + claimedBy = it[TasksTable.claimedBy], + consumed = it[TasksTable.consumed], + lastCheckIn = it[TasksTable.lastCheckIn], + persistedAt = it[TasksTable.persistedAt] + ) + } + }.getOrDefault(emptyList()) + } +} \ No newline at end of file diff --git a/shared/common/src/main/resources/application.yml b/shared/common/src/main/resources/application.yml new file mode 100644 index 00000000..7ea54f57 --- /dev/null +++ b/shared/common/src/main/resources/application.yml @@ -0,0 +1,10 @@ +spring: + flyway: + enabled: true + locations: classpath:flyway + baseline-on-migrate: true + datasource: + url: jdbc:h2:mem:testdb;DB_CLOSE_DELAY=-1 + driver-class-name: org.h2.Driver + username: sa + password: diff --git a/shared/common/src/main/resources/flyway/V1__create_events_table.sql b/shared/common/src/main/resources/flyway/V1__create_events_table.sql new file mode 100644 index 00000000..d6e12101 --- /dev/null +++ b/shared/common/src/main/resources/flyway/V1__create_events_table.sql @@ -0,0 +1,9 @@ +CREATE TABLE Events ( + id SERIAL PRIMARY KEY, + REFERENCE_ID UUID NOT NULL, + EVENT_ID UUID NOT NULL, + EVENT VARCHAR(100) NOT NULL, + DATA TEXT NOT NULL, + PERSISTED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT events_unique UNIQUE (REFERENCE_ID, EVENT_ID, EVENT) +); diff --git a/shared/common/src/main/resources/flyway/V2__create_tasks_table.sql b/shared/common/src/main/resources/flyway/V2__create_tasks_table.sql new file mode 100644 index 00000000..f1e73b04 --- /dev/null +++ b/shared/common/src/main/resources/flyway/V2__create_tasks_table.sql @@ -0,0 +1,13 @@ +CREATE TABLE Tasks ( + id SERIAL PRIMARY KEY, + REFERENCE_ID UUID NOT NULL, + TASK_ID UUID NOT NULL, + TASK VARCHAR(100) NOT NULL, + STATUS VARCHAR(50) NOT NULL, + DATA TEXT NOT NULL, + CLAIMED BOOLEAN NOT NULL DEFAULT FALSE, + CLAIMED_BY VARCHAR(100), + CONSUMED BOOLEAN NOT NULL DEFAULT FALSE, + LAST_CHECK_IN TIMESTAMP, + PERSISTED_AT TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseDeserializerTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseDeserializerTest.kt deleted file mode 100644 index bc9c1d5c..00000000 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/DatabaseDeserializerTest.kt +++ /dev/null @@ -1,100 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common - -import no.iktdev.mediaprocessing.shared.common.contract.Events -import no.iktdev.mediaprocessing.shared.common.contract.data.EpisodeInfo -import no.iktdev.mediaprocessing.shared.common.contract.data.MediaMetadataReceivedEvent -import no.iktdev.mediaprocessing.shared.common.contract.data.MediaOutInformationConstructedEvent -import no.iktdev.mediaprocessing.shared.common.contract.data.StartEventData -import no.iktdev.mediaprocessing.shared.common.contract.jsonToEvent -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test - -class DatabaseDeserializerTest { - - @Test - fun validateParsingOfStartEvent() { - //language=json - val data = """{"metadata":{"eventId":"45e92856-37a6-4266-81cd-a8cf16dd7eb2","referenceId":"a42bf0a4-d31d-4715-8e4d-1432d52f9786","status":"Success","created":"2025-02-10T21:35:52.437791666","source":"Coordinator"},"data":{"type":"FLOW","operations":["ENCODE","EXTRACT","CONVERT"],"file":"/potato"},"eventType":"EventMediaProcessStarted"}""" - val result = data.jsonToEvent("event:media-process:started") - assertThat(result.data!!.javaClass).hasSameClassAs(StartEventData::class.java) - assertThat(result.eventType).isNotNull() - assertThat(result.eventType).isEqualTo(Events.ProcessStarted) - } - - @Test - fun validateMediaInfo() { - //language=json - val data = """ - { - "metadata": { - "derivedFromEventId": "c2ec1424-3d8f-444c-ad85-ce04e6c583fd", - "eventId": "0b164f37-fa23-4b43-a31e-edfa56325eb2", - "referenceId": "920e67cc-1f07-43f0-b121-e5ae87195122", - "status": "Success", - "created": "2025-02-24T00:39:16.057643776", - "source": "MediaOutInformationTaskListener" - }, - "eventType": "ReadOutNameAndType", - "data": { - "info": { - "type": "serie", - "title": "The Potato", - "episode": 1, - "season": 2, - "episodeTitle": "", - "fullName": "The Bit potato" - } - } - } - """.trimIndent() - val result = data.jsonToEvent("event:media-read-out-name-and-type:performed") - assertThat(result.data!!.javaClass).hasSameClassAs(MediaOutInformationConstructedEvent::class.java) - assertThat(result.eventType).isNotNull() - val serieInfo = (result as MediaOutInformationConstructedEvent).data?.toValueObject() - assertThat(serieInfo).isNotNull() - assertThat(serieInfo!!.javaClass).hasSameClassAs(EpisodeInfo::class.java) - } - - @Test - fun validateMetadataRead() { - //language=json - val data = """ - { - "metadata": { - "derivedFromEventId": "855b6de0-38f1-4ac9-9397-1ca7fc83fa4d", - "eventId": "c2ec1424-3d8f-444c-ad85-ce04e6c583fd", - "referenceId": "920e67cc-1f07-43f0-b121-e5ae87195122", - "status": "Success", - "created": "2025-02-24T00:39:15.674278", - "source": "metadataApp" - }, - "eventType": "EventMediaMetadataSearchPerformed", - "data": { - "title": "Cabbage", - "altTitle": [ - "Cabbage man" - ], - "cover": "https://cabbageman.co", - "banner": null, - "type": "serie", - "summary": [ - { - "summary": "Forced to becoma a cabbage farmer after getting their passport confiscated", - "language": "eng" - } - ], - "genres": [ - "Drama", - "Mystery" - ], - "source": "yt" - } - } - """.trimIndent() - val result = data.jsonToEvent("event:media-metadata-search:performed") - assertThat(result.data!!.javaClass).hasSameClassAs(MediaMetadataReceivedEvent::class.java) - assertThat(result.eventType).isNotNull() - assertThat(result.data).isNotNull() - } - -} \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/FlywayMigrationTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/FlywayMigrationTest.kt new file mode 100644 index 00000000..d5c85224 --- /dev/null +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/FlywayMigrationTest.kt @@ -0,0 +1,59 @@ +package no.iktdev.mediaprocessing.shared.common + +import mu.KotlinLogging +import no.iktdev.mediaprocessing.shared.common.database.Access +import no.iktdev.mediaprocessing.shared.common.database.DatabaseConfig +import no.iktdev.mediaprocessing.shared.common.database.DbType +import no.iktdev.mediaprocessing.shared.common.database.withTransaction +import org.flywaydb.core.Flyway +import org.jetbrains.exposed.sql.statements.jdbc.JdbcConnectionImpl +import org.jetbrains.exposed.sql.transactions.TransactionManager +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.test.context.junit.jupiter.SpringExtension + +@ExtendWith(SpringExtension::class) +class FlywayMigrationTest { + + private val log = KotlinLogging.logger {} + + + @Test + fun `should run flyway migrations and create expected tables`() { + val access = Access( + username = "sa", + password = "", + address = "", // ikke brukt for H2 + port = 0, // ikke brukt for H2 + databaseName = "testdb", + dbType = DbType.H2 + ) + + val connection = DatabaseConfig.connect(access) + + val flyway = Flyway.configure() + .dataSource(connection.second) + .locations("classpath:flyway") + .baselineOnMigrate(true) + .load() + + // Flyway migrering + flyway.migrate() + + // Verifiser at tabellene finnes + + withTransaction { + val jdbc = (TransactionManager.current().connection as JdbcConnectionImpl).connection + + val meta = jdbc.metaData + val eventsExists = meta.getTables(null, null, "EVENTS", null).next() + val tasksExists = meta.getTables(null, null, "TASKS", null).next() + + assertTrue(eventsExists, "Events table should exist") + assertTrue(tasksExists, "Tasks table should exist") + + log.info { "Found migrations: ${flyway.info().all().map { it.script }}" } + } + } +} diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/H2DataSource.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/H2DataSource.kt deleted file mode 100644 index 1d101081..00000000 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/H2DataSource.kt +++ /dev/null @@ -1,78 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common - -import no.iktdev.eventi.database.DatabaseConnectionConfig -import no.iktdev.eventi.database.MySqlDataSource -import org.h2.jdbcx.JdbcDataSource -import java.io.PrintWriter -import java.sql.Connection -import java.sql.SQLFeatureNotSupportedException -import java.util.logging.Logger -import javax.sql.DataSource - -class H2DataSource(private val jdbcDataSource: JdbcDataSource, databaseName: String) : DataSource, MySqlDataSource( - DatabaseConnectionConfig( - databaseName = databaseName, address = jdbcDataSource.getUrl(), username = jdbcDataSource.user, password = "", port = null - ) -) { - - companion object { - val connectionUrl = "jdbc:h2:test;MODE=MySQL" //"jdbc:h2:mem:test;MODE=MySQL;DB_CLOSE_DELAY=-1;" - fun getDatasource(): JdbcDataSource { - val ds = JdbcDataSource() - ds.setUrl(connectionUrl) - ds.user = "test" - ds.password = "" - return ds - } - } - - override fun getConnection(): Connection { - return jdbcDataSource.connection - } - - override fun getConnection(username: String?, password: String?): Connection { - return jdbcDataSource.getConnection(username, password) - } - - override fun setLoginTimeout(seconds: Int) { - jdbcDataSource.loginTimeout = seconds - } - - override fun getLoginTimeout(): Int { - return jdbcDataSource.loginTimeout - } - - override fun getLogWriter(): PrintWriter? { - return jdbcDataSource.logWriter - } - - override fun setLogWriter(out: PrintWriter?) { - jdbcDataSource.logWriter = out - } - - override fun getParentLogger(): Logger? { - throw SQLFeatureNotSupportedException("getParentLogger is not supported") - } - - override fun unwrap(iface: Class?): T { - if (iface != null && iface.isAssignableFrom(this.javaClass)) { - return this as T - } - return jdbcDataSource.unwrap(iface) - } - - override fun isWrapperFor(iface: Class<*>?): Boolean { - if (iface != null && iface.isAssignableFrom(this.javaClass)) { - return true - } - return jdbcDataSource.isWrapperFor(iface) - } - - override fun createDatabaseStatement(): String { - return "CREATE SCHEMA ${config.databaseName}" - } - - override fun toConnectionUrl(): String { - return connectionUrl // "jdbc:h2:mem:test;MODE=MySQL;DB_CLOSE_DELAY=-1;" - } -} diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/H2DataSource2.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/H2DataSource2.kt deleted file mode 100644 index 5b2ad11c..00000000 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/H2DataSource2.kt +++ /dev/null @@ -1,23 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common - -import no.iktdev.eventi.database.DatabaseConnectionConfig -import no.iktdev.eventi.database.MySqlDataSource -import org.jetbrains.exposed.sql.Database - -class H2DataSource2(conf: DatabaseConnectionConfig): MySqlDataSource(conf) { - - override fun createDatabaseStatement(): String { - return "CREATE SCHEMA ${config.databaseName};" - } - - override fun toDatabaseConnectionUrl(database: String): String { - return toConnectionUrl() - } - override fun toDatabase(): Database { - return super.toDatabase() - } - override fun toConnectionUrl(): String { - return "jdbc:h2:mem:test;MODE=MySQL;DB_CLOSE_DELAY=-1;CASE_INSENSITIVE_IDENTIFIERS=TRUE;" - } - -} \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/PersistentMessageFromJsonDump.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/PersistentMessageFromJsonDump.kt deleted file mode 100644 index 80049c97..00000000 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/PersistentMessageFromJsonDump.kt +++ /dev/null @@ -1,48 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common - -/* - -class PersistentMessageFromJsonDump(events: String) { - private var data: JsonArray? - - init { - val jsonArray = Json.parseToJsonElement(events) as JsonArray - data = jsonArray.firstOrNull { it.jsonObject["data"] != null }?.jsonObject?.get("data") as? JsonArray - } - - fun getPersistentMessages(): List { - return data?.mapNotNull { - try { - mapToPersistentMessage(it) - } catch (e: Exception) { - System.err.print(it.toString()) - e.printStackTrace() - null - } - } ?: emptyList() - } - - val dzz = DeserializingRegistry() - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS") - private fun mapToPersistentMessage(e: JsonElement): PersistentMessage? { - val referenceId: String = e.jsonObject["referenceId"]?.jsonPrimitive?.content ?: throw RuntimeException("No ReferenceId found") - val eventId: String = e.jsonObject["eventId"]?.jsonPrimitive?.content ?: throw RuntimeException("No EventId") - val event: String = e.jsonObject["event"]?.jsonPrimitive?.content ?: throw RuntimeException("No Event") - val data: String = e.jsonObject["data"]?.jsonPrimitive?.content ?: throw RuntimeException("No data") - val created: String = e.jsonObject["created"]?.jsonPrimitive?.content ?: throw RuntimeException("No Created date time found") - - val kev = KafkaEvents.toEvent(event) ?: throw RuntimeException("Not able to convert event to Enum") - val dzdata = dzz.deserializeData(kev, data) - - return PersistentMessage( - referenceId = referenceId, - eventId = eventId, - event = kev, - data = dzdata, - created = LocalDateTime.parse(created, formatter) - ) - - } - - -}*/ \ No newline at end of file diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt index d191d912..392870d4 100644 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt +++ b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminateTest.kt @@ -1,6 +1,6 @@ package no.iktdev.mediaprocessing.shared.common.parsing -import no.iktdev.mediaprocessing.shared.common.contract.data.EpisodeInfo +import no.iktdev.mediaprocessing.shared.common.model.EpisodeInfo import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test diff --git a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/tests/PersistentEventMangerTestBase.kt b/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/tests/PersistentEventMangerTestBase.kt deleted file mode 100644 index 4ff9fcc5..00000000 --- a/shared/common/src/test/kotlin/no/iktdev/mediaprocessing/shared/common/tests/PersistentEventMangerTestBase.kt +++ /dev/null @@ -1,480 +0,0 @@ -package no.iktdev.mediaprocessing.shared.common.tests - -/* -class PersistentEventMangerTestBase { - val defaultReferenceId = UUID.randomUUID().toString() - val dataSource = H2DataSource2(DatabaseConnectionConfig( - address = "", - username = "", - password = "", - databaseName = "test", - port = null - )) - val eventManager: PersistentEventManager = PersistentEventManager(dataSource) - - init { - val kafkaTables = listOf( - events, // For kafka - ) - dataSource.createDatabase() - dataSource.createTables(*kafkaTables.toTypedArray()) - } - - @Test - fun testDatabaseIsCreated() { - val success = dataSource.createDatabase() - assertThat(success).isNotNull() - } - - @Test - fun testDatabaseInit() { - val referenceId = UUID.randomUUID().toString() - val mStart = Message( - referenceId = referenceId, - eventId = UUID.randomUUID().toString(), - data = MediaProcessStarted( - status = Status.COMPLETED, - file = "Nan" - ) - ) - eventManager.setEvent(KafkaEvents.EventMediaProcessStarted, mStart) - val stored = eventManager.getEventsWith(referenceId); - assertThat(stored).isNotEmpty() - } - - @Test - fun testSuperseded1() { - val startEvent = EventToMessage(KafkaEvents.EventMediaProcessStarted, createMessage()) - val oldStack = listOf( - EventToMessage(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(eventId = "48c72454-6c7b-406b-b598-fc0a961dabde", derivedFromEventId = startEvent.message.eventId)), - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", derivedFromEventId ="48c72454-6c7b-406b-b598-fc0a961dabde")), - EventToMessage(KafkaEvents.EventMediaReadBaseInfoPerformed, - createMessage(eventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e", derivedFromEventId ="1d8d995d-a7e4-4d6e-a501-fe82f521cf72")), - EventToMessage(KafkaEvents.EventMediaMetadataSearchPerformed, - createMessage(eventId = "cbb1e871-e9a5-496d-a655-db719ac4903c", derivedFromEventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e")), - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaReadOutCover, - createMessage(eventId = "98a39721-41ff-4d79-905e-ced260478524", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - - EventToMessage(KafkaEvents.EventMediaParameterEncodeCreated, - createMessage(eventId = "9e8f2e04-4950-437f-a203-cfd566203078", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - ) - eventManager.setEvent(startEvent.event, startEvent.message) - for (entry in oldStack) { - eventManager.setEvent(entry.event, entry.message) - } - val currentTableWithOldStack = eventManager.getEventsWith(defaultReferenceId) - assertThat(currentTableWithOldStack).hasSize(oldStack.size +1) - - val supersedingStack = listOf( - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "2c3a40bb-2225-4dd4-a8c3-32c6356f8764", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")) - ).forEach {entry -> eventManager.setEvent(entry.event, entry.message)} - - - // Final check - - val result = eventManager.getEventsWith(defaultReferenceId) - val idsThatShouldBeRemoved = listOf( - "9e8f2e04-4950-437f-a203-cfd566203078", - "af7f2519-0f1d-4679-82bd-0314d1b97b68" - ) - val search = result.filter { it.eventId in idsThatShouldBeRemoved } - assertThat(search).isEmpty() - - - val expectedInList = listOf( - startEvent.message.eventId, - "48c72454-6c7b-406b-b598-fc0a961dabde", - "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", - "f6cae204-7c8e-4003-b598-f7b4e566d03e", - "cbb1e871-e9a5-496d-a655-db719ac4903c", - "98a39721-41ff-4d79-905e-ced260478524", - "2c3a40bb-2225-4dd4-a8c3-32c6356f8764" - ) - val searchForExpected = result.map { it.eventId } - assertThat(expectedInList).isEqualTo(searchForExpected) - withTransaction(dataSource) { - events.deleteAll() - } - } - - @Test - fun testSuperseded2() { - val startEvent = EventToMessage(KafkaEvents.EventMediaProcessStarted, createMessage()).also { - eventManager.setEvent(it.event, it.message) - } - val keepStack = listOf( - EventToMessage(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(eventId = "48c72454-6c7b-406b-b598-fc0a961dabde", derivedFromEventId = startEvent.message.eventId)), - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", derivedFromEventId ="48c72454-6c7b-406b-b598-fc0a961dabde")), - EventToMessage(KafkaEvents.EventMediaReadBaseInfoPerformed, - createMessage(eventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e", derivedFromEventId ="1d8d995d-a7e4-4d6e-a501-fe82f521cf72")), - EventToMessage(KafkaEvents.EventMediaMetadataSearchPerformed, - createMessage(eventId = "cbb1e871-e9a5-496d-a655-db719ac4903c", derivedFromEventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e")), - EventToMessage(KafkaEvents.EventMediaReadOutCover, - createMessage(eventId = "98a39721-41ff-4d79-905e-ced260478524", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val toBeReplaced = listOf( - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaParameterEncodeCreated, - createMessage(eventId = "9e8f2e04-4950-437f-a203-cfd566203078", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - - val currentTableWithOldStack = eventManager.getEventsWith(defaultReferenceId) - assertThat(currentTableWithOldStack).hasSize(keepStack.size + toBeReplaced.size +1) - - val supersedingStack = listOf( - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "2c3a40bb-2225-4dd4-a8c3-32c6356f8764", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")) - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - - // Final check - - val result = eventManager.getEventsWith(defaultReferenceId) - - val idsRemoved = toBeReplaced.map { it.message.eventId } - val search = result.filter { it.eventId in idsRemoved } - assertThat(search).isEmpty() - - - val expectedInList = listOf(startEvent.message.eventId) + keepStack.map { it.message.eventId } + supersedingStack.map { it.message.eventId } - val searchForExpected = result.map { it.eventId } - assertThat(expectedInList).isEqualTo(searchForExpected) - - withTransaction(dataSource) { - events.deleteAll() - } - } - - @Test - fun testSuperseded3() { - val startEvent = EventToMessage(KafkaEvents.EventMediaProcessStarted, createMessage()).also { - eventManager.setEvent(it.event, it.message) - } - val keepStack = listOf( - EventToMessage(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(eventId = "48c72454-6c7b-406b-b598-fc0a961dabde", derivedFromEventId = startEvent.message.eventId)), - - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val toBeReplaced = listOf( - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", derivedFromEventId ="48c72454-6c7b-406b-b598-fc0a961dabde")), - EventToMessage(KafkaEvents.EventMediaReadBaseInfoPerformed, - createMessage(eventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e", derivedFromEventId ="1d8d995d-a7e4-4d6e-a501-fe82f521cf72")), - EventToMessage(KafkaEvents.EventMediaMetadataSearchPerformed, - createMessage(eventId = "cbb1e871-e9a5-496d-a655-db719ac4903c", derivedFromEventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e")), - EventToMessage(KafkaEvents.EventMediaReadOutCover, - createMessage(eventId = "98a39721-41ff-4d79-905e-ced260478524", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaParameterEncodeCreated, - createMessage(eventId = "9e8f2e04-4950-437f-a203-cfd566203078", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - - val currentTableWithOldStack = eventManager.getEventsWith(defaultReferenceId) - assertThat(currentTableWithOldStack).hasSize(keepStack.size + toBeReplaced.size +1) - - val supersedingStack = listOf( - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "2c3a40bb-2225-4dd4-a8c3-32c6356f8764", derivedFromEventId = "48c72454-6c7b-406b-b598-fc0a961dabde")) - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - - // Final check - - val result = eventManager.getEventsWith(defaultReferenceId) - - val idsRemoved = toBeReplaced.map { it.message.eventId } - val search = result.filter { it.eventId in idsRemoved } - assertThat(search).isEmpty() - - - val expectedInList = listOf(startEvent.message.eventId) + keepStack.map { it.message.eventId } + supersedingStack.map { it.message.eventId } - val searchForExpected = result.map { it.eventId } - assertThat(expectedInList).isEqualTo(searchForExpected) - - withTransaction(dataSource) { - events.deleteAll() - } - } - - @Test - fun testSupersededButKeepWork() { - val startEventPayload = createMessage() - val keepStack = listOf( - EventToMessage(KafkaEvents.EventMediaProcessStarted, startEventPayload), - EventToMessage(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(eventId = "48c72454-6c7b-406b-b598-fc0a961dabde", derivedFromEventId = startEventPayload.eventId)), - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", derivedFromEventId ="48c72454-6c7b-406b-b598-fc0a961dabde")), - EventToMessage(KafkaEvents.EventMediaReadBaseInfoPerformed, - createMessage(eventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e", derivedFromEventId ="1d8d995d-a7e4-4d6e-a501-fe82f521cf72")), - EventToMessage(KafkaEvents.EventMediaMetadataSearchPerformed, - createMessage(eventId = "cbb1e871-e9a5-496d-a655-db719ac4903c", derivedFromEventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e")), - EventToMessage(KafkaEvents.EventMediaReadOutCover, - createMessage(eventId = "98a39721-41ff-4d79-905e-ced260478524", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaParameterEncodeCreated, - createMessage(eventId = "9e8f2e04-4950-437f-a203-cfd566203078", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "ad93a41a-db08-436b-84e4-55adb4752f38", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val newEvents = listOf( - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cfeee961-69c1-4eed-8ec5-82ebca01c9e1", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "64625872-bbfe-4604-85cd-02f58e904267", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "0ab96b32-45a5-4517-b0c0-c03d48145340", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cabd9038-307f-48e4-ac99-88232b1a817c", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "10c0fd42-b5be-42b2-a27b-12ecccc51635", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "b69fb306-e390-4a9e-8d11-89d0688dff16", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val result = eventManager.getEventsWith(defaultReferenceId) - - val expected = (keepStack + newEvents).map { it.message.eventId } - val missing = expected - result.map { it.eventId } - assertThat(missing).isEmpty() - assertThat(expected.size).isEqualTo(result.size) - - withTransaction(dataSource) { - events.deleteAll() - } - } - - @Test - fun testSupersededWork() { - val startEventPayload = createMessage() - val keepStack = listOf( - EventToMessage(KafkaEvents.EventMediaProcessStarted, startEventPayload), - EventToMessage(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(eventId = "48c72454-6c7b-406b-b598-fc0a961dabde", derivedFromEventId = startEventPayload.eventId)), - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", derivedFromEventId ="48c72454-6c7b-406b-b598-fc0a961dabde")), - EventToMessage(KafkaEvents.EventMediaReadBaseInfoPerformed, - createMessage(eventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e", derivedFromEventId ="1d8d995d-a7e4-4d6e-a501-fe82f521cf72")), - EventToMessage(KafkaEvents.EventMediaMetadataSearchPerformed, - createMessage(eventId = "cbb1e871-e9a5-496d-a655-db719ac4903c", derivedFromEventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e")), - EventToMessage(KafkaEvents.EventMediaReadOutCover, - createMessage(eventId = "98a39721-41ff-4d79-905e-ced260478524", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaParameterEncodeCreated, - createMessage(eventId = "9e8f2e04-4950-437f-a203-cfd566203078", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val newEvents = listOf( - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "ad93a41a-db08-436b-84e4-55adb4752f38", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cfeee961-69c1-4eed-8ec5-82ebca01c9e1", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "64625872-bbfe-4604-85cd-02f58e904267", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "0ab96b32-45a5-4517-b0c0-c03d48145340", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cabd9038-307f-48e4-ac99-88232b1a817c", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "10c0fd42-b5be-42b2-a27b-12ecccc51635", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val replacedWith = listOf( - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "e40b2096-2e6f-4672-9c5a-6c81fe8fc302", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "b69fb306-e390-4a9e-8d11-89d0688dff16", derivedFromEventId = "e40b2096-2e6f-4672-9c5a-6c81fe8fc302")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val result = eventManager.getEventsWith(defaultReferenceId) - - val expected = (keepStack + replacedWith).map { it.message.eventId } - val missing = expected - result.map { it.eventId } - assertThat(missing).isEmpty() - assertThat(expected.size).isEqualTo(result.size) - - withTransaction(dataSource) { - events.deleteAll() - } - } - - @Test - fun testConvertBatchFromExtract() { - val startEventPayload = createMessage() - val keepStack = listOf( - EventToMessage(KafkaEvents.EventMediaProcessStarted, startEventPayload), - EventToMessage(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(eventId = "48c72454-6c7b-406b-b598-fc0a961dabde", derivedFromEventId = startEventPayload.eventId)), - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", derivedFromEventId ="48c72454-6c7b-406b-b598-fc0a961dabde")), - EventToMessage(KafkaEvents.EventMediaReadBaseInfoPerformed, - createMessage(eventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e", derivedFromEventId ="1d8d995d-a7e4-4d6e-a501-fe82f521cf72")), - EventToMessage(KafkaEvents.EventMediaMetadataSearchPerformed, - createMessage(eventId = "cbb1e871-e9a5-496d-a655-db719ac4903c", derivedFromEventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e")), - EventToMessage(KafkaEvents.EventMediaReadOutCover, - createMessage(eventId = "98a39721-41ff-4d79-905e-ced260478524", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaParameterEncodeCreated, - createMessage(eventId = "9e8f2e04-4950-437f-a203-cfd566203078", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val convertEvents = mutableListOf(); - - val extractEvents = listOf( - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "ad93a41a-db08-436b-84e4-55adb4752f38", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cfeee961-69c1-4eed-8ec5-82ebca01c9e1", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "64625872-bbfe-4604-85cd-02f58e904267", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "0ab96b32-45a5-4517-b0c0-c03d48145340", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cabd9038-307f-48e4-ac99-88232b1a817c", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "10c0fd42-b5be-42b2-a27b-12ecccc51635", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "b69fb306-e390-4a9e-8d11-89d0688dff16", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - ).onEach { entry -> - run { - eventManager.setEvent(entry.event, entry.message) - convertEvents.add(EventToMessage(KafkaEvents.EventWorkConvertCreated, - createMessage(derivedFromEventId = entry.message.eventId))) - } - } - - val simpleCascade = eventManager.getEventsWith(defaultReferenceId) - assertThat(simpleCascade.size).isEqualTo(keepStack.size+extractEvents.size) - - assertThat(convertEvents.size).isEqualTo(extractEvents.size) - convertEvents.forEach { - eventManager.setEvent(it.event, it.message) - } - - val result = eventManager.getEventsWith(defaultReferenceId) - - assertThat(result.size).isEqualTo(keepStack.size+extractEvents.size+convertEvents.size) - - withTransaction(dataSource) { - events.deleteAll() - } - } - - - @Test - fun testSomeAreSingleSomeAreNot() { - val startEventPayload = createMessage() - val keepStack = listOf( - EventToMessage(KafkaEvents.EventMediaProcessStarted, startEventPayload), - EventToMessage(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(eventId = "48c72454-6c7b-406b-b598-fc0a961dabde", derivedFromEventId = startEventPayload.eventId)), - EventToMessage(KafkaEvents.EventMediaParseStreamPerformed, - createMessage(eventId = "1d8d995d-a7e4-4d6e-a501-fe82f521cf72", derivedFromEventId ="48c72454-6c7b-406b-b598-fc0a961dabde")), - EventToMessage(KafkaEvents.EventMediaReadBaseInfoPerformed, - createMessage(eventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e", derivedFromEventId ="1d8d995d-a7e4-4d6e-a501-fe82f521cf72")), - EventToMessage(KafkaEvents.EventMediaMetadataSearchPerformed, - createMessage(eventId = "cbb1e871-e9a5-496d-a655-db719ac4903c", derivedFromEventId = "f6cae204-7c8e-4003-b598-f7b4e566d03e")), - EventToMessage(KafkaEvents.EventMediaReadOutCover, - createMessage(eventId = "98a39721-41ff-4d79-905e-ced260478524", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaReadOutNameAndType, - createMessage(eventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9", derivedFromEventId = "cbb1e871-e9a5-496d-a655-db719ac4903c")), - EventToMessage(KafkaEvents.EventMediaParameterEncodeCreated, - createMessage(eventId = "9e8f2e04-4950-437f-a203-cfd566203078", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventMediaParameterExtractCreated, - createMessage(eventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68", derivedFromEventId = "3f376b72-f55a-4dd7-af87-fb1755ba4ad9")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "ad93a41a-db08-436b-84e4-55adb4752f38", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val newEvents = listOf( - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cfeee961-69c1-4eed-8ec5-82ebca01c9e1", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "64625872-bbfe-4604-85cd-02f58e904267", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "0ab96b32-45a5-4517-b0c0-c03d48145340", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventWorkExtractCreated, - createMessage(eventId = "cabd9038-307f-48e4-ac99-88232b1a817c", derivedFromEventId = "af7f2519-0f1d-4679-82bd-0314d1b97b68")), - EventToMessage(KafkaEvents.EventMediaProcessCompleted, - createMessage(eventId = "10c0fd42-b5be-42b2-a27b-12ecccc51635", derivedFromEventId = "cabd9038-307f-48e4-ac99-88232b1a817c")), - EventToMessage(KafkaEvents.EventMediaProcessCompleted, - createMessage(eventId = "3519af2e-0767-4dbb-b0c5-f19cb926900d", derivedFromEventId = "cabd9038-307f-48e4-ac99-88232b1a817c")), - - EventToMessage(KafkaEvents.EventCollectAndStore, - createMessage(eventId = "b69fb306-e390-4a9e-8d11-89d0688dff16", derivedFromEventId = "3519af2e-0767-4dbb-b0c5-f19cb926900d")), - EventToMessage(KafkaEvents.EventCollectAndStore, - createMessage(eventId = "4e6d3a6a-ab89-4627-9158-3c3f92ff7b4c", derivedFromEventId = "3519af2e-0767-4dbb-b0c5-f19cb926900d")), - EventToMessage(KafkaEvents.EventCollectAndStore, - createMessage(eventId = "4e6d3a6a-ab89-4627-9158-3c3f92ff7b4c", derivedFromEventId = "3519af2e-0767-4dbb-b0c5-f19cb926900d")), - ).onEach { entry -> eventManager.setEvent(entry.event, entry.message) } - - val result = eventManager.getEventsWith(defaultReferenceId) - val singles = result.filter { it.event != KafkaEvents.EventWorkExtractCreated } - singles.forEach { - val instancesOfMe = singles.filter { sit -> it.event == sit.event } - assertThat(instancesOfMe).hasSize(1) - } - assertThat(result.filter { it.event == KafkaEvents.EventCollectAndStore }).hasSize(1) - - - - withTransaction(dataSource) { - events.deleteAll() - } - } - - @Test - fun testDerivedOrphanNotInserted() { - val startEvent = EventToMessage(KafkaEvents.EventMediaProcessStarted, createMessage()).also { - eventManager.setEvent(it.event, it.message) - } - val result = eventManager.setEvent(KafkaEvents.EventMediaReadStreamPerformed, - createMessage(derivedFromEventId = UUID.randomUUID().toString())) - assertThat(result).isFalse() - } - - data class EventToMessage(val event: KafkaEvents, val message: Message<*>) - - private fun createMessage(referenceId: String = defaultReferenceId, eventId: String = UUID.randomUUID().toString(), derivedFromEventId: String? = null): Message{ - return Message( - referenceId = referenceId, - eventId = eventId, - data = SimpleMessageData( - status = Status.COMPLETED, - message = "Potato", - derivedFromEventId = derivedFromEventId - ) - ) - } - -}*/ \ No newline at end of file diff --git a/shared/ffmpeg/build.gradle.kts b/shared/ffmpeg/build.gradle.kts new file mode 100644 index 00000000..324b6ca3 --- /dev/null +++ b/shared/ffmpeg/build.gradle.kts @@ -0,0 +1,36 @@ +plugins { + kotlin("jvm") +} + +group = "no.iktdev.mediaprocessing" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() + maven("https://jitpack.io") + maven { + url = uri("https://reposilite.iktdev.no/releases") + } + maven { + url = uri("https://reposilite.iktdev.no/snapshots") + } +} + +dependencies { + + + implementation("com.github.pgreze:kotlin-process:1.5.1") + implementation("com.google.code.gson:gson:2.8.9") + + implementation("no.iktdev:exfl:1.0-rc1") + + + testImplementation(kotlin("test")) +} + +tasks.test { + useJUnitPlatform() +} +kotlin { + jvmToolchain(21) +} \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFinfo.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFinfo.kt new file mode 100644 index 00000000..0eb789ad --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFinfo.kt @@ -0,0 +1,43 @@ +package no.iktdev.mediaprocessing.ffmpeg + +import com.github.pgreze.process.ProcessResult +import com.github.pgreze.process.Redirect +import com.github.pgreze.process.process +import com.google.gson.Gson +import com.google.gson.JsonObject +import no.iktdev.mediaprocessing.ffmpeg.data.FFinfoOutput + +abstract class FFinfo { + open val defaultArguments: List = listOf("-v", "quiet") + abstract val executable: String + + open suspend fun readJsonStreams(inputFile: String): FFinfoOutput { + var error: String? = null + val output = mutableListOf() + val args = defaultArguments + listOf("-print_format", "json", "-show_format", "-show_streams", inputFile) + val processResult = execute(args) { output.add(it) } + + val success = processResult.resultCode == 0 + val longString = output.joinToString(" ") + val json = try { + Gson().fromJson(longString, JsonObject::class.java) + } catch (e: Exception) { + error = "Could not parse ffinfo output to JSON: ${e.message}" + null + } + return FFinfoOutput( + success = success, + data = json, + error = error + ) + } + + private suspend fun execute(arguments: List, output: (String) -> Unit): ProcessResult { + return process(executable, *arguments.toTypedArray(), + stderr = Redirect.CAPTURE, + stdout = Redirect.CAPTURE, + consumer = { + output(it) + }) + } +} \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFmpeg.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFmpeg.kt new file mode 100644 index 00000000..bfa29e0f --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/FFmpeg.kt @@ -0,0 +1,101 @@ +package no.iktdev.mediaprocessing.ffmpeg + +import com.github.pgreze.process.ProcessResult +import com.github.pgreze.process.Redirect +import com.github.pgreze.process.process +import no.iktdev.exfl.using +import no.iktdev.mediaprocessing.ffmpeg.arguments.MpegArgument +import no.iktdev.mediaprocessing.ffmpeg.decoder.FfmpegDecodedProgress +import no.iktdev.mediaprocessing.ffmpeg.decoder.FfmpegProgressDecoder +import java.io.File +import java.io.FileOutputStream +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +abstract class FFmpeg { + abstract val executable: String + abstract val logDir: File + open val listener: Listener? = null + + private var progress: FfmpegDecodedProgress? = null + val decoder = FfmpegProgressDecoder() + private val outputCache = mutableListOf() + + //region Log File formatting + val currentDateTime = LocalDateTime.now() + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd.HH.mm") + val formattedDateTime = currentDateTime.format(formatter) + //endregion + lateinit var logFile: File + + lateinit var result: ProcessResult + protected set + + private lateinit var inputFile: String + open suspend fun run(argument: MpegArgument) { + inputFile = if (argument.inputFile == null) throw RuntimeException("Input file is required") else argument.inputFile!! + logFile = logDir.using("$formattedDateTime-${File(inputFile).nameWithoutExtension}.log") + listener?.onStarted(argument.inputFile!!) + result = execute(argument.build()) { + onNewOutput(it) + } + onNewOutput("Received exit code: ${result.resultCode}") + if (result.resultCode != 0) { + listener?.onError(inputFile, result.output.joinToString("\n")) + } else { + val success = moveAndVerify(argument) + if (!success) { + listener?.onError(inputFile, "Could not find output file at ${argument.outputFile}") + } else { + listener?.onCompleted(inputFile, argument.outputFile!!) + } + } + } + + private fun moveAndVerify(argument: MpegArgument): Boolean { + return if (argument.outputCacheFile) { + File(argument.getOutputFileUsed()).renameTo(File(argument.outputFile!!)) + } else File(argument.outputFile!!).exists() + } + + private suspend fun execute(arguments: List, output: (String) -> Unit): ProcessResult { + return process(executable, *arguments.toTypedArray(), + stdout = Redirect.CAPTURE, + stderr = Redirect.CAPTURE, + consumer = { + output(it) + }, + destroyForcibly = true + ) + } + + open fun onNewOutput(line: String) { + outputCache.add(line) + writeToLog(line) + decoder.defineDuration(line) + decoder.parseVideoProgress(outputCache.toList())?.let { decoded -> + try { + val _progress = decoder.getProgress(decoded) + if (progress == null || _progress.progress > (progress?.progress ?: -1)) { + progress = _progress + listener?.onProgressChanged(inputFile, _progress) + } + } catch (e: Exception) { + e.printStackTrace() + } + } + } + + open fun writeToLog(line: String) { + FileOutputStream(logFile, true).bufferedWriter(Charsets.UTF_8).use { + it.appendLine(line) + } + } + + interface Listener { + fun onStarted(inputFile: String) + fun onCompleted(inputFile: String, outputFile: String) + fun onProgressChanged(inputFile: String, progress: FfmpegDecodedProgress) + fun onError(inputFile: String, message: String) {} + } +} \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/arguments/MpegArgument.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/arguments/MpegArgument.kt new file mode 100644 index 00000000..8c598421 --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/arguments/MpegArgument.kt @@ -0,0 +1,89 @@ +package no.iktdev.mediaprocessing.ffmpeg.arguments + +import java.io.File + +class MpegArgument { + private val defaultArguments = listOf( + "-nostdin", + "-nostats", + "-hide_banner" + ) + var inputFile: String? = null + private set + var outputFile: String? = null + private set + private var overwrite: Boolean = false + private var progress: Boolean = false + private var suppliedArgs: List = emptyList() + var outputCacheFile: Boolean = true + private set + + fun inputFile(inputFile: String) = apply { + this.inputFile = inputFile + } + + fun outputFile(outputFile: String) = apply { + this.outputFile = outputFile + } + + fun allowOverwrite(allowOverwrite: Boolean) = apply { + this.overwrite = allowOverwrite + } + + fun withProgress(withProgress: Boolean) = apply { + this.progress = withProgress + } + + fun args(args: List) = apply { + this.suppliedArgs = args + } + + fun useCacheFile(useCacheFile: Boolean) = apply { + this.outputCacheFile = useCacheFile + } + + fun getCacheOutputFile(): String { + return if (outputCacheFile) { + File(outputFile!!).let { + File(it.parentFile.absoluteFile, "${it.nameWithoutExtension}.work.${it.extension}") + }.absolutePath + } else { + this.outputFile!! + } + } + + fun getOutputFileUsed(): String { + if (outputFile == null || outputFile?.isBlank() == true) { + throw RuntimeException("Outputfile is required") + } + return if (outputCacheFile) { + File(outputFile!!).let { + File(it.parentFile.absoluteFile, "${it.nameWithoutExtension}.work.${it.extension}") + }.absolutePath + } else { + this.outputFile!! + } + } + + fun build(): List { + val args = mutableListOf() + val inFile = if (inputFile == null || inputFile?.isBlank() == true) { + throw RuntimeException("Inputfile is required") + } else this.inputFile!! + val outFile: String = getOutputFileUsed() + if (overwrite) { + args.add("-y") + } + args.addAll(defaultArguments) + args.addAll(listOf("-i", inFile)) + args.addAll(suppliedArgs) + args.add(outFile) + if (progress) { + args.addAll(listOf("-progress", "pipe:1")) + } + + + + return args + } +} \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFOutput.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFOutput.kt new file mode 100644 index 00000000..9b543c27 --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFOutput.kt @@ -0,0 +1,5 @@ +package no.iktdev.mediaprocessing.ffmpeg.data + +abstract class FFOutput { + abstract val success: Boolean +} \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFinfoOutput.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFinfoOutput.kt new file mode 100644 index 00000000..3ee03f02 --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFinfoOutput.kt @@ -0,0 +1,10 @@ +package no.iktdev.mediaprocessing.ffmpeg.data + +import com.google.gson.JsonObject + +data class FFinfoOutput( + override val success: Boolean, + val error: String? = null, + val data: JsonObject? = null +) : FFOutput() { +} \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFmpegOutput.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFmpegOutput.kt new file mode 100644 index 00000000..abcf9efa --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/data/FFmpegOutput.kt @@ -0,0 +1,6 @@ +package no.iktdev.mediaprocessing.ffmpeg.data + +data class FFmpegOutput( + override val success: Boolean +) : FFOutput() { +} \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/decoder/FfmpegDecodedProgress.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/decoder/FfmpegDecodedProgress.kt new file mode 100755 index 00000000..6f944502 --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/decoder/FfmpegDecodedProgress.kt @@ -0,0 +1,30 @@ +package no.iktdev.mediaprocessing.ffmpeg.decoder + + +data class FfmpegDecodedProgress( + val progress: Int = -1, + val time: String, + val duration: String, + val speed: String, + val estimatedCompletionSeconds: Long = -1, + val estimatedCompletion: String = "Unknown", +) { + fun toProcessProgress(): ProcesserProgress { + return ProcesserProgress( + progress = this.progress, + speed = this.speed, + timeWorkedOn = this.time, + timeLeft = this.estimatedCompletion + ) + + } +} + +data class ProcesserProgress( + val progress: Int = -1, + val speed: String? = null, + val timeWorkedOn: String? = null, + val timeLeft: String? = "Unknown", // HH mm +) + +data class ECT(val day: Int = 0, val hour: Int = 0, val minute: Int = 0, val second: Int = 0) \ No newline at end of file diff --git a/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/decoder/FfmpegProgressDecoder.kt b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/decoder/FfmpegProgressDecoder.kt new file mode 100755 index 00000000..045c8194 --- /dev/null +++ b/shared/ffmpeg/src/main/kotlin/no/iktdev/mediaprocessing/ffmpeg/decoder/FfmpegProgressDecoder.kt @@ -0,0 +1,173 @@ +package no.iktdev.mediaprocessing.ffmpeg.decoder + +import java.lang.StringBuilder +import java.time.LocalTime +import java.time.format.DateTimeFormatter +import java.util.concurrent.TimeUnit +import kotlin.math.floor + +class FfmpegProgressDecoder { + + data class DecodedProgressData( + val frame: Int?, + val fps: Double?, + val stream_0_0_q: Double?, + val bitrate: String?, + val total_size: Int?, + val out_time_us: Long?, + val out_time_ms: Long?, + val out_time: String?, + val dup_frames: Int?, + val drop_frames: Int?, + val speed: Double?, + val progress: String? + ) + + val expectedKeys = listOf( + "frame=", + "fps=", + "stream_0_0_q=", + "bitrate=", + "total_size=", + "out_time_us=", + "out_time_ms=", + "out_time=", + "dup_frames=", + "drop_frames=", + "speed=", + "progress=" + ) + var duration: Int? = null + set(value) { + if (field == null || field == 0) + field = value + } + var durationTime: String = "NA" + fun parseVideoProgress(lines: List): DecodedProgressData? { + var frame: Int? = null + var progress: String? = null + val metadataMap = mutableMapOf() + + try { + val eqValue = Regex("=") + for (line in lines) { + val keyValuePairs = Regex("=\\s*").replace(line, "=").split(" ").filter { it.isNotBlank() }.filter { eqValue.containsMatchIn(it) } + for (keyValuePair in keyValuePairs) { + val (key, value) = keyValuePair.split("=") + metadataMap[key] = value + } + + if (frame == null) { + frame = metadataMap["frame"]?.toIntOrNull() + } + + progress = metadataMap["progress"] + } + } catch (e: Exception) { + e.printStackTrace() + } + + return if (progress != null) { + // When "progress" is found, build and return the VideoMetadata object + DecodedProgressData( + frame, metadataMap["fps"]?.toDoubleOrNull(), metadataMap["stream_0_0_q"]?.toDoubleOrNull(), + metadataMap["bitrate"], metadataMap["total_size"]?.toIntOrNull(), metadataMap["out_time_us"]?.toLongOrNull(), + metadataMap["out_time_ms"]?.toLongOrNull(), metadataMap["out_time"], metadataMap["dup_frames"]?.toIntOrNull(), + metadataMap["drop_frames"]?.toIntOrNull(), metadataMap["speed"]?.replace("x", "", ignoreCase = true)?.toDoubleOrNull(), progress + ) + } else { + null // If "progress" is not found, return null + } + } + + + val parsedDurations: MutableList = mutableListOf() + var hasReadContinue: Boolean = false + fun defineDuration(value: String) { + if (hasReadContinue) { + return + } + if (value.contains(Regex("progress=continue"))) { + hasReadContinue = true + } + + val results = Regex("Duration:\\s*([^,]+),").find(value)?.groupValues?.firstOrNull() ?: return + + val parsedDuration = Regex("[0-9]+:[0-9]+:[0-9]+.[0-9]+").find(results.toString())?.value ?: return + + if (!parsedDurations.contains(parsedDuration)) { + parsedDurations.add(parsedDuration) + } + + val longestDuration = parsedDurations.mapNotNull { + timeSpanToSeconds(it) + }.maxOrNull() ?: return + + duration = longestDuration + } + + private fun timeSpanToSeconds(time: String?): Int? + { + time ?: return null + val timeString = Regex("[0-9]+:[0-9]+:[0-9]+.[0-9]+").find(time) ?: return null + val strippedMS = Regex("[0-9]+:[0-9]+:[0-9]+").find(timeString.value) ?: return null + val outTime = LocalTime.parse(strippedMS.value, DateTimeFormatter.ofPattern("HH:mm:ss")) + return outTime.toSecondOfDay() + } + + + fun getProgress(decoded: DecodedProgressData): FfmpegDecodedProgress { + if (duration == null) + return FfmpegDecodedProgress(duration = durationTime, time = "NA", speed = "NA") + val progressTime = timeSpanToSeconds(decoded.out_time) ?: 0 + val progress = floor((progressTime.toDouble() / duration!!.toDouble()) *100).toInt() + + val ect = getEstimatedTimeRemaining(decoded) + + return FfmpegDecodedProgress( + progress = progress, + estimatedCompletionSeconds = ect, + estimatedCompletion = getETA(ect), + duration = durationTime, + time = decoded.out_time ?: "NA", + speed = decoded.speed?.toString() ?: "NA" + ) + } + + fun getEstimatedTimeRemaining(decoded: DecodedProgressData): Long { + val position = timeSpanToSeconds(decoded.out_time) ?: 0 + return if(duration == null || decoded.speed == null) -1 else + Math.round(Math.round(duration!!.toDouble() - position.toDouble()) / decoded.speed) + } + + fun getECT(time: Long): ECT { + var seconds = time + val day = TimeUnit.SECONDS.toDays(seconds) + seconds -= TimeUnit.DAYS.toSeconds(day) + + val hour = TimeUnit.SECONDS.toHours(seconds) + seconds -= TimeUnit.HOURS.toSeconds(hour) + + val minute = TimeUnit.SECONDS.toMinutes(seconds) + seconds -= TimeUnit.MINUTES.toSeconds(minute) + + return ECT(day.toInt(), hour.toInt(), minute.toInt(), seconds.toInt()) + } + private fun getETA(time: Long): String { + val etc = getECT(time) ?: return "Unknown" + val str = StringBuilder() + if (etc.day > 0) { + str.append("${etc.day}d").append(" ") + } + if (etc.hour > 0) { + str.append("${etc.hour}h").append(" ") + } + if (etc.day == 0 && etc.minute > 0) { + str.append("${etc.minute}m").append(" ") + } + if (etc.hour == 0 && etc.second > 0) { + str.append("${etc.second}s").append(" ") + } + return str.toString().trim() + } +} \ No newline at end of file