Updated version + Naming parsing

This commit is contained in:
Brage Skjønborg 2025-10-04 23:34:09 +02:00
parent b1061b3c9e
commit 35d4299e74
19 changed files with 1026 additions and 452 deletions

1
.idea/gradle.xml generated
View File

@ -5,6 +5,7 @@
<option name="linkedExternalProjectsSettings">
<GradleProjectSettings>
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="gradleJvm" value="17" />
<option name="modules">
<set>
<option value="$PROJECT_DIR$" />

2
.idea/kotlinc.xml generated
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="KotlinJpsPluginSettings">
<option name="version" value="1.9.20" />
<option name="version" value="2.1.0" />
</component>
</project>

View File

@ -0,0 +1,26 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="UIApplicationKt" type="JetRunConfigurationType" nameIsGenerated="true">
<envs>
<env name="DATABASE_ADDRESS" value="192.168.2.250" />
<env name="DATABASE_NAME_E" value="eventsV4" />
<env name="DATABASE_NAME_S" value="streamitv3" />
<env name="DATABASE_PASSWORD" value="shFZ27eL2x2NoxyEDBMfDWkvFO" />
<env name="DATABASE_PORT" value="3306" />
<env name="DATABASE_USERNAME" value="root" />
<env name="DIRECTORY_CONTENT_INCOMING" value="G:\MediaProcessingPlayground\input" />
<env name="DIRECTORY_CONTENT_OUTGOING" value="G:\MediaProcessingPlayground\output" />
<env name="DISABLE_COMPLETE" value="true" />
<env name="DISABLE_PRODUCE" value="true" />
<env name="EncoderWs" value="ws://192.168.2.250:6081/ws" />
<env name="METADATA_TIMEOUT" value="0" />
<env name="SUPPORTING_EXECUTABLE_FFMPEG" value="G:\MediaProcessingPlayground\ffmpeg.exe" />
<env name="SUPPORTING_EXECUTABLE_FFPROBE" value="G:\MediaProcessingPlayground\ffprobe.exe" />
</envs>
<option name="MAIN_CLASS_NAME" value="no.iktdev.mediaprocessing.ui.UIApplicationKt" />
<module name="MediaProcessing.apps.ui.main" />
<shortenClasspath name="NONE" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

1194
.idea/workspace.xml generated

File diff suppressed because it is too large Load Diff

View File

@ -2,8 +2,8 @@ plugins {
id("java")
kotlin("jvm")
kotlin("plugin.spring") version "1.5.31"
id("org.springframework.boot") version "2.5.5"
id("io.spring.dependency-management") version "1.0.11.RELEASE"
id("org.springframework.boot") version "3.2.0"
id("io.spring.dependency-management") version "1.1.4"
id("org.jetbrains.kotlin.plugin.serialization") version "1.5.0" // Legg til Kotlin Serialization-plugin
}
@ -26,18 +26,19 @@ val exposedVersion = "0.44.0"
dependencies {
/*Spring boot*/
implementation("org.springframework.boot:spring-boot-starter")
implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.boot:spring-boot-starter:3.2.0")
// implementation("org.springframework.kafka:spring-kafka:3.0.1")
implementation("org.springframework.kafka:spring-kafka:2.8.5")
implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3")
implementation("org.springframework.boot:spring-boot-starter-actuator")
implementation("org.springframework.boot:spring-boot-starter-websocket")
implementation("org.springframework:spring-tx")
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
implementation("com.google.code.gson:gson:2.8.9")
implementation("org.json:json:20210307")
implementation("no.iktdev:exfl:0.0.16-SNAPSHOT")
implementation("no.iktdev.streamit.library:streamit-library-db:1.0.0-alpha14")
implementation("no.iktdev.streamit.library:streamit-library-db:1.0-rc1")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
@ -71,16 +72,6 @@ dependencies {
testImplementation("org.mockito:mockito-core:3.+")
testImplementation("org.assertj:assertj-core:3.4.1")
/*testImplementation("org.junit.vintage:junit-vintage-engine")
testImplementation("org.junit.jupiter:junit-jupiter:5.10.1")
testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1")
testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.1")
testRuntimeOnly ("org.junit.jupiter:junit-jupiter-engine:5.10.1")
testImplementation("org.mockito:mockito-core:5.8.0") // Oppdater versjonen hvis det er nyere tilgjengelig
testImplementation("org.mockito:mockito-junit-jupiter:5.8.0")
testImplementation(platform("org.junit:junit-bom:5.10.1"))
testImplementation("org.junit.platform:junit-platform-runner:1.10.1")*/
testImplementation(platform("org.junit:junit-bom:5.9.1"))
testImplementation("org.junit.jupiter:junit-jupiter")
testImplementation("junit:junit:4.13.2")

View File

@ -1,6 +1,7 @@
package no.iktdev.mediaprocessing.coordinator
import jakarta.annotation.PreDestroy
import mu.KotlinLogging
import no.iktdev.exfl.coroutines.CoroutinesDefault
import no.iktdev.exfl.coroutines.CoroutinesIO
@ -10,15 +11,22 @@ import no.iktdev.eventi.database.MySqlDataSource
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
import no.iktdev.streamit.library.db.tables.*
import no.iktdev.streamit.library.db.tables.helper.cast_errors
import no.iktdev.streamit.library.db.tables.helper.data_audio
import no.iktdev.streamit.library.db.tables.helper.data_video
import no.iktdev.streamit.library.db.tables.content.CatalogTable
import no.iktdev.streamit.library.db.tables.content.GenreTable
import no.iktdev.streamit.library.db.tables.content.MovieTable
import no.iktdev.streamit.library.db.tables.content.ProgressTable
import no.iktdev.streamit.library.db.tables.content.SerieTable
import no.iktdev.streamit.library.db.tables.content.SubtitleTable
import no.iktdev.streamit.library.db.tables.content.SummaryTable
import no.iktdev.streamit.library.db.tables.content.TitleTable
import no.iktdev.streamit.library.db.tables.other.CastErrorTable
import no.iktdev.streamit.library.db.tables.other.DataAudioTable
import no.iktdev.streamit.library.db.tables.other.DataVideoTable
import no.iktdev.streamit.library.db.tables.user.UserTable
import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.boot.runApplication
import org.springframework.context.annotation.Bean
import org.springframework.transaction.annotation.Transactional
import javax.annotation.PreDestroy
val log = KotlinLogging.logger {}
lateinit var eventDatabase: EventsDatabase
@ -101,18 +109,18 @@ fun main(args: Array<String>) {
val tables = arrayOf(
catalog,
genre,
movie,
serie,
subtitle,
summary,
users,
progress,
data_audio,
data_video,
cast_errors,
titles
CatalogTable,
GenreTable,
MovieTable,
SerieTable,
SubtitleTable,
SummaryTable,
UserTable,
ProgressTable,
DataAudioTable,
DataVideoTable,
CastErrorTable,
TitleTable
)
storeDatabase.createTables(*tables)

View File

@ -7,16 +7,10 @@ import no.iktdev.mediaprocessing.shared.common.SharedConfig
import no.iktdev.mediaprocessing.shared.common.database.tables.files
import no.iktdev.mediaprocessing.shared.common.extended.isSupportedVideoFile
import no.iktdev.mediaprocessing.shared.common.md5
import no.iktdev.streamit.library.db.withTransaction
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.deleteWhere
import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.selectAll
import org.springframework.scheduling.annotation.EnableScheduling
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Service
import java.io.File
@Service
@EnableScheduling

View File

@ -6,9 +6,9 @@ import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
import no.iktdev.mediaprocessing.shared.common.contract.reader.VideoDetails
import no.iktdev.streamit.library.db.executeWithStatus
import no.iktdev.streamit.library.db.insertWithSuccess
import no.iktdev.streamit.library.db.query.MovieQuery
import no.iktdev.streamit.library.db.tables.catalog
import no.iktdev.streamit.library.db.tables.serie
import no.iktdev.streamit.library.db.tables.content.CatalogTable
import no.iktdev.streamit.library.db.tables.content.MovieTable
import no.iktdev.streamit.library.db.tables.content.SerieTable
import org.jetbrains.exposed.sql.*
object ContentCatalogStore {
@ -20,56 +20,56 @@ object ContentCatalogStore {
*/
fun getCollectionByTitleAndType(type: String, titles: List<String>): String? {
return withTransaction(getStoreDatabase()) {
catalog.select {
(catalog.type eq type) and
((catalog.title inList titles) or
(catalog.collection inList titles))
CatalogTable.select {
(CatalogTable.type eq type) and
((CatalogTable.title inList titles) or
(CatalogTable.collection inList titles))
}.map {
it[catalog.collection]
it[CatalogTable.collection]
}.firstOrNull()
}
}
private fun getCover(collection: String, type: String): String? {
return withTransaction(getStoreDatabase()) {
catalog.select {
(catalog.collection eq collection) and
(catalog.type eq type)
}.map { it[catalog.cover] }.firstOrNull()
CatalogTable.select {
(CatalogTable.collection eq collection) and
(CatalogTable.type eq type)
}.map { it[CatalogTable.cover] }.firstOrNull()
}
}
fun storeCatalog(title: String, titles: List<String>, collection: String, type: String, cover: String?, genres: String?): Int? {
val status = executeWithStatus(getStoreDatabase().database, block = {
val existingRow = catalog.select {
(catalog.collection eq collection) and
(catalog.type eq type)
val status = executeWithStatus(getStoreDatabase().database, run = {
val existingRow = CatalogTable.select {
(CatalogTable.collection eq collection) and
(CatalogTable.type eq type)
}.firstOrNull()
if (existingRow == null) {
log.info { "$collection does not exist, and will be created" }
catalog.insert {
it[catalog.title] = title
it[catalog.cover] = cover
it[catalog.type] = type
it[catalog.collection] = collection
it[catalog.genres] = genres
CatalogTable.insert {
it[CatalogTable.title] = title
it[CatalogTable.cover] = cover
it[CatalogTable.type] = type
it[CatalogTable.collection] = collection
it[CatalogTable.genres] = genres
}
} else {
val id = existingRow[catalog.id]
val storedTitle = existingRow[catalog.title]
val useCover = existingRow[catalog.cover] ?: cover
val useGenres = existingRow[catalog.genres] ?: genres
val id = existingRow[CatalogTable.id]
val storedTitle = existingRow[CatalogTable.title]
val useCover = existingRow[CatalogTable.cover] ?: cover
val useGenres = existingRow[CatalogTable.genres] ?: genres
catalog.update({
(catalog.id eq id) and
(catalog.collection eq collection)
CatalogTable.update({
(CatalogTable.id eq id) and
(CatalogTable.collection eq collection)
}) {
it[catalog.cover] = useCover
it[catalog.genres] = useGenres
it[CatalogTable.cover] = useCover
it[CatalogTable.genres] = useGenres
}
}
}, {
}, onError = {
log.error { "Failed to store catalog $collection: ${it.message}" }
})
if (status) {
@ -81,17 +81,17 @@ object ContentCatalogStore {
}
private fun storeMovie(catalogId: Int, videoDetails: VideoDetails) {
val iid = MovieQuery(videoDetails.fileName).insertAndGetId() ?: run {
val iid = MovieTable.insertAndGetId(videoDetails.fileName)?.value ?: run {
log.error { "Movie id was not returned!" }
return
}
val status = executeWithStatus(getStoreDatabase().database, block = {
catalog.update({
(catalog.id eq catalogId)
val status = executeWithStatus(getStoreDatabase().database, run = {
CatalogTable.update({
(CatalogTable.id eq catalogId)
}) {
it[catalog.iid] = iid
it[CatalogTable.iid] = iid
}
}, {
}, onError = {
log.error { "Failed to store movie ${videoDetails.fileName}: ${it.message}" }
})
if (status) {
@ -107,28 +107,28 @@ object ContentCatalogStore {
log.error { "serieInfo in videoDetails is null!" }
return
}
val status = insertWithSuccess(getStoreDatabase().database, block = {
serie.insert {
val status = insertWithSuccess(getStoreDatabase().database, run = {
SerieTable.insert {
it[title] = serieInfo.episodeTitle
it[episode] = serieInfo.episodeNumber
it[season] = serieInfo.seasonNumber
it[video] = videoDetails.fileName
it[serie.collection] = collection
it[SerieTable.collection] = collection
}
}, onError = {
log.error { "Failed to store serie ${videoDetails.fileName}: ${it.message}" }
})
if (!status) {
log.error { "Failed to insert ${videoDetails.fileName} with episode: ${serieInfo.episodeNumber} and season ${serieInfo.seasonNumber}" }
val finalStatus = insertWithSuccess(getStoreDatabase().database, block = {
serie.insert {
val finalStatus = insertWithSuccess(getStoreDatabase().database, run = {
SerieTable.insert {
it[title] = serieInfo.episodeTitle
it[episode] = serieInfo.episodeNumber
it[season] = 0
it[video] = videoDetails.fileName
it[serie.collection] = collection
it[SerieTable.collection] = collection
}
}, { log.error { "Failed to store serie: ${it.message}" } })
}, onError = { log.error { "Failed to store serie: ${it.message}" } })
if (!finalStatus) {
log.error { "Failed to insert ${videoDetails.fileName} with fallback season 0" }
} else {
@ -157,12 +157,12 @@ object ContentCatalogStore {
private fun getId(title: String, titles: List<String>, collection: String, type: String): Int? {
val ids = withTransaction(getStoreDatabase().database) {
catalog.select {
((catalog.title eq title)
or (catalog.collection eq collection)
or (catalog.title inList titles)) and
(catalog.type eq type)
}.map { it[catalog.id].value }
CatalogTable.select {
((CatalogTable.title eq title)
or (CatalogTable.collection eq collection)
or (CatalogTable.title inList titles)) and
(CatalogTable.type eq type)
}.map { it[CatalogTable.id].value }
} ?: run {
log.warn { "No values found on $title with type $type" }
return null

View File

@ -2,15 +2,18 @@ package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
import no.iktdev.eventi.database.withTransaction
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
import no.iktdev.streamit.library.db.query.GenreQuery
import no.iktdev.streamit.library.db.tables.content.GenreTable
import org.jetbrains.exposed.sql.insertIgnoreAndGetId
object ContentGenresStore {
fun storeAndGetIds(genres: List<String>): String? {
return try {
withTransaction(getStoreDatabase()) {
val gq = GenreQuery( *genres.toTypedArray() )
gq.insertAndGetIds()
gq.getIds().joinToString(",")
val receivedGenreIdMap = genres.associateWith { genreName ->
GenreTable.insertIgnoreAndGetId { it[GenreTable.genre] = genreName }?.value
}
receivedGenreIdMap.values.filterNotNull()
.joinToString(",")
}
} catch (e: Exception) {
e.printStackTrace()

View File

@ -3,17 +3,17 @@ package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
import no.iktdev.mediaprocessing.shared.common.contract.reader.SummaryInfo
import no.iktdev.streamit.library.db.executeOrException
import no.iktdev.streamit.library.db.query.SummaryQuery
import no.iktdev.streamit.library.db.tables.content.SummaryTable
object ContentMetadataStore {
fun storeSummary(catalogId: Int, summaryInfo: SummaryInfo) {
val result = executeOrException(getStoreDatabase().database, block = {
SummaryQuery(
cid = catalogId,
val result = executeOrException(getStoreDatabase().database, run = {
SummaryTable.insertIgnore(
catalogId = catalogId,
language = summaryInfo.language,
description = summaryInfo.summary
).insert()
content = summaryInfo.summary
)
})
}
}

View File

@ -3,8 +3,7 @@ package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
import mu.KotlinLogging
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
import no.iktdev.streamit.library.db.executeWithStatus
import no.iktdev.streamit.library.db.query.SubtitleQuery
import no.iktdev.streamit.library.db.tables.subtitle
import no.iktdev.streamit.library.db.tables.content.SubtitleTable
import org.jetbrains.exposed.sql.insert
import java.io.File
@ -12,8 +11,8 @@ object ContentSubtitleStore {
val log = KotlinLogging.logger {}
fun storeSubtitles(collection: String, destinationFile: File): Boolean {
return executeWithStatus (getStoreDatabase().database, block = {
subtitle.insert {
return executeWithStatus (getStoreDatabase().database, run = {
SubtitleTable.insert {
it[this.associatedWithVideo] = destinationFile.nameWithoutExtension
it[this.language] = destinationFile.parentFile.nameWithoutExtension
it[this.collection] = collection

View File

@ -2,7 +2,7 @@ package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
import no.iktdev.mediaprocessing.shared.common.parsing.NameHelper
import no.iktdev.streamit.library.db.tables.titles
import no.iktdev.streamit.library.db.tables.content.TitleTable
import no.iktdev.streamit.library.db.withTransaction
import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.or
@ -12,19 +12,17 @@ object ContentTitleStore {
fun store(mainTitle: String, otherTitles: List<String>) {
try {
withTransaction(getStoreDatabase().database, block = {
withTransaction(getStoreDatabase().database, run = {
val titlesToUse = otherTitles + listOf(
NameHelper.normalize(mainTitle)
).filter { it != mainTitle }
titlesToUse.forEach { t ->
titles.insertIgnore {
TitleTable.insertIgnore {
it[masterTitle] = mainTitle
it[alternativeTitle] = t
}
}
}, {
})
} catch (e: Exception) {
e.printStackTrace()
@ -32,15 +30,13 @@ object ContentTitleStore {
}
fun findMasterTitles(titleList: List<String>): List<String> {
return withTransaction(getStoreDatabase().database, block = {
titles.select {
(titles.alternativeTitle inList titleList) or
(titles.masterTitle inList titleList)
return withTransaction(getStoreDatabase().database, run = {
TitleTable.select {
(TitleTable.alternativeTitle inList titleList) or
(TitleTable.masterTitle inList titleList)
}.map {
it[titles.masterTitle]
it[TitleTable.masterTitle]
}.distinctBy { it }
}, {
}) ?: emptyList()
}
}

View File

@ -18,16 +18,15 @@ object ProcessedFileStore {
val checksum = getChecksum(inputFilePath)
withTransaction(eventDatabase.database.database, block = {
withTransaction(eventDatabase.database.database, run = {
filesProcessed.insert {
it[this.title] = title
it[this.inputFile] = inputFilePath
it[this.data] = Gson().toJson(summary)
it[this.checksum] = checksum
}
}) {
}, onError = {
it.printStackTrace()
}
})
}
}

View File

@ -3,6 +3,7 @@ package no.iktdev.mediaprocessing.coordinator.watcher
import dev.vishna.watchservice.KWatchEvent.Kind.Deleted
import dev.vishna.watchservice.KWatchEvent.Kind.Initialized
import dev.vishna.watchservice.asWatchChannel
import jakarta.annotation.PreDestroy
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.channels.consumeEach
import kotlinx.coroutines.delay
@ -22,7 +23,6 @@ import org.jetbrains.exposed.sql.insertIgnore
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import java.io.File
import javax.annotation.PreDestroy
interface FileWatcherEvents {

View File

@ -3,3 +3,6 @@ logging.level.org.apache.kafka=INFO
logging.level.root=INFO
logging.level.Exposed=OFF
logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats = WARN
management.endpoints.web.exposure.include=*
management.endpoint.health.show-details=always
management.endpoints.web.base-path=/actuator

View File

@ -1,7 +1,7 @@
plugins {
id("java")
kotlin("plugin.spring") version "1.5.31"
kotlin("jvm") version "1.9.20"
kotlin("jvm") version "2.1.0"
}
group = "no.iktdev.mediaprocessing"

View File

@ -1,6 +1,6 @@
plugins {
id("java")
kotlin("jvm") version "1.9.20"
kotlin("jvm") version "2.1.0"
}
group = "no.iktdev.mediaprocessing"

View File

@ -40,6 +40,7 @@ class FileNameParser(val fileName: String) {
else -> cleanedFileName
}.trim()
.replace(Regex("[-\\s]+$"), "") // fjern trailing "-" og whitespace
}
fun guessDesiredTitle(): String {
@ -52,6 +53,7 @@ class FileNameParser(val fileName: String) {
} else desiredFileName
result.trim()
}.trim('.', '-').trim()
.replace(Regex("[-\\s]+$"), "") // fjern trailing "-" og whitespace
}
fun guessSearchableTitle(): MutableList<String> {

View File

@ -77,8 +77,11 @@ abstract class EventCoordinator<T : EventImpl, E : EventsManagerImpl<T>> {
private suspend fun onEventsReceived(events: List<T>): Boolean = coroutineScope {
val listeners = getListeners()
log.debug("onEventsReceived called with ${events.size} events for referenceId: ${events.firstOrNull()?.referenceId() ?: "unknown"}")
events.forEach { event ->
log.debug { "Processing event: ${event.eventType} with referenceId: ${event.referenceId()}" }
listeners.forEach { listener ->
log.debug { "Checking listener: ${listener::class.java.simpleName} for event: ${event.eventType}" }
if (listener.shouldIProcessAndHandleEvent(event, events)) {
val consumableEvent = ConsumableEvent(event)
listener.onEventsReceived(consumableEvent, events)
@ -218,12 +221,13 @@ abstract class EventCoordinator<T : EventImpl, E : EventsManagerImpl<T>> {
suspend fun waitForConditionOrTimeout(timeout: Long, condition: () -> Boolean) {
val startTime = System.currentTimeMillis()
log.debug("Waiting for condition with timeout: $timeout ms")
try {
withTimeout(timeout) {
while (!condition()) {
delay(100)
if (System.currentTimeMillis() - startTime >= timeout) {
log.debug("Condition not met within timeout: $timeout ms")
break
}
}