diff --git a/.github/workflows/v4.yml b/.github/workflows/v4.yml index 0244704d..389478c2 100644 --- a/.github/workflows/v4.yml +++ b/.github/workflows/v4.yml @@ -14,43 +14,65 @@ jobs: pre-check: runs-on: ubuntu-latest outputs: - pyMetadata: ${{ steps.filter.outputs.pyMetadata }} - coordinator: ${{ steps.filter.outputs.coordinator }} - processer: ${{ steps.filter.outputs.processer }} - converter: ${{ steps.filter.outputs.converter }} - shared: ${{ steps.filter.outputs.shared }} + pyMetadata: ${{ steps.pyMetadataChangesDetected.outputs.changed }} + sharedLibrary: ${{ steps.sharedLibraryChangesDetected.outputs.changed }} + coordinator: ${{ steps.coordinatorChangesDetected.outputs.changed }} + processer: ${{ steps.processerChangesDetected.outputs.changed }} + converter: ${{ steps.converterChangesDetected.outputs.changed }} + ui: ${{ steps.uiChangesDetected.outputs.changed }} steps: - name: Checkout repository uses: actions/checkout@v2 - - uses: dorny/paths-filter@v2 - id: filter + - name: Detect if pyMetadata has changed + id: pyMetadataChangesDetected + uses: tchupp/actions-detect-directory-changes@v1.0.15 with: - filters: | - pyMetadata: - - 'apps/pyMetadata/**' - apps/coordinator: - - 'apps/coordinator/**' - apps/processer: - - 'apps/processer/**' - apps/converter: - - 'apps/converter/**' - - shared: - - 'shared/**' + included-paths: "apps/pyMetadata/**" + + - name: Detect if shared library + id: sharedLibraryChangesDetected + uses: tchupp/actions-detect-directory-changes@v1.0.15 + with: + included-paths: "shared/**" + + - name: Detect if coordinator has changed + id: coordinatorChangesDetected + uses: tchupp/actions-detect-directory-changes@v1.0.15 + with: + included-paths: "apps/coordinator/**" + + - name: Detect if processer has changed + id: processerChangesDetected + uses: tchupp/actions-detect-directory-changes@v1.0.15 + with: + included-paths: "apps/processer/**" + + - name: Detect if converter has changed + id: converterChangesDetected + uses: tchupp/actions-detect-directory-changes@v1.0.15 + with: + included-paths: "apps/converter/**" + + - name: Detect if ui has changed + id: uiChangesDetected + uses: tchupp/actions-detect-directory-changes@v1.0.15 + with: + included-paths: "apps/ui/**" + # Step to print the outputs from "pre-check" job - name: Print Outputs from pre-check job run: | echo "Apps\n" - echo "app:pyMetadata: ${{ needs.pre-check.outputs.pyMetadata }}" - echo "app:coordinator: ${{ needs.pre-check.outputs.coordinator }}" - echo "app:processer: ${{ needs.pre-check.outputs.processer }}" - echo "app:converter: ${{ needs.pre-check.outputs.converter }}" + echo "app:pyMetadata: ${{ steps.pyMetadataChangesDetected.outputs.changed }}" + echo "app:coordinator: ${{ steps.coordinatorChangesDetected.outputs.changed }}" + echo "app:processer: ${{ steps.processerChangesDetected.outputs.changed }}" + echo "app:converter: ${{ steps.converterChangesDetected.outputs.changed }}" + echo "app:ui: ${{ steps.uiChangesDetected.outputs.changed }}" echo "Shared" - echo "shared: ${{ needs.pre-check.outputs.shared }}" - echo "\n" + echo "shared: ${{ steps.sharedLibraryChangesDetected.outputs.changed }}" echo "${{ needs.pre-check.outputs }}" echo "${{ needs.pre-check }}" @@ -69,7 +91,7 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }} - name: Build Shared code - if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.shared == 'true' || github.event_name == 'workflow_dispatch' + if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.sharedLibrary == 'true' || github.event_name == 'workflow_dispatch' run: | chmod +x ./gradlew ./gradlew :shared:build --stacktrace --info @@ -77,7 +99,7 @@ jobs: build-processer: needs: build-shared - if: ${{ needs.pre-check.outputs.processer == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }} + if: ${{ needs.pre-check.outputs.processer == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }} runs-on: ubuntu-latest #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} @@ -134,7 +156,7 @@ jobs: build-converter: needs: build-shared - if: ${{ needs.pre-check.outputs.converter == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }} + if: ${{ needs.pre-check.outputs.converter == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }} runs-on: ubuntu-latest #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} @@ -192,7 +214,7 @@ jobs: build-coordinator: needs: build-shared - if: ${{ needs.pre-check.outputs.coordinator == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }} + if: ${{ needs.pre-check.outputs.coordinator == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }} runs-on: ubuntu-latest #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt index 68d5baec..eaf7f0b4 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt @@ -8,8 +8,11 @@ import no.iktdev.mediaprocessing.shared.common.database.tables.files import no.iktdev.mediaprocessing.shared.common.extended.isSupportedVideoFile import no.iktdev.mediaprocessing.shared.common.md5 import no.iktdev.streamit.library.db.withTransaction +import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.sql.deleteWhere import org.jetbrains.exposed.sql.insertIgnore import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.selectAll import org.springframework.scheduling.annotation.EnableScheduling import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service @@ -23,7 +26,7 @@ class UnattendedIndexing { @Scheduled(fixedDelay = 60_000*60) fun indexContent() { logger.info { "Performing indexing of input root: ${SharedConfig.inputRoot.absolutePath}" } - val fileList = SharedConfig.inputRoot.walkTopDown().filter { it.isFile && it.isSupportedVideoFile() } + val fileList = SharedConfig.inputRoot.walkTopDown().filter { it.isFile && it.isSupportedVideoFile() }.toList() fileList.forEach { file -> withTransaction(eventDatabase.database) { files.insertIgnore { @@ -35,6 +38,16 @@ class UnattendedIndexing { } } logger.info { "Indexing completed" } - + /*val storedFiles = withTransaction(eventDatabase.database) { + files.selectAll() + .mapNotNull { it[files.fileName] } + }?.forEach { file -> + if (!File(file).exists()) { + logger.info { "Detected file no longer existing. Performing removal i db" } + files.deleteWhere { + (fileName eq file) + } + } + }*/ } } \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CompletedTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CompletedTaskListener.kt index 5d01c1b8..eaf688d7 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CompletedTaskListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CompletedTaskListener.kt @@ -147,19 +147,19 @@ class CompletedTaskListener : CoordinatorEventListener() { val newVideoPath = mover.moveVideo() - try { - getVideo(events)?.let { video -> - ContentCatalogStore.storeMedia( - title = mediaInfo.title, - collection = usableCollection, - type = mediaInfo.type, - videoDetails = video - ) - } - } catch (e: Exception) { - e.printStackTrace() + val videoInfo = getVideo(events) + if (videoInfo != null) { + assert(newVideoPath == null) + ContentCatalogStore.storeMedia( + title = mediaInfo.title, + collection = usableCollection, + type = mediaInfo.type, + videoDetails = videoInfo + ) } + + val newSubtitles = mover.moveSubtitles() try { @@ -196,6 +196,8 @@ class CompletedTaskListener : CoordinatorEventListener() { subtitlesMoved = newSubtitles?.map { s -> SubtitlesMoved(s.source, s.destination) } ?: emptyList() ) )) + } else { + log.warn { "Do not produce complete is enabled!" } } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/MetadataWaitOrDefaultTaskListener.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/MetadataWaitOrDefaultTaskListener.kt index 1bdeeb1c..24a9bd56 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/MetadataWaitOrDefaultTaskListener.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/MetadataWaitOrDefaultTaskListener.kt @@ -102,7 +102,10 @@ class MetadataWaitOrDefaultTaskListener() : CoordinatorEventListener() { } expired.forEach { log.info { "Producing timeout for ${it.key} ${LocalDateTime.now()}" } - coordinator?.produceNewEvent( + if (coordinator == null) { + log.error { "Coordinator is null, not able to get timeout stored!" } + } + coordinator!!.produceNewEvent( MediaMetadataReceivedEvent( metadata = EventMetadata( referenceId = it.key, diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt index cd8dd871..02685ac8 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt @@ -148,6 +148,7 @@ object ContentCatalogStore { fun storeMedia(title: String, collection: String, type: String, videoDetails: VideoDetails) { val catalogId = getId(title, collection, type) ?: return + log.info { "$title is identified as $type" } when (type) { "movie" -> storeMovie(catalogId, videoDetails) "serie" -> storeSerie(collection, videoDetails) diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/validator/CompletionValidator.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/validator/CompletionValidator.kt index 16e3e443..182d004b 100644 --- a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/validator/CompletionValidator.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/validator/CompletionValidator.kt @@ -1,6 +1,7 @@ package no.iktdev.mediaprocessing.coordinator.tasksV2.validator import no.iktdev.eventi.data.dataAs +import no.iktdev.eventi.data.isSkipped import no.iktdev.eventi.data.isSuccessful import no.iktdev.mediaprocessing.shared.common.contract.Events import no.iktdev.mediaprocessing.shared.common.contract.data.* @@ -96,6 +97,10 @@ object CompletionValidator { */ fun req4(events: List): Boolean { val metadata = events.find { it.eventType == Events.EventMediaMetadataSearchPerformed } + if (metadata?.isSkipped() == true) { + return true + } + if (metadata?.isSuccessful() != true) { return true } diff --git a/apps/coordinator/src/main/resources/application.properties b/apps/coordinator/src/main/resources/application.properties index 715c1020..775a2d6c 100644 --- a/apps/coordinator/src/main/resources/application.properties +++ b/apps/coordinator/src/main/resources/application.properties @@ -1,4 +1,5 @@ spring.output.ansi.enabled=always logging.level.org.apache.kafka=INFO logging.level.root=INFO -logging.level.Exposed=OFF \ No newline at end of file +logging.level.Exposed=OFF +logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats = WARN \ No newline at end of file diff --git a/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt b/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt index b2d5b621..75f94305 100644 --- a/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt +++ b/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt @@ -187,13 +187,14 @@ abstract class EventCoordinator> { */ fun produceNewEvent(event: T): Boolean { if (doNotProduce) { + log.warn { "Do not produce is enabled!" } newEventProduced = true return true } val isStored = eventManager.storeEvent(event) if (isStored) { - log.info { "Stored event: ${event.eventType}" } + log.debug { "Stored event: ${event.eventType}" } newEventProduced = true } else { log.error { "Failed to store event: ${event.eventType}" }