Logging + warn + now casts exception if not initialized

.github/workflows/v4.yml

@@ -14,43 +14,65 @@ jobs:
   pre-check:
     runs-on: ubuntu-latest
     outputs:
-      pyMetadata: ${{ steps.filter.outputs.pyMetadata }}
+      pyMetadata: ${{ steps.pyMetadataChangesDetected.outputs.changed }}
-      coordinator: ${{ steps.filter.outputs.coordinator }}
+      sharedLibrary: ${{ steps.sharedLibraryChangesDetected.outputs.changed }}
-      processer: ${{ steps.filter.outputs.processer }}
+      coordinator: ${{ steps.coordinatorChangesDetected.outputs.changed }}
-      converter: ${{ steps.filter.outputs.converter }}
+      processer: ${{ steps.processerChangesDetected.outputs.changed }}
-      shared: ${{ steps.filter.outputs.shared }}
+      converter: ${{ steps.converterChangesDetected.outputs.changed }}
+      ui: ${{ steps.uiChangesDetected.outputs.changed }}
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - uses: dorny/paths-filter@v2
+      - name: Detect if pyMetadata has changed
-        id: filter
+        id: pyMetadataChangesDetected
+        uses: tchupp/actions-detect-directory-changes@v1.0.15
         with:
-          filters: |
+          included-paths: "apps/pyMetadata/**"
-            pyMetadata:
+
-              - 'apps/pyMetadata/**'
+      - name: Detect if shared library
-            apps/coordinator:
+        id: sharedLibraryChangesDetected
-              - 'apps/coordinator/**'
+        uses: tchupp/actions-detect-directory-changes@v1.0.15
-            apps/processer:
+        with:
-              - 'apps/processer/**'
+          included-paths: "shared/**"
-            apps/converter:
+
-              - 'apps/converter/**'
+      - name: Detect if coordinator has changed
+        id: coordinatorChangesDetected
+        uses: tchupp/actions-detect-directory-changes@v1.0.15
+        with:
+          included-paths: "apps/coordinator/**"
+
+      - name: Detect if processer has changed
+        id: processerChangesDetected
+        uses: tchupp/actions-detect-directory-changes@v1.0.15
+        with:
+          included-paths: "apps/processer/**"
+
+      - name: Detect if converter has changed
+        id: converterChangesDetected
+        uses: tchupp/actions-detect-directory-changes@v1.0.15
+        with:
+          included-paths: "apps/converter/**"
+
+      - name: Detect if ui has changed
+        id: uiChangesDetected
+        uses: tchupp/actions-detect-directory-changes@v1.0.15
+        with:
+          included-paths: "apps/ui/**"
 
-            shared:
-              - 'shared/**'
       # Step to print the outputs from "pre-check" job
       - name: Print Outputs from pre-check job
         run: |
           echo "Apps\n"
-          echo "app:pyMetadata: ${{ needs.pre-check.outputs.pyMetadata }}"
+          echo "app:pyMetadata: ${{ steps.pyMetadataChangesDetected.outputs.changed }}"
-          echo "app:coordinator: ${{ needs.pre-check.outputs.coordinator }}"
+          echo "app:coordinator: ${{ steps.coordinatorChangesDetected.outputs.changed }}"
-          echo "app:processer: ${{ needs.pre-check.outputs.processer }}"
+          echo "app:processer: ${{ steps.processerChangesDetected.outputs.changed }}"
-          echo "app:converter: ${{ needs.pre-check.outputs.converter }}"
+          echo "app:converter: ${{ steps.converterChangesDetected.outputs.changed }}"
+          echo "app:ui: ${{ steps.uiChangesDetected.outputs.changed }}"
           
           echo "Shared"
-          echo "shared: ${{ needs.pre-check.outputs.shared }}"
+          echo "shared: ${{ steps.sharedLibraryChangesDetected.outputs.changed }}"
-          echo "\n"
           echo "${{ needs.pre-check.outputs }}"
           echo "${{ needs.pre-check }}"
 
@@ -69,7 +91,7 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
 
       - name: Build Shared code
-        if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.shared == 'true' || github.event_name == 'workflow_dispatch'
+        if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.sharedLibrary == 'true' || github.event_name == 'workflow_dispatch'
         run: |
           chmod +x ./gradlew
           ./gradlew :shared:build --stacktrace --info
@@ -77,7 +99,7 @@ jobs:
 
   build-processer:
     needs: build-shared
-    if: ${{ needs.pre-check.outputs.processer == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }}
+    if: ${{ needs.pre-check.outputs.processer == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
     runs-on: ubuntu-latest
     #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
 
@@ -134,7 +156,7 @@ jobs:
 
   build-converter:
     needs: build-shared
-    if: ${{ needs.pre-check.outputs.converter == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }}
+    if: ${{ needs.pre-check.outputs.converter == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
     runs-on: ubuntu-latest
     #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
 
@@ -192,7 +214,7 @@ jobs:
             
   build-coordinator:
     needs: build-shared
-    if: ${{ needs.pre-check.outputs.coordinator == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }}
+    if: ${{ needs.pre-check.outputs.coordinator == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
     runs-on: ubuntu-latest
     #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
 

apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt

@@ -8,8 +8,11 @@ import no.iktdev.mediaprocessing.shared.common.database.tables.files
 import no.iktdev.mediaprocessing.shared.common.extended.isSupportedVideoFile
 import no.iktdev.mediaprocessing.shared.common.md5
 import no.iktdev.streamit.library.db.withTransaction
+import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
+import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insertIgnore
 import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.selectAll
 import org.springframework.scheduling.annotation.EnableScheduling
 import org.springframework.scheduling.annotation.Scheduled
 import org.springframework.stereotype.Service
@@ -23,7 +26,7 @@ class UnattendedIndexing {
     @Scheduled(fixedDelay = 60_000*60)
     fun indexContent() {
         logger.info { "Performing indexing of input root: ${SharedConfig.inputRoot.absolutePath}" }
-        val fileList =  SharedConfig.inputRoot.walkTopDown().filter { it.isFile && it.isSupportedVideoFile() }
+        val fileList =  SharedConfig.inputRoot.walkTopDown().filter { it.isFile && it.isSupportedVideoFile() }.toList()
         fileList.forEach { file ->
             withTransaction(eventDatabase.database) {
                 files.insertIgnore {
@@ -35,6 +38,16 @@ class UnattendedIndexing {
             }
         }
         logger.info { "Indexing completed" }
-
+        /*val storedFiles = withTransaction(eventDatabase.database) {
+            files.selectAll()
+                .mapNotNull { it[files.fileName] }
+        }?.forEach { file ->
+            if (!File(file).exists()) {
+                logger.info { "Detected file no longer existing. Performing removal i db" }
+                files.deleteWhere {
+                    (fileName eq file)
+                }
+            }
+        }*/
     }
 }

apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CompletedTaskListener.kt

@@ -147,19 +147,19 @@ class CompletedTaskListener : CoordinatorEventListener() {
 
 
         val newVideoPath = mover.moveVideo()
-        try {
+        val videoInfo = getVideo(events)
-            getVideo(events)?.let { video ->
+        if (videoInfo != null) {
-                ContentCatalogStore.storeMedia(
+            assert(newVideoPath == null)
-                    title = mediaInfo.title,
+            ContentCatalogStore.storeMedia(
-                    collection = usableCollection,
+                title = mediaInfo.title,
-                    type = mediaInfo.type,
+                collection = usableCollection,
-                    videoDetails = video
+                type = mediaInfo.type,
-                )
+                videoDetails = videoInfo
-            }
+            )
-        } catch (e: Exception) {
-            e.printStackTrace()
         }
 
+
+
         val newSubtitles = mover.moveSubtitles()
 
         try {
@@ -196,6 +196,8 @@ class CompletedTaskListener : CoordinatorEventListener() {
                     subtitlesMoved = newSubtitles?.map { s -> SubtitlesMoved(s.source, s.destination) } ?: emptyList()
                 )
             ))
+        } else {
+            log.warn { "Do not produce complete is enabled!" }
         }
 
 

apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/MetadataWaitOrDefaultTaskListener.kt

@@ -102,7 +102,10 @@ class MetadataWaitOrDefaultTaskListener() : CoordinatorEventListener() {
         }
         expired.forEach {
             log.info { "Producing timeout for ${it.key} ${LocalDateTime.now()}" }
-            coordinator?.produceNewEvent(
+            if (coordinator == null) {
+                log.error { "Coordinator is null, not able to get timeout stored!" }
+            }
+            coordinator!!.produceNewEvent(
                 MediaMetadataReceivedEvent(
                     metadata = EventMetadata(
                         referenceId = it.key,

apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt

@@ -148,6 +148,7 @@ object ContentCatalogStore {
 
     fun storeMedia(title: String, collection: String, type: String, videoDetails: VideoDetails) {
         val catalogId = getId(title, collection, type) ?: return
+        log.info { "$title is identified as $type" }
         when (type) {
             "movie" -> storeMovie(catalogId, videoDetails)
             "serie" -> storeSerie(collection, videoDetails)

apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/validator/CompletionValidator.kt

@@ -1,6 +1,7 @@
 package no.iktdev.mediaprocessing.coordinator.tasksV2.validator
 
 import no.iktdev.eventi.data.dataAs
+import no.iktdev.eventi.data.isSkipped
 import no.iktdev.eventi.data.isSuccessful
 import no.iktdev.mediaprocessing.shared.common.contract.Events
 import no.iktdev.mediaprocessing.shared.common.contract.data.*
@@ -96,6 +97,10 @@ object CompletionValidator {
      */
     fun req4(events: List<Event>): Boolean {
         val metadata = events.find { it.eventType == Events.EventMediaMetadataSearchPerformed }
+        if (metadata?.isSkipped() == true) {
+            return true
+        }
+
         if (metadata?.isSuccessful() != true) {
             return true
         }

apps/coordinator/src/main/resources/application.properties

@@ -2,3 +2,4 @@ spring.output.ansi.enabled=always
 logging.level.org.apache.kafka=INFO
 logging.level.root=INFO
 logging.level.Exposed=OFF
+logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats = WARN

shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt

@@ -187,13 +187,14 @@ abstract class EventCoordinator<T : EventImpl, E : EventsManagerImpl<T>> {
      */
     fun produceNewEvent(event: T): Boolean {
         if (doNotProduce) {
+            log.warn { "Do not produce is enabled!" }
             newEventProduced = true
             return true
         }
 
         val isStored = eventManager.storeEvent(event)
         if (isStored) {
-            log.info { "Stored event: ${event.eventType}" }
+            log.debug { "Stored event: ${event.eventType}" }
             newEventProduced = true
         } else {
             log.error { "Failed to store event: ${event.eventType}" }