v5 init
This commit is contained in:
parent
ef9aeb1dd2
commit
69ae0ba5ab
360
.idea/workspace.xml
generated
360
.idea/workspace.xml
generated
@ -5,25 +5,193 @@
|
||||
</component>
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="79fc3e25-8083-4c15-b17f-a1e0d61c77a6" name="Changes" comment="Publishing reason for failure">
|
||||
<change afterPath="$PROJECT_DIR$/.idea/runConfigurations/UIApplicationKt.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/.idea/gradle.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/gradle.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/.idea/kotlinc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/kotlinc.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/build.gradle.kts" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/build.gradle.kts" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentGenresStore.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentGenresStore.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentMetadataStore.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentMetadataStore.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentSubtitleStore.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentSubtitleStore.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentTitleStore.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentTitleStore.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ProcessedFileStore.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ProcessedFileStore.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/watcher/InputDirectoryWatcher.kt" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/watcher/InputDirectoryWatcher.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/resources/application.properties" beforeDir="false" afterPath="$PROJECT_DIR$/apps/coordinator/src/main/resources/application.properties" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/build.gradle.kts" beforeDir="false" afterPath="$PROJECT_DIR$/build.gradle.kts" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/build.gradle.kts" beforeDir="false" afterPath="$PROJECT_DIR$/shared/build.gradle.kts" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParser.kt" beforeDir="false" afterPath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParser.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt" beforeDir="false" afterPath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterApplication.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/Implementations.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/TaskCoordinator.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/Converter2.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/tasks/ConvertService.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorEventCoordinator.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorEventListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorUtils.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/EventsDatabase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Implementations.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/ActionEventController.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/InfoController.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/PollController.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/controller/RequestEventController.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/coordination/ProcesserSocketMessageListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/services/UnattendedIndexing.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/implementations/WorkTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/BaseInfoFromFileTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CompletedTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/ConvertWorkTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CoverDownloadTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/CoverFromMetadataTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/EncodeWorkArgumentsTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/EncodeWorkTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/ExtractWorkArgumentsTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/ExtractWorkTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/MediaOutInformationTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/MetadataWaitOrDefaultTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/ParseMediaFileStreamsTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/PersistContentTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/listeners/ReadMediaFileStreamsTaskListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/EncodeWorkArgumentsMapping.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/EventsSummaryMapping.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/ExtractWorkArgumentsMapping.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/FFmpegBase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCatalogStore.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentCompletionMover.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentGenresStore.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentMetadataStore.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentSubtitleStore.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ContentTitleStore.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/store/ProcessedFileStore.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/AudioArguments.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/SubtitleArguments.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/mapping/streams/VideoArguments.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasksV2/validator/CompletionValidator.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/watcher/FileWatcherQueue.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/watcher/InputDirectoryWatcher.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/Implementations.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ProcesserApplication.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ProcesserEnv.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/Reporter.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/TaskCoordinator.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/controller/CancelController.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ffmpeg/FfmpegArgumentsBuilder.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ffmpeg/FfmpegListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ffmpeg/FfmpegRunner.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ffmpeg/FfmpegTaskService.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ffmpeg/progress/FfmpegDecodedProgress.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/ffmpeg/progress/FfmpegProgressDecoder.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/services/EncodeService.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/processer/src/main/kotlin/no/iktdev/mediaprocessing/processer/services/ExtractService.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/Configuration.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/UIApplication.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/UIEnv.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/dto/EventChain.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/dto/EventDataDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/dto/EventSummary.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/dto/explore/ExplorerAttr.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/dto/explore/ExplorerCursor.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/explorer/ExplorerCore.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/service/CompletedEventsService.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/ChainedEventsTopic.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/ExplorerTopic.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/FileRequestTopic.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/ProcesserTasksTopic.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/SocketClient.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/SocketListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/SocketMessageHandler.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/UnprocessedFilesTopic.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/apps/ui/src/main/kotlin/no/iktdev/mediaprocessing/ui/socket/a2a/ProcesserListenerService.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Defaults.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DownloadClient.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/LogHelper.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/Preference.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/SharedConfig.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/TaskCoordinatorBase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/Events.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/EventsListenerContract.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/EventsManagerContract.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/EventsUtil.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ProcessType.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/BaseInfoEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/ConvertWorkCreatedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/ConvertWorkPerformed.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/EncodeArgumentCreatedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/EncodeWorkCreatedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/EncodeWorkPerformedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/Event.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/ExtractArgumentCreatedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/ExtractWorkCreatedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/ExtractWorkPerformedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaCoverDownloadedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaCoverInfoReceivedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaFileStreamsParsedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaFileStreamsReadEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaMetadataReceivedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaOutInformationConstructedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaProcessCompletedEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/MediaProcessStartEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/PermitWorkCreationEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/data/PersistedContentEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/dto/Enums.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/dto/EventRequest.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/dto/EventSummary.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/dto/ProcesserEventInfo.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/dto/RequestWorkProceed.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/dto/Requester.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/dto/tasks/TaskData.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/AudioArgumentsDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/MediaStreams.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/PreferenceDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/SubtitleArgumentsDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/VideoAndAudioDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/ffmpeg/VideoArgumentsDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/reader/MetadataDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/reader/SubtitlesDto.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/contract/reader/VideoDetails.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/EventsDatabase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/cal/EventsManager.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/cal/RunnerManager.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/cal/TasksManager.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/allEvents.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/events.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/files.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/filesProcessed.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/runners.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/database/tables/tasks.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/extended/FileExt.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameDeterminate.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/FileNameParser.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/NameHelper.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/parsing/Regexes.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/runner/ResultRunner.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/services/TaskService.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/socket/SocketImplementation.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/task/Task.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/task/TaskDoz.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/task/TaskType.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/build.gradle.kts" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/LogHelper.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/core/ConsumableEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/core/LocalDateTimeAdapter.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/core/PersistentMessageHelper.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/core/WGson.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/data/EventImpl.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/database/DataSource.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/database/DatabaseConnectionConfig.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/database/MySqlDataSource.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/database/TableDefaultOperations.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventCoordinator.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventListenerImpl.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/main/kotlin/no/iktdev/eventi/implementations/EventsManagerImpl.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/EventiApplication.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/EventiApplicationTests.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/EventiImplementationBase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/TestConfig.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/MockDataEventListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/MockDataSource.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/MockEventCoordinator.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/MockEventManager.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/data/FirstEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/data/InitEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/data/SecondEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/data/ThirdEvent.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/listeners/FirstEventListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/listeners/ForthEventListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/listeners/SecondEventListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/mock/listeners/ThirdEventListener.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/tests/FirstEventListenerImplTestBase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/tests/ForthEventListenerImplTestBase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/tests/SecondEventListenerImplTestBase.kt" beforeDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/shared/eventi/src/test/kotlin/no/iktdev/eventi/tests/ThirdEventListenerImplTestBase.kt" beforeDir="false" />
|
||||
</list>
|
||||
<option name="SHOW_DIALOG" value="false" />
|
||||
<option name="HIGHLIGHT_CONFLICTS" value="true" />
|
||||
@ -105,7 +273,7 @@
|
||||
<component name="Git.Settings">
|
||||
<option name="RECENT_BRANCH_BY_REPOSITORY">
|
||||
<map>
|
||||
<entry key="$PROJECT_DIR$" value="v4-ui" />
|
||||
<entry key="$PROJECT_DIR$" value="v4" />
|
||||
</map>
|
||||
</option>
|
||||
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
|
||||
@ -123,81 +291,81 @@
|
||||
<option name="hideEmptyMiddlePackages" value="true" />
|
||||
<option name="showLibraryContents" value="true" />
|
||||
</component>
|
||||
<component name="PropertiesComponent">{
|
||||
"keyToString": {
|
||||
"Gradle.AudioArgumentsTest.executor": "Run",
|
||||
"Gradle.AudioArgumentsTest.validateChecks1.executor": "Run",
|
||||
"Gradle.AudioArgumentsTest.validateChecks3.executor": "Run",
|
||||
"Gradle.Build MediaProcessing2.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validateCreationOfConvertTasks.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validateParsingOfEvents.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validate_shouldIProcessAndHandleEvent1.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validate_shouldIProcessAndHandleEvent2.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validate_shouldIProcessAndHandleEvent3.executor": "Run",
|
||||
"Gradle.DatabaseDeserializerTest.executor": "Run",
|
||||
"Gradle.DatabaseDeserializerTest.validateMediaInfo.executor": "Run",
|
||||
"Gradle.DatabaseDeserializerTest.validateMetadataRead.executor": "Debug",
|
||||
"Gradle.DatabaseDeserializerTest.validateParsingOfStartEvent.executor": "Run",
|
||||
"Gradle.Download Sources.executor": "Run",
|
||||
"Gradle.EncodeArgumentCreatorTaskTest.executor": "Run",
|
||||
"Gradle.FfmpegProgressDecoderTest.executor": "Debug",
|
||||
"Gradle.FfmpegProgressDecoderTest.parseReadout1.executor": "Debug",
|
||||
"Gradle.FileNameParserTest.assertDotRemoval.executor": "Run",
|
||||
"Gradle.FileNameParserTest.assertTitleFails.executor": "Run",
|
||||
"Gradle.FileNameParserTest.executor": "Run",
|
||||
"Gradle.MediaProcessing2 [apps:build].executor": "Run",
|
||||
"Gradle.MediaProcessing2 [apps:clean].executor": "Run",
|
||||
"Gradle.MediaProcessing2 [build].executor": "Run",
|
||||
"Gradle.MediaProcessing2 [clean].executor": "Run",
|
||||
"Gradle.MediaProcessing2:apps [assemble].executor": "Run",
|
||||
"Gradle.MediaProcessing2:apps:coordinator [:apps:coordinator:no.iktdev.mediaprocessing.coordinator.CoordinatorApplicationKt.main()].executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.validate_shouldIProcessAndHandleEvent1.executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.validate_shouldIProcessAndHandleEvent2.executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.validate_shouldIProcessAndHandleEvent3.executor": "Debug",
|
||||
"Gradle.ParseMediaFileStreamsTaskListenerTest.testParse.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.assertThatCommentaryIsNotSelected.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.assertThatCorrectTrackIsSelected.executor": "Debug",
|
||||
"Gradle.SubtitleArgumentsTest.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate1.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate2.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate2_2.executor": "Debug",
|
||||
"Gradle.SubtitleArgumentsTest.validate3.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate3_2.executor": "Debug",
|
||||
"Gradle.Tests in 'MediaProcessing.apps.coordinator.test'.executor": "Run",
|
||||
"Gradle.Tests in 'MediaProcessing.apps.processer'.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.h264Stream1.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.h264Stream2.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.hevcStream1.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.hevcStream2.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.vc1Stream.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.vc1Stream2.executor": "Run",
|
||||
"JUnit.All in MediaProcessing.executor": "Run",
|
||||
"Kotlin.Env - CoordinatorApplicationKt.executor": "Debug",
|
||||
"Kotlin.UIApplicationKt.executor": "Run",
|
||||
"RunOnceActivity.OpenProjectViewOnStart": "true",
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true",
|
||||
"RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true",
|
||||
"RunOnceActivity.git.unshallow": "true",
|
||||
"SHARE_PROJECT_CONFIGURATION_FILES": "true",
|
||||
"com.intellij.testIntegration.createTest.CreateTestDialog.defaultLibrary": "JUnit5",
|
||||
"com.intellij.testIntegration.createTest.CreateTestDialog.defaultLibrarySuperClass.JUnit5": "",
|
||||
"git-widget-placeholder": "v4",
|
||||
"ignore.virus.scanning.warn.message": "true",
|
||||
"kotlin-language-version-configured": "true",
|
||||
"last_opened_file_path": "/mount/870 EVO 1TB/Workspace/MediaProcessing2",
|
||||
"project.structure.last.edited": "Modules",
|
||||
"project.structure.proportion": "0.0",
|
||||
"project.structure.side.proportion": "0.0",
|
||||
"settings.editor.selected.configurable": "reference.settingsdialog.project.gradle"
|
||||
<component name="PropertiesComponent"><![CDATA[{
|
||||
"keyToString": {
|
||||
"Gradle.AudioArgumentsTest.executor": "Run",
|
||||
"Gradle.AudioArgumentsTest.validateChecks1.executor": "Run",
|
||||
"Gradle.AudioArgumentsTest.validateChecks3.executor": "Run",
|
||||
"Gradle.Build MediaProcessing2.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validateCreationOfConvertTasks.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validateParsingOfEvents.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validate_shouldIProcessAndHandleEvent1.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validate_shouldIProcessAndHandleEvent2.executor": "Run",
|
||||
"Gradle.ConvertWorkTaskListenerTest.validate_shouldIProcessAndHandleEvent3.executor": "Run",
|
||||
"Gradle.DatabaseDeserializerTest.executor": "Run",
|
||||
"Gradle.DatabaseDeserializerTest.validateMediaInfo.executor": "Run",
|
||||
"Gradle.DatabaseDeserializerTest.validateMetadataRead.executor": "Debug",
|
||||
"Gradle.DatabaseDeserializerTest.validateParsingOfStartEvent.executor": "Run",
|
||||
"Gradle.Download Sources.executor": "Run",
|
||||
"Gradle.EncodeArgumentCreatorTaskTest.executor": "Run",
|
||||
"Gradle.FfmpegProgressDecoderTest.executor": "Debug",
|
||||
"Gradle.FfmpegProgressDecoderTest.parseReadout1.executor": "Debug",
|
||||
"Gradle.FileNameParserTest.assertDotRemoval.executor": "Run",
|
||||
"Gradle.FileNameParserTest.assertTitleFails.executor": "Run",
|
||||
"Gradle.FileNameParserTest.executor": "Run",
|
||||
"Gradle.MediaProcessing2 [apps:build].executor": "Run",
|
||||
"Gradle.MediaProcessing2 [apps:clean].executor": "Run",
|
||||
"Gradle.MediaProcessing2 [build].executor": "Run",
|
||||
"Gradle.MediaProcessing2 [clean].executor": "Run",
|
||||
"Gradle.MediaProcessing2:apps [assemble].executor": "Run",
|
||||
"Gradle.MediaProcessing2:apps:coordinator [:apps:coordinator:no.iktdev.mediaprocessing.coordinator.CoordinatorApplicationKt.main()].executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.validate_shouldIProcessAndHandleEvent1.executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.validate_shouldIProcessAndHandleEvent2.executor": "Run",
|
||||
"Gradle.MetadataWaitOrDefaultTaskListenerTest.validate_shouldIProcessAndHandleEvent3.executor": "Debug",
|
||||
"Gradle.ParseMediaFileStreamsTaskListenerTest.testParse.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.assertThatCommentaryIsNotSelected.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.assertThatCorrectTrackIsSelected.executor": "Debug",
|
||||
"Gradle.SubtitleArgumentsTest.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate1.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate2.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate2_2.executor": "Debug",
|
||||
"Gradle.SubtitleArgumentsTest.validate3.executor": "Run",
|
||||
"Gradle.SubtitleArgumentsTest.validate3_2.executor": "Debug",
|
||||
"Gradle.Tests in 'MediaProcessing.apps.coordinator.test'.executor": "Run",
|
||||
"Gradle.Tests in 'MediaProcessing.apps.processer'.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.h264Stream1.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.h264Stream2.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.hevcStream1.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.hevcStream2.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.vc1Stream.executor": "Run",
|
||||
"Gradle.VideoArgumentsTest.vc1Stream2.executor": "Run",
|
||||
"JUnit.All in MediaProcessing.executor": "Run",
|
||||
"Kotlin.Env - CoordinatorApplicationKt.executor": "Debug",
|
||||
"Kotlin.UIApplicationKt.executor": "Run",
|
||||
"RunOnceActivity.OpenProjectViewOnStart": "true",
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true",
|
||||
"RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true",
|
||||
"RunOnceActivity.git.unshallow": "true",
|
||||
"SHARE_PROJECT_CONFIGURATION_FILES": "true",
|
||||
"com.intellij.testIntegration.createTest.CreateTestDialog.defaultLibrary": "JUnit5",
|
||||
"com.intellij.testIntegration.createTest.CreateTestDialog.defaultLibrarySuperClass.JUnit5": "",
|
||||
"git-widget-placeholder": "v5",
|
||||
"ignore.virus.scanning.warn.message": "true",
|
||||
"kotlin-language-version-configured": "true",
|
||||
"last_opened_file_path": "/mount/870 EVO 1TB/Workspace/mediaprocesserv5",
|
||||
"project.structure.last.edited": "Modules",
|
||||
"project.structure.proportion": "0.0",
|
||||
"project.structure.side.proportion": "0.0",
|
||||
"settings.editor.selected.configurable": "reference.settingsdialog.project.gradle"
|
||||
},
|
||||
"keyToStringList": {
|
||||
"com.intellij.ide.scratch.ScratchImplUtil$2/New Scratch File": [
|
||||
"JSON"
|
||||
"keyToStringList": {
|
||||
"com.intellij.ide.scratch.ScratchImplUtil$2/New Scratch File": [
|
||||
"JSON"
|
||||
]
|
||||
}
|
||||
}</component>
|
||||
}]]></component>
|
||||
<component name="RecentsManager">
|
||||
<key name="CopyFile.RECENT_KEYS">
|
||||
<recent name="D:\Workspace\MediaProcessing2\apps\processer\src\test\kotlin\no\iktdev\mediaprocessing\processer" />
|
||||
@ -368,7 +536,6 @@
|
||||
<item itemvalue="Gradle.VideoArgumentsTest.vc1Stream" />
|
||||
<item itemvalue="Gradle.VideoArgumentsTest.vc1Stream2" />
|
||||
<item itemvalue="Kotlin.UIApplicationKt" />
|
||||
<item itemvalue="Kotlin.Env - CoordinatorApplicationKt" />
|
||||
</list>
|
||||
<recent_temporary>
|
||||
<list>
|
||||
@ -1112,15 +1279,6 @@
|
||||
<line>32</line>
|
||||
<option name="timeStamp" value="189" />
|
||||
</line-breakpoint>
|
||||
<line-breakpoint enabled="true" type="kotlin-function">
|
||||
<url>file://$PROJECT_DIR$/apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/TaskCoordinator.kt</url>
|
||||
<line>48</line>
|
||||
<properties class="no.iktdev.mediaprocessing.converter.TaskCoordinator" method="pullForAvailableTasks">
|
||||
<option name="EMULATED" value="true" />
|
||||
<option name="WATCH_EXIT" value="false" />
|
||||
</properties>
|
||||
<option name="timeStamp" value="191" />
|
||||
</line-breakpoint>
|
||||
</breakpoints>
|
||||
</breakpoint-manager>
|
||||
</component>
|
||||
|
||||
@ -1,71 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||
import no.iktdev.exfl.observable.Observables
|
||||
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||
import no.iktdev.eventi.database.MySqlDataSource
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.runners
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.tasks
|
||||
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
||||
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||
import org.springframework.boot.runApplication
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import javax.annotation.PreDestroy
|
||||
|
||||
@SpringBootApplication
|
||||
class ConvertApplication {
|
||||
@PreDestroy
|
||||
fun onShutdown() {
|
||||
doTransactionalCleanup()
|
||||
}
|
||||
|
||||
@Transactional
|
||||
fun doTransactionalCleanup() {
|
||||
runnerManager.unlist()
|
||||
}
|
||||
}
|
||||
|
||||
val ioCoroutine = CoroutinesIO()
|
||||
val defaultCoroutine = CoroutinesDefault()
|
||||
|
||||
lateinit var taskManager: TasksManager
|
||||
lateinit var runnerManager: RunnerManager
|
||||
|
||||
|
||||
private lateinit var eventsDatabase: MySqlDataSource
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
fun getEventsDatabase(): MySqlDataSource {
|
||||
return eventsDatabase
|
||||
}
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
defaultCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
eventsDatabase = DatabaseEnvConfig.toEventsDatabase()
|
||||
eventsDatabase.createDatabase()
|
||||
eventsDatabase.createTables(tasks, runners)
|
||||
taskManager = TasksManager(eventsDatabase)
|
||||
|
||||
runnerManager = RunnerManager(dataSource = getEventsDatabase(), applicationName = ConvertApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
|
||||
runApplication<ConvertApplication>(*args)
|
||||
log.info { "App Version: ${getAppVersion()}" }
|
||||
}
|
||||
//private val logger = KotlinLogging.logger {}
|
||||
@ -1,15 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import no.iktdev.exfl.using
|
||||
import java.io.File
|
||||
|
||||
class ConverterEnv {
|
||||
companion object {
|
||||
val allowOverwrite = System.getenv("ALLOW_OVERWRITE").toBoolean() ?: false
|
||||
val syncDialogs = System.getenv("SYNC_DIALOGS").toBoolean()
|
||||
val outFormats: List<String> = System.getenv("OUT_FORMATS")?.split(",")?.toList() ?: emptyList()
|
||||
|
||||
val logDirectory = if (!System.getenv("LOG_DIR").isNullOrBlank()) File(System.getenv("LOG_DIR")) else
|
||||
File("data").using("logs", "convert")
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.context.annotation.Import
|
||||
|
||||
@Configuration
|
||||
class SocketLocalInit: SocketImplementation()
|
||||
@ -1,92 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.*
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.ActiveMode
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
class TaskCoordinator(): TaskCoordinatorBase() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
override fun onProduceEvent(event: Event) {
|
||||
taskManager.produceEvent(event)
|
||||
}
|
||||
|
||||
override fun onCoordinatorReady() {
|
||||
super.onCoordinatorReady()
|
||||
runnerManager = RunnerManager(dataSource = getEventsDatabase(), applicationName = ConvertApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
}
|
||||
|
||||
|
||||
override val taskAvailabilityEventListener: MutableMap<TaskType, MutableList<TaskQueueListener>> = mutableMapOf(
|
||||
TaskType.Convert to mutableListOf(),
|
||||
)
|
||||
|
||||
private val taskListeners: MutableSet<TaskEvents> = mutableSetOf()
|
||||
fun getTaskListeners(): List<TaskEvents> {
|
||||
return taskListeners.toList()
|
||||
}
|
||||
fun addTaskEventListener(listener: TaskEvents) {
|
||||
taskListeners.add(listener)
|
||||
}
|
||||
|
||||
fun addConvertTaskListener(listener: TaskQueueListener) {
|
||||
addTaskListener(TaskType.Convert, listener)
|
||||
}
|
||||
|
||||
override fun addTaskListener(type: TaskType, listener: TaskQueueListener) {
|
||||
super.addTaskListener(type, listener)
|
||||
pullForAvailableTasks()
|
||||
}
|
||||
|
||||
|
||||
override fun pullForAvailableTasks() {
|
||||
if (runnerManager.iAmSuperseded()) {
|
||||
// This will let the application complete but not consume new
|
||||
val prevState = taskMode
|
||||
taskMode = ActiveMode.Passive
|
||||
if (taskMode != prevState && taskMode == ActiveMode.Passive) {
|
||||
log.warn { "A newer version has been detected. Changing mode to $taskMode, no new tasks will be processed" }
|
||||
}
|
||||
return
|
||||
}
|
||||
val available = taskManager.getClaimableTasks().asClaimable()
|
||||
available.forEach { (type, list) ->
|
||||
taskAvailabilityEventListener[type]?.forEach { listener ->
|
||||
list.foreachOrUntilClaimed {
|
||||
listener.onTaskAvailable(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun clearExpiredClaims() {
|
||||
val expiredClaims = taskManager.getTasksWithExpiredClaim().filter { it.task in listOf(TaskType.Convert) }
|
||||
expiredClaims.forEach {
|
||||
log.info { "Found event with expired claim: ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
}
|
||||
expiredClaims.forEach {
|
||||
val result = taskManager.deleteTaskClaim(referenceId = it.referenceId, eventId = it.eventId)
|
||||
if (result) {
|
||||
log.info { "Released claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
} else {
|
||||
log.error { "Failed to release claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun getEnabledState(): Boolean {
|
||||
return runnerManager.amIEnabled()
|
||||
}
|
||||
|
||||
interface TaskEvents {
|
||||
fun onCancelOrStopProcess(eventId: String)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.convert
|
||||
|
||||
interface ConvertListener {
|
||||
fun onStarted(inputFile: String)
|
||||
fun onCompleted(inputFile: String, outputFiles: List<String>)
|
||||
fun onError(inputFile: String, message: String) {}
|
||||
}
|
||||
@ -1,78 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.convert
|
||||
|
||||
import no.iktdev.library.subtitle.Configuration
|
||||
import no.iktdev.library.subtitle.Syncro
|
||||
import no.iktdev.library.subtitle.classes.Dialog
|
||||
import no.iktdev.library.subtitle.classes.DialogType
|
||||
import no.iktdev.library.subtitle.export.Export
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import no.iktdev.library.subtitle.reader.Reader
|
||||
import no.iktdev.mediaprocessing.converter.ConverterEnv
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.SubtitleFormats
|
||||
import java.io.File
|
||||
import kotlin.jvm.Throws
|
||||
|
||||
class Converter2(val data: ConvertData,
|
||||
private val listener: ConvertListener) {
|
||||
|
||||
@Throws(FileUnavailableException::class)
|
||||
private fun getReader(): BaseReader? {
|
||||
val file = File(data.inputFile)
|
||||
if (!file.canRead())
|
||||
throw FileUnavailableException("Can't open file for reading..")
|
||||
return Reader(file).getSubtitleReader()
|
||||
}
|
||||
|
||||
private fun syncDialogs(input: List<Dialog>): List<Dialog> {
|
||||
return if (ConverterEnv.syncDialogs) Syncro().sync(input) else input
|
||||
}
|
||||
|
||||
fun canRead(): Boolean {
|
||||
try {
|
||||
val reader = getReader()
|
||||
return reader != null
|
||||
} catch (e: FileUnavailableException) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(FileUnavailableException::class, FileIsNullOrEmpty::class)
|
||||
fun execute() {
|
||||
val file = File(data.inputFile)
|
||||
listener.onStarted(file.absolutePath)
|
||||
try {
|
||||
Configuration.exportJson = true
|
||||
val read = getReader()?.read() ?: throw FileIsNullOrEmpty()
|
||||
if (read.isEmpty())
|
||||
throw FileIsNullOrEmpty()
|
||||
val filtered = read.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) }
|
||||
val syncOrNotSync = syncDialogs(filtered)
|
||||
|
||||
val exporter = Export(file, File(data.outputDirectory), data.outputFileName)
|
||||
|
||||
val outFiles = if (data.formats.isEmpty()) {
|
||||
exporter.write(syncOrNotSync)
|
||||
} else {
|
||||
val exported = mutableListOf<File>()
|
||||
if (data.formats.contains(SubtitleFormats.SRT)) {
|
||||
exported.add(exporter.writeSrt(syncOrNotSync))
|
||||
}
|
||||
if (data.formats.contains(SubtitleFormats.SMI)) {
|
||||
exported.add(exporter.writeSmi(syncOrNotSync))
|
||||
}
|
||||
if (data.formats.contains(SubtitleFormats.VTT)) {
|
||||
exported.add(exporter.writeVtt(syncOrNotSync))
|
||||
}
|
||||
exported
|
||||
}
|
||||
listener.onCompleted(file.absolutePath, outFiles.map { it.absolutePath })
|
||||
} catch (e: Exception) {
|
||||
listener.onError(file.absolutePath, e.message ?: e.localizedMessage)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException()
|
||||
class FileUnavailableException(override val message: String): RuntimeException()
|
||||
}
|
||||
@ -1,136 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.tasks
|
||||
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.mediaprocessing.converter.*
|
||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
||||
import no.iktdev.mediaprocessing.converter.convert.Converter2
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.Status
|
||||
import no.iktdev.mediaprocessing.shared.common.services.TaskService
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertWorkPerformed
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertedData
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
|
||||
@Service
|
||||
class ConvertService(
|
||||
@Autowired var tasks: TaskCoordinator,
|
||||
) : TaskService(), ConvertListener, TaskCoordinator.TaskEvents {
|
||||
|
||||
fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
override val log = KotlinLogging.logger {}
|
||||
override val logDir = ConverterEnv.logDirectory
|
||||
|
||||
override fun getServiceId(serviceName: String): String {
|
||||
return super.getServiceId(this::class.java.simpleName)
|
||||
}
|
||||
|
||||
|
||||
var worker: Converter2? = null
|
||||
|
||||
override fun onAttachListener() {
|
||||
tasks.addConvertTaskListener(this)
|
||||
tasks.addTaskEventListener(this)
|
||||
}
|
||||
|
||||
|
||||
override fun isReadyToConsume(): Boolean {
|
||||
return worker == null
|
||||
}
|
||||
|
||||
override fun isTaskClaimable(task: Task): Boolean {
|
||||
return !taskManager.isTaskClaimed(referenceId = task.referenceId, eventId = task.eventId)
|
||||
}
|
||||
|
||||
|
||||
override fun onTaskAssigned(task: Task) {
|
||||
startConvert(task)
|
||||
}
|
||||
|
||||
fun startConvert(task: Task) {
|
||||
val convert = task.data as ConvertData
|
||||
worker = Converter2(convert, this)
|
||||
worker?.execute()
|
||||
}
|
||||
|
||||
override fun onStarted(inputFile: String) {
|
||||
val task = assignedTask ?: return
|
||||
taskManager.markTaskAsClaimed(task.referenceId, task.eventId, serviceId)
|
||||
log.info { "Convert started for ${task.referenceId}" }
|
||||
|
||||
}
|
||||
|
||||
|
||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
||||
val task = assignedTask ?: return
|
||||
val taskData: ConvertData = task.data as ConvertData
|
||||
log.info { "Convert completed for ${task.referenceId}" }
|
||||
val claimSuccessful = taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||
|
||||
runBlocking {
|
||||
delay(1000)
|
||||
if (!claimSuccessful) {
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||
delay(1000)
|
||||
}
|
||||
var readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||
while (!readbackIsSuccess) {
|
||||
delay(1000)
|
||||
readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||
}
|
||||
|
||||
tasks.onProduceEvent(ConvertWorkPerformed(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = ConvertedData(
|
||||
language = taskData.language,
|
||||
outputFiles = outputFiles,
|
||||
baseName = taskData.storeFileName
|
||||
)
|
||||
))
|
||||
onClearTask()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onError(inputFile: String, message: String) {
|
||||
val task = assignedTask ?: return
|
||||
super.onError(inputFile, message)
|
||||
log.info { "Convert error for ${task.referenceId}\nmessage: $message" }
|
||||
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId, Status.ERROR, message)
|
||||
|
||||
tasks.onProduceEvent(ConvertWorkPerformed(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Failed,
|
||||
source = getProducerName()
|
||||
)
|
||||
))
|
||||
onClearTask()
|
||||
}
|
||||
|
||||
|
||||
override fun onClearTask() {
|
||||
super.onClearTask()
|
||||
worker = null
|
||||
}
|
||||
|
||||
override fun onCancelOrStopProcess(eventId: String) {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,140 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
|
||||
import jakarta.annotation.PreDestroy
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||
import no.iktdev.exfl.observable.Observables
|
||||
import no.iktdev.mediaprocessing.shared.common.*
|
||||
import no.iktdev.eventi.database.MySqlDataSource
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||
import no.iktdev.streamit.library.db.tables.catalog
|
||||
import no.iktdev.streamit.library.db.tables.genre
|
||||
import no.iktdev.streamit.library.db.tables.helper.data_audio
|
||||
import no.iktdev.streamit.library.db.tables.helper.data_video
|
||||
import no.iktdev.streamit.library.db.tables.movie
|
||||
import no.iktdev.streamit.library.db.tables.serie
|
||||
import no.iktdev.streamit.library.db.tables.subtitle
|
||||
import no.iktdev.streamit.library.db.tables.summary
|
||||
import no.iktdev.streamit.library.db.tables.titles
|
||||
import no.iktdev.streamit.library.db.tables.users
|
||||
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||
import org.springframework.boot.runApplication
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
|
||||
val log = KotlinLogging.logger {}
|
||||
lateinit var eventDatabase: EventsDatabase
|
||||
private lateinit var eventsManager: EventsManager
|
||||
lateinit var runnerManager: RunnerManager
|
||||
|
||||
|
||||
@SpringBootApplication
|
||||
class CoordinatorApplication {
|
||||
|
||||
@Bean
|
||||
fun eventManager(): EventsManager {
|
||||
return eventsManager
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
fun onShutdown() {
|
||||
doTransactionalCleanup()
|
||||
}
|
||||
|
||||
@Transactional
|
||||
fun doTransactionalCleanup() {
|
||||
runnerManager.unlist()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private lateinit var storeDatabase: MySqlDataSource
|
||||
|
||||
val ioCoroutine = CoroutinesIO().
|
||||
also {
|
||||
it.addListener(object : Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
log.error { "IO Coroutine" + value.printStackTrace() }
|
||||
}
|
||||
})
|
||||
}
|
||||
val defaultCoroutine = CoroutinesDefault().
|
||||
also {
|
||||
it.addListener(object : Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
log.error { "Default Coroutine" + value.printStackTrace() }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
fun getStoreDatabase(): MySqlDataSource {
|
||||
return storeDatabase
|
||||
}
|
||||
|
||||
lateinit var taskManager: TasksManager
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
|
||||
|
||||
printSharedConfig()
|
||||
|
||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
defaultCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
eventDatabase = EventsDatabase().also {
|
||||
eventsManager = EventsManager(it.database)
|
||||
}
|
||||
|
||||
|
||||
|
||||
storeDatabase = DatabaseEnvConfig.toStoredDatabase()
|
||||
storeDatabase.createDatabase()
|
||||
|
||||
|
||||
taskManager = TasksManager(eventDatabase.database)
|
||||
|
||||
|
||||
val tables = arrayOf(
|
||||
catalog,
|
||||
genre,
|
||||
movie,
|
||||
serie,
|
||||
subtitle,
|
||||
summary,
|
||||
users,
|
||||
data_audio,
|
||||
data_video,
|
||||
titles
|
||||
)
|
||||
storeDatabase.createTables(*tables)
|
||||
|
||||
runnerManager = RunnerManager(dataSource = eventDatabase.database, applicationName = CoordinatorApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
|
||||
runApplication<CoordinatorApplication>(*args)
|
||||
log.info { "App Version: ${getAppVersion()}" }
|
||||
}
|
||||
|
||||
fun printSharedConfig() {
|
||||
log.info { "File Input: ${SharedConfig.incomingContent}" }
|
||||
log.info { "File Output: ${SharedConfig.outgoingContent}" }
|
||||
log.info { "Ffprobe: ${SharedConfig.ffprobe}" }
|
||||
log.info { "Ffmpeg: ${SharedConfig.ffmpeg}" }
|
||||
|
||||
/*log.info { "Database: ${DatabaseConfig.database} @ ${DatabaseConfig.address}:${DatabaseConfig.port}" }
|
||||
log.info { "Username: ${DatabaseConfig.username}" }
|
||||
log.info { "Password: ${if (DatabaseConfig.password.isNullOrBlank()) "Is not set" else "Is set"}" }*/
|
||||
}
|
||||
@ -1,107 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.eventId
|
||||
import no.iktdev.eventi.implementations.ActiveMode
|
||||
import no.iktdev.eventi.implementations.EventCoordinator
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaProcessStartEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.PermitWorkCreationEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.StartEventData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent
|
||||
import org.springframework.context.ApplicationContext
|
||||
import org.springframework.context.event.EventListener
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class Coordinator(
|
||||
@Autowired
|
||||
override var applicationContext: ApplicationContext,
|
||||
@Autowired
|
||||
override var eventManager: EventsManager
|
||||
|
||||
) : EventCoordinator<Event, EventsManager>() {
|
||||
|
||||
@EventListener(ApplicationReadyEvent::class)
|
||||
fun onApplicationReady() {
|
||||
onReady()
|
||||
}
|
||||
|
||||
fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
public fun startProcess(file: File, type: ProcessType) {
|
||||
val operations: List<OperationEvents> = listOf(
|
||||
OperationEvents.ENCODE,
|
||||
OperationEvents.EXTRACT,
|
||||
OperationEvents.CONVERT
|
||||
)
|
||||
startProcess(file, type, operations)
|
||||
}
|
||||
|
||||
fun startProcess(file: File, type: ProcessType, operations: List<OperationEvents>): UUID {
|
||||
val referenceId: UUID = UUID.randomUUID()
|
||||
val event = MediaProcessStartEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = referenceId.toString(),
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = StartEventData(
|
||||
file = file.absolutePath,
|
||||
type = type,
|
||||
operations = operations
|
||||
)
|
||||
)
|
||||
|
||||
produceNewEvent(event)
|
||||
return referenceId
|
||||
}
|
||||
|
||||
fun permitWorkToProceedOn(referenceId: String, events: List<Event>, message: String) {
|
||||
val defaultRequiredBy = listOf(Events.EncodeParameterCreated, Events.ExtractParameterCreated)
|
||||
val eventToAttachTo = if (events.any { it.eventType in defaultRequiredBy }) {
|
||||
events.findLast { it.eventType in defaultRequiredBy }
|
||||
} else events.find { it.eventType == Events.ProcessStarted }
|
||||
if (eventToAttachTo == null) {
|
||||
log.error { "No event to attach permit to" }
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
produceNewEvent(
|
||||
PermitWorkCreationEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = referenceId,
|
||||
derivedFromEventId = eventToAttachTo.eventId(),
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = message
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override fun getActiveTaskMode(): ActiveMode {
|
||||
if (runnerManager.iAmSuperseded()) {
|
||||
// This will let the application complete but not consume new
|
||||
taskMode = ActiveMode.Passive
|
||||
}
|
||||
return taskMode
|
||||
}
|
||||
|
||||
override fun updateEnabledState(): Boolean {
|
||||
isEnabled = runnerManager.amIEnabled()
|
||||
return isEnabled
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.EventsListenerContract
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
|
||||
abstract class CoordinatorEventListener(): EventsListenerContract<EventsManager, Coordinator>() {
|
||||
abstract override val produceEvent: Events
|
||||
abstract override val listensForEvents: List<Events>
|
||||
abstract override var coordinator: Coordinator?
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import dev.vishna.watchservice.KWatchChannel
|
||||
import dev.vishna.watchservice.asWatchChannel
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.io.File
|
||||
|
||||
data class FileWatcher(val file: File, val watcher: KWatchChannel)
|
||||
|
||||
suspend fun File.asWatcher(block: suspend (KWatchChannel) -> Unit): FileWatcher {
|
||||
val channel = this.asWatchChannel()
|
||||
block(channel)
|
||||
return FileWatcher(this, channel)
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.*
|
||||
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||
|
||||
class EventsDatabase() {
|
||||
val database = DatabaseEnvConfig.toEventsDatabase()
|
||||
val tables = listOf(
|
||||
events, // For kafka
|
||||
allEvents,
|
||||
tasks,
|
||||
runners,
|
||||
filesProcessed,
|
||||
files
|
||||
)
|
||||
|
||||
init {
|
||||
database.createDatabase()
|
||||
database.createTables(*tables.toTypedArray())
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.context.annotation.Import
|
||||
|
||||
@Configuration
|
||||
class SocketLocalInit: SocketImplementation() {
|
||||
}
|
||||
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import com.google.gson.Gson
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.RequestWorkProceed
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.bind.annotation.PostMapping
|
||||
import org.springframework.web.bind.annotation.RequestBody
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
|
||||
@Controller
|
||||
@RequestMapping(path = ["/action"])
|
||||
class ActionEventController(@Autowired var coordinator: Coordinator, @Autowired var eventsManager: EventsManager) {
|
||||
|
||||
|
||||
@PostMapping("/flow/proceed")
|
||||
fun permitRunOnSequence(@RequestBody data: RequestWorkProceed): ResponseEntity<String> {
|
||||
|
||||
val set = eventsManager.getEventsWith(data.referenceId)
|
||||
if (set.isEmpty()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(data))
|
||||
}
|
||||
coordinator.permitWorkToProceedOn(data.referenceId, set, "Requested by ${data.source}")
|
||||
|
||||
//EVENT_MEDIA_WORK_PROCEED_PERMITTED("event:media-work-proceed:permitted")
|
||||
return ResponseEntity.ok(null)
|
||||
}
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
|
||||
@RestController
|
||||
class InfoController {
|
||||
@Autowired lateinit var coordinator: Coordinator
|
||||
|
||||
@GetMapping("/cachedReferenceList")
|
||||
fun cachedReferenceList(): String {
|
||||
return coordinator.cachedReferenceList.joinToString(",")
|
||||
}
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
|
||||
@Controller
|
||||
@RequestMapping(path = ["/polls"])
|
||||
class PollController(@Autowired var coordinator: Coordinator) {
|
||||
|
||||
@GetMapping()
|
||||
fun polls(): String {
|
||||
val stat = coordinator.getActivePolls()
|
||||
return "Active Polls ${stat.active}/${stat.total}"
|
||||
}
|
||||
}
|
||||
@ -1,89 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import com.google.gson.Gson
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventRequest
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.bind.annotation.PostMapping
|
||||
import org.springframework.web.bind.annotation.RequestBody
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.ResponseStatus
|
||||
import java.io.File
|
||||
|
||||
@Controller
|
||||
@RequestMapping(path = ["/request"])
|
||||
class RequestEventController(@Autowired var coordinator: Coordinator) {
|
||||
|
||||
@PostMapping("/convert")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestConvert(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload.file)
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, payload.mode, listOf(OperationEvents.CONVERT)).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
|
||||
@PostMapping("/extract")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestExtract(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(payload))
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, payload.mode, listOf(OperationEvents.EXTRACT)).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
|
||||
@PostMapping("/encode")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestEncode(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(payload))
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, payload.mode, listOf(OperationEvents.ENCODE)).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
|
||||
@PostMapping("/all")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestAll(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload.file)
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, type = payload.mode).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.coordination
|
||||
|
||||
/**
|
||||
* Class to handle messages from websockets, produced by Processer instances.
|
||||
* This is due to keep a overview of progress by processer
|
||||
*/
|
||||
class ProcesserSocketMessageListener {
|
||||
|
||||
|
||||
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.services
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.database.withTransaction
|
||||
import no.iktdev.mediaprocessing.coordinator.eventDatabase
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.files
|
||||
import no.iktdev.mediaprocessing.shared.common.extended.isSupportedVideoFile
|
||||
import no.iktdev.mediaprocessing.shared.common.md5
|
||||
import org.jetbrains.exposed.sql.insertIgnore
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
class UnattendedIndexing {
|
||||
private val logger = KotlinLogging.logger {}
|
||||
|
||||
@Scheduled(fixedDelay = 60_000*60)
|
||||
fun indexContent() {
|
||||
val allFiles = SharedConfig.incomingContent.flatMap { folder ->
|
||||
logger.info { "Performing indexing of folder: ${folder.name}" }
|
||||
folder.walkTopDown()
|
||||
.filter { it.isFile && it.isSupportedVideoFile() }
|
||||
.toMutableList()
|
||||
}
|
||||
val ignoredParents = allFiles
|
||||
.asSequence()
|
||||
.mapNotNull { it.parentFile }
|
||||
.filter { parent -> parent.resolve(".ignore").exists() }
|
||||
.toSet()
|
||||
|
||||
val fileList = allFiles
|
||||
.filter { file -> file.parentFile !in ignoredParents }
|
||||
|
||||
fileList.forEach { file ->
|
||||
withTransaction(eventDatabase.database) {
|
||||
files.insertIgnore {
|
||||
it[this.fileName] = file.absolutePath
|
||||
it[this.baseName] = file.nameWithoutExtension
|
||||
it[this.folder] = file.parentFile.absolutePath
|
||||
it[this.checksum] = file.md5()
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.info { "Indexing completed" }
|
||||
/*val storedFiles = withTransaction(eventDatabase.database) {
|
||||
files.selectAll()
|
||||
.mapNotNull { it[files.fileName] }
|
||||
}?.forEach { file ->
|
||||
if (!File(file).exists()) {
|
||||
logger.info { "Detected file no longer existing. Performing removal i db" }
|
||||
files.deleteWhere {
|
||||
(fileName eq file)
|
||||
}
|
||||
}
|
||||
}*/
|
||||
}
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.implementations
|
||||
|
||||
import com.google.gson.Gson
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.referenceId
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaProcessStartEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.az
|
||||
|
||||
abstract class WorkTaskListener: CoordinatorEventListener() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
if (!super.shouldIProcessAndHandleEvent(incomingEvent, events)) {
|
||||
return false
|
||||
}
|
||||
return canStart(incomingEvent, events)
|
||||
}
|
||||
|
||||
fun canStart(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val autoStart = events.find { it.eventType == Events.ProcessStarted }?.az<MediaProcessStartEvent>()?.data
|
||||
if (autoStart == null) {
|
||||
log.error { "Start event not found. Requiring permit event" }
|
||||
}
|
||||
return if (incomingEvent.eventType == Events.WorkProceedPermitted) {
|
||||
return true
|
||||
} else {
|
||||
if (autoStart == null || autoStart.type == ProcessType.MANUAL) {
|
||||
log.warn { "${incomingEvent.metadata.referenceId} waiting for Proceed event due to Manual process" }
|
||||
false
|
||||
} else true
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.FileNameParser
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.isOnly
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
|
||||
@Service
|
||||
class BaseInfoFromFileTaskListener() : CoordinatorEventListener() {
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override val produceEvent: Events = Events.BaseInfoRead
|
||||
override val listensForEvents: List<Events> = listOf(Events.ProcessStarted)
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
if (!super.shouldIProcessAndHandleEvent(incomingEvent, events)) {
|
||||
return false
|
||||
}
|
||||
val startedWith = events.findFirstEventOf<MediaProcessStartEvent>()?.data?.operations;
|
||||
if (startedWith?.isOnly(OperationEvents.CONVERT) == true) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
val message = try {
|
||||
readFileInfo(event.data as StartEventData, event.metadata.eventId)?.let {
|
||||
BaseInfoEvent(metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()), data = it)
|
||||
} ?: BaseInfoEvent(metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName()))
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
BaseInfoEvent(metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName()))
|
||||
}
|
||||
onProduceEvent(message)
|
||||
active = false
|
||||
}
|
||||
|
||||
|
||||
@Throws(Exception::class)
|
||||
fun readFileInfo(started: StartEventData, eventId: String): BaseInfo? {
|
||||
return try {
|
||||
val fileName = File(started.file).nameWithoutExtension
|
||||
val fileNameParser = FileNameParser(fileName)
|
||||
BaseInfo(
|
||||
title = fileNameParser.guessDesiredTitle().also {
|
||||
if (it.isBlank()) {
|
||||
throw RuntimeException("No title found!")
|
||||
}
|
||||
},
|
||||
sanitizedName = fileNameParser.guessDesiredFileName(),
|
||||
searchTitles = fileNameParser.guessSearchableTitle()
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
log.error { "Failed to read info from file\neventId: $eventId" }
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(BaseInfoEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,301 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import com.google.gson.GsonBuilder
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.LocalDateTimeAdapter
|
||||
import no.iktdev.eventi.data.*
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.EventsSummaryMapping
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store.*
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.validator.CompletionValidator
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.NameHelper
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.reader.*
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
import java.time.LocalDateTime
|
||||
|
||||
@Service
|
||||
class CompletedTaskListener : CoordinatorEventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
var doNotProduceComplete = System.getenv("DISABLE_COMPLETE").toBoolean() ?: false
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
override fun onReady() {
|
||||
super.onReady()
|
||||
if (doNotProduceComplete) {
|
||||
log.warn { "DoNotProduceComplete is set!\n\tNo complete event will be triggered!\n\tTo enable production of complete vents, remove this line in your environment: \"DISABLE_COMPLETE\"" }
|
||||
}
|
||||
}
|
||||
|
||||
override fun shouldIHandleFailedEvents(incomingEvent: Event): Boolean {
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
override val produceEvent: Events = Events.ProcessCompleted
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.CoverDownloaded,
|
||||
Events.ConvertTaskCompleted,
|
||||
Events.EncodeTaskCompleted,
|
||||
Events.ExtractTaskCompleted,
|
||||
Events.PersistContent
|
||||
)
|
||||
|
||||
|
||||
override fun isPrerequisitesFulfilled(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val started = events.find { it.eventType == Events.ProcessStarted }?.az<MediaProcessStartEvent>()
|
||||
if (started == null) {
|
||||
log.info { "No Start event" }
|
||||
return false
|
||||
}
|
||||
val viableEvents = events.filter { it.isSuccessful() }
|
||||
|
||||
if (!CompletionValidator.req1(started, events)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!CompletionValidator.req2(started.data?.operations ?: emptyList(), viableEvents)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!CompletionValidator.req3(started.data?.operations ?: emptyList(), events)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!CompletionValidator.req4(events)) {
|
||||
return false
|
||||
}
|
||||
|
||||
return super.isPrerequisitesFulfilled(incomingEvent, events)
|
||||
}
|
||||
|
||||
fun getVideo(events: List<Event>): VideoDetails? {
|
||||
val mediaInfo = events.find { it.eventType == Events.ReadOutNameAndType }?.az<MediaOutInformationConstructedEvent>()
|
||||
val encoded = events.find { it.eventType == Events.EncodeTaskCompleted }?.dataAs<EncodedData>()?.outputFile
|
||||
if (encoded == null) {
|
||||
log.warn { "No encode no video details!" }
|
||||
return null
|
||||
}
|
||||
|
||||
val proper = mediaInfo?.data?.toValueObject() ?: run {
|
||||
log.error { "Unable to get media object from data" }
|
||||
return null
|
||||
}
|
||||
|
||||
return VideoDetails(
|
||||
type = proper.type,
|
||||
fileName = File(encoded).name,
|
||||
serieInfo = if (proper !is EpisodeInfo) null else SerieInfo(
|
||||
episodeTitle = proper.episodeTitle,
|
||||
episodeNumber = proper.episode,
|
||||
seasonNumber = proper.season,
|
||||
title = proper.title
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
if (doNotProduceComplete) {
|
||||
return false
|
||||
}
|
||||
val result = super.shouldIProcessAndHandleEvent(incomingEvent, events)
|
||||
return result && incomingEvent.eventType == Events.PersistContent
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume() ?: return
|
||||
active = true
|
||||
|
||||
val mediaInfo: ComposedMediaInfo = composeMediaInfo(events) ?: run {
|
||||
log.error { "Unable to compose media info for ${event.referenceId()}" }
|
||||
return
|
||||
}
|
||||
|
||||
val existingTitles = ContentTitleStore.findMasterTitles(mediaInfo.titles)
|
||||
|
||||
val usableCollection: String = if (existingTitles.isNotEmpty())
|
||||
ContentCatalogStore.getCollectionByTitleAndType(mediaInfo.type, existingTitles) ?: run {
|
||||
log.warn { "Did not receive collection based on titles provided in list ${existingTitles.joinToString(",")}, falling back to fallbackCollection: ${mediaInfo.fallbackCollection}" }
|
||||
mediaInfo.fallbackCollection
|
||||
} else mediaInfo.fallbackCollection
|
||||
|
||||
val genreIdsForCatalog = ContentGenresStore.storeAndGetIds(mediaInfo.genres)
|
||||
|
||||
val persistedContent: PersistedContent? = events.find { it.eventType == Events.PersistContent }?.az<PersistedContentEvent>()?.data
|
||||
if (persistedContent == null) {
|
||||
log.error { "PersistedContent is null! can't continue" }
|
||||
return
|
||||
}
|
||||
|
||||
val completedData = CompletedData(
|
||||
eventIdsCollected = events.map { it.eventId() },
|
||||
metadataStored = MetadataStored(
|
||||
title = mediaInfo.title,
|
||||
titles = mediaInfo.titles,
|
||||
type = mediaInfo.type,
|
||||
cover = persistedContent.cover?.storeDestinationFileName?.let { File(it).name },
|
||||
collection = usableCollection,
|
||||
summary = mediaInfo.summaries,
|
||||
foundTitles = existingTitles,
|
||||
genres = mediaInfo.genres,
|
||||
genreIds = genreIdsForCatalog
|
||||
)
|
||||
)
|
||||
|
||||
completedData.metadataStored.let { meta ->
|
||||
val catalogId = ContentCatalogStore.storeCatalog(
|
||||
title = meta.title,
|
||||
titles = mediaInfo.titles,
|
||||
collection = meta.collection,
|
||||
type = meta.type,
|
||||
cover = meta.cover,
|
||||
genres = meta.genreIds
|
||||
)
|
||||
catalogId?.let { id ->
|
||||
meta.summary.forEach { summary ->
|
||||
ContentMetadataStore.storeSummary(id, summary)
|
||||
}
|
||||
}
|
||||
ContentTitleStore.store(meta.title, meta.titles)
|
||||
}
|
||||
|
||||
|
||||
val videoInfo = getVideo(events)
|
||||
if (videoInfo != null) {
|
||||
ContentCatalogStore.storeMedia(
|
||||
title = completedData.metadataStored.title,
|
||||
titles = mediaInfo.titles,
|
||||
collection = completedData.metadataStored.collection,
|
||||
type = completedData.metadataStored.type,
|
||||
videoDetails = videoInfo
|
||||
)
|
||||
} else {
|
||||
log.info { "VideoInfo is null" }
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
persistedContent.subtitles.let { subtitles ->
|
||||
subtitles.map {
|
||||
ContentSubtitleStore.storeSubtitles(
|
||||
collection = completedData.metadataStored.collection,
|
||||
destinationFile = File(it.storeDestinationFileName)
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
ProcessedFileStore.store(
|
||||
mediaInfo.title,
|
||||
events,
|
||||
EventsSummaryMapping().map(events)
|
||||
)
|
||||
|
||||
if (!doNotProduceComplete) {
|
||||
onProduceEvent(MediaProcessCompletedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = completedData
|
||||
))
|
||||
} else {
|
||||
log.warn { "Do not produce complete is enabled!" }
|
||||
}
|
||||
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(MediaProcessCompletedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
|
||||
internal data class ComposedMediaInfo(
|
||||
val title: String,
|
||||
val fallbackCollection: String,
|
||||
|
||||
val titles: List<String>,
|
||||
val type: String,
|
||||
val summaries: List<SummaryInfo>,
|
||||
val genres: List<String>
|
||||
)
|
||||
|
||||
private fun composeMediaInfo(events: List<Event>): ComposedMediaInfo? {
|
||||
val baseInfo =
|
||||
events.find { it.eventType == Events.BaseInfoRead }?.az<BaseInfoEvent>()?.let {
|
||||
it.data
|
||||
} ?: run {
|
||||
log.info { "Cant find BaseInfoEvent on ${Events.BaseInfoRead}" }
|
||||
return null
|
||||
}
|
||||
val metadataInfo = getMetadata(events)
|
||||
val mediaInfo: MediaInfo = events.find { it.eventType == Events.ReadOutNameAndType }
|
||||
?.az<MediaOutInformationConstructedEvent>()?.let {
|
||||
it.data?.toValueObject()
|
||||
} ?: run {
|
||||
log.info { "Cant find MediaOutInformationConstructedEvent on ${Events.ReadOutNameAndType}" }
|
||||
return null
|
||||
}
|
||||
|
||||
val summaries = metadataInfo?.summary?.filter { it.summary != null }
|
||||
?.map { SummaryInfo(language = it.language, summary = it.summary!!) } ?: emptyList()
|
||||
|
||||
val titles: MutableList<String> = mutableListOf(mediaInfo.title)
|
||||
metadataInfo?.let {
|
||||
titles.addAll(it.altTitle)
|
||||
titles.add(it.title)
|
||||
titles.add(NameHelper.normalize(it.title))
|
||||
}
|
||||
|
||||
return ComposedMediaInfo(
|
||||
title = NameHelper.normalize(metadataInfo?.title ?: mediaInfo.title),
|
||||
fallbackCollection = baseInfo.title,
|
||||
titles = titles,
|
||||
type = metadataInfo?.type ?: mediaInfo.type,
|
||||
summaries = summaries,
|
||||
genres = metadataInfo?.genres ?: emptyList()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
private fun getMetadata(events: List<Event>): pyMetadata? {
|
||||
val referenceId = events.firstNotNullOf { it.referenceId() }
|
||||
|
||||
val gson = GsonBuilder()
|
||||
.registerTypeAdapter(LocalDateTime::class.java, LocalDateTimeAdapter())
|
||||
.setPrettyPrinting()
|
||||
.create()
|
||||
|
||||
//log.info { "Events in complete:\n${gson.toJson(events)}" }
|
||||
|
||||
val metadataFound = events.find { it.eventType == Events.MetadataSearchPerformed }
|
||||
if (metadataFound == null) {
|
||||
log.warn { "ReferenceId: $referenceId -> ${Events.MetadataSearchPerformed} was not found in events" }
|
||||
return null
|
||||
}
|
||||
val data = metadataFound.az<MediaMetadataReceivedEvent>()
|
||||
if (data == null) {
|
||||
log.warn { "ReferenceId: $referenceId -> ${Events.MetadataSearchPerformed} does not contain any data.." }
|
||||
return null
|
||||
}
|
||||
|
||||
return data.data
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@ -1,149 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import com.google.gson.Gson
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.*
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.taskManager
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.implementations.WorkTaskListener
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.isOnly
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
|
||||
@Service
|
||||
class ConvertWorkTaskListener: WorkTaskListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
override val produceEvent: Events = Events.ConvertTaskCreated
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.ExtractTaskCompleted,
|
||||
Events.ProcessStarted
|
||||
)
|
||||
|
||||
override fun canProduceMultipleEvents(): Boolean {
|
||||
return true
|
||||
}
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val mainCheckOk = super.shouldIProcessAndHandleEvent(incomingEvent, events);
|
||||
if (!mainCheckOk) {
|
||||
return false
|
||||
}
|
||||
|
||||
val startOperation = events.findFirstOf(Events.ProcessStarted)?.dataAs<StartEventData>()
|
||||
if (startOperation == null) {
|
||||
log.error { "Could not find 'ProcessStarted' event" }
|
||||
return false
|
||||
}
|
||||
|
||||
if (incomingEvent.isOfEvent(Events.ProcessStarted)) {
|
||||
return startOperation.operations.isOnly(OperationEvents.CONVERT)
|
||||
}
|
||||
|
||||
val producedEvents = events.filter { it.eventType == produceEvent }
|
||||
val shouldIHandleAndProduce = producedEvents.none { it.derivedFromEventId() == incomingEvent.eventId() }
|
||||
|
||||
val extractedEvent = events.findFirstEventOf<ExtractWorkPerformedEvent>()
|
||||
if (extractedEvent?.isSuccessful() == true && shouldIHandleAndProduce) {
|
||||
log.info { "Permitting handling of event: ${extractedEvent.data?.outputFile}" }
|
||||
}
|
||||
|
||||
return shouldIHandleAndProduce
|
||||
}
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
var language: String? = null
|
||||
var storeAsFile: String? = null
|
||||
|
||||
|
||||
val file = if (event.eventType == Events.ExtractTaskCompleted) {
|
||||
val foundEvent = event.az<ExtractWorkPerformedEvent>()?.data
|
||||
language = foundEvent?.language
|
||||
storeAsFile = foundEvent?.storeFileName
|
||||
foundEvent?.outputFile
|
||||
} else if (event.eventType == Events.ProcessStarted) {
|
||||
val startEvent = event.az<MediaProcessStartEvent>()?.data
|
||||
if (startEvent?.operations?.isOnly(OperationEvents.CONVERT) == true) {
|
||||
startEvent.file
|
||||
} else null
|
||||
} else {
|
||||
events.find { it.eventType == Events.ExtractTaskCompleted }
|
||||
?.az<ExtractWorkPerformedEvent>()?.data?.outputFile
|
||||
}
|
||||
|
||||
|
||||
val convertFile = file?.let { File(it) }
|
||||
if (language.isNullOrEmpty()) {
|
||||
convertFile?.parentFile?.nameWithoutExtension?.let {
|
||||
if (it.length == 3) {
|
||||
language = it.lowercase()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (convertFile == null || !convertFile.exists()) {
|
||||
onProduceEvent(ConvertWorkCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName())
|
||||
))
|
||||
return
|
||||
} else {
|
||||
val convertData = ConvertData(
|
||||
language = language ?: "unk",
|
||||
inputFile = convertFile.absolutePath,
|
||||
outputFileName = convertFile.nameWithoutExtension,
|
||||
storeFileName = storeAsFile ?: convertFile.nameWithoutExtension,
|
||||
outputDirectory = convertFile.parentFile.absolutePath,
|
||||
allowOverwrite = true
|
||||
)
|
||||
|
||||
|
||||
ConvertWorkCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = convertData
|
||||
).also { event ->
|
||||
val taskCreatedSuccessfully = taskManager.createTask(
|
||||
referenceId = event.referenceId(),
|
||||
eventId = event.eventId(),
|
||||
derivedFromEventId = event.derivedFromEventId(),
|
||||
task = TaskType.Convert,
|
||||
data = WGson.gson.toJson(event.data!!),
|
||||
inputFile = event.data!!.inputFile
|
||||
)
|
||||
|
||||
if (!taskCreatedSuccessfully) {
|
||||
log.error { "Failed to create task for events on referenceId: ${event.referenceId()}" }
|
||||
} else {
|
||||
onProduceEvent(event)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(ConvertWorkCreatedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -1,92 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.shared.common.DownloadClient
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class CoverDownloadTaskListener : CoordinatorEventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
override val produceEvent: Events = Events.CoverDownloaded
|
||||
override val listensForEvents: List<Events> = listOf(Events.ReadOutCover)
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
val failedEventDefault = MediaCoverDownloadedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName())
|
||||
)
|
||||
|
||||
val data = event.az<MediaCoverInfoReceivedEvent>()?.data
|
||||
if (data == null) {
|
||||
log.error { "No valid data for use to obtain cover" }
|
||||
onProduceEvent(failedEventDefault)
|
||||
active = false
|
||||
return
|
||||
}
|
||||
|
||||
val client = DownloadClient(data.url, SharedConfig.cachedContent, data.outFileBaseName)
|
||||
|
||||
val outFile = runBlocking {
|
||||
client.getOutFile()
|
||||
}
|
||||
|
||||
val coversInDifferentFormats = SharedConfig.cachedContent.listFiles { it -> it.isFile && it.extension.lowercase() in client.contentTypeToExtension().values }
|
||||
?.filter { it.nameWithoutExtension.contains(data.outFileBaseName, ignoreCase = true) } ?: emptyList()
|
||||
|
||||
val result = if (outFile?.exists() == true) {
|
||||
outFile
|
||||
} else if (coversInDifferentFormats.isNotEmpty()) {
|
||||
coversInDifferentFormats.random()
|
||||
} else if (outFile != null) {
|
||||
runBlocking {
|
||||
client.download(outFile)
|
||||
}
|
||||
} else {
|
||||
null
|
||||
}
|
||||
|
||||
if (result == null) {
|
||||
log.error { "Could not download cover, check logs ${event.metadata.eventId} " }
|
||||
} else {
|
||||
if (!result.exists() || !result.canRead()) {
|
||||
onProduceEvent(failedEventDefault)
|
||||
active = false
|
||||
return
|
||||
}
|
||||
onProduceEvent(MediaCoverDownloadedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = DownloadedCover(result.absolutePath)
|
||||
))
|
||||
}
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(MediaCoverDownloadedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -1,102 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.isSuccessful
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.NameHelper
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.Regexes
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class CoverFromMetadataTaskListener: CoordinatorEventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
override val produceEvent: Events = Events.ReadOutCover
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.MetadataSearchPerformed
|
||||
)
|
||||
|
||||
override fun isPrerequisitesFulfilled(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
return (events.any { it.eventType == Events.ReadOutNameAndType && it.isSuccessful() })
|
||||
}
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val state = super.shouldIProcessAndHandleEvent(incomingEvent, events)
|
||||
if (!state) {
|
||||
return false
|
||||
}
|
||||
if (!incomingEvent.isSuccessful())
|
||||
return false
|
||||
return listensForEvents.any { it == incomingEvent.eventType }
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
val baseInfo = events.find { it.eventType == Events.BaseInfoRead }?.az<BaseInfoEvent>()?.data
|
||||
if (baseInfo == null) {
|
||||
log.info { "No base info" }
|
||||
active = false
|
||||
return
|
||||
}
|
||||
|
||||
val metadataEvent = if (event.eventType == Events.MetadataSearchPerformed) event else events.findLast { it.eventType == Events.MetadataSearchPerformed }
|
||||
val metadata = metadataEvent?.az<MediaMetadataReceivedEvent>()?.data
|
||||
?: return
|
||||
val mediaOutInfo = events.find { it.eventType == Events.ReadOutNameAndType }?.az<MediaOutInformationConstructedEvent>()?.data
|
||||
if (mediaOutInfo == null) {
|
||||
log.info { "No Media out info" }
|
||||
active = false
|
||||
return
|
||||
}
|
||||
val videoInfo = mediaOutInfo.toValueObject()
|
||||
|
||||
var coverTitle = metadata.title ?: videoInfo?.title ?: baseInfo.title
|
||||
coverTitle = Regexes.illegalCharacters.replace(coverTitle, " - ")
|
||||
coverTitle = Regexes.trimWhiteSpaces.replace(coverTitle, " ")
|
||||
|
||||
val coverUrl = metadata.cover
|
||||
val result = if (coverUrl.isNullOrBlank()) {
|
||||
log.warn { "No cover available for ${baseInfo.title}" }
|
||||
MediaCoverInfoReceivedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Skipped, getProducerName())
|
||||
)
|
||||
} else {
|
||||
MediaCoverInfoReceivedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = CoverDetails(
|
||||
url = coverUrl,
|
||||
outFileBaseName = NameHelper.normalize(coverTitle),
|
||||
)
|
||||
)
|
||||
}
|
||||
onProduceEvent(result)
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(MediaCoverInfoReceivedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -1,114 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.dataAs
|
||||
import no.iktdev.eventi.data.eventId
|
||||
import no.iktdev.eventi.data.referenceId
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.EncodeWorkArgumentsMapping
|
||||
import no.iktdev.mediaprocessing.shared.common.Preference
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class EncodeWorkArgumentsTaskListener: CoordinatorEventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
override val produceEvent: Events = Events.EncodeParameterCreated
|
||||
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.StreamParsed,
|
||||
Events.ReadOutNameAndType
|
||||
)
|
||||
val preference = Preference.getPreference()
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val state = super.shouldIProcessAndHandleEvent(incomingEvent, events)
|
||||
val eventType = events.map { it.eventType }
|
||||
|
||||
val startOperation = events.findFirstOf(Events.ProcessStarted)?.dataAs<StartEventData>() ?: return false
|
||||
if (startOperation.operations.none { it == OperationEvents.ENCODE }) {
|
||||
return false
|
||||
}
|
||||
|
||||
return state && eventType.containsAll(listensForEvents)
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
val started = events.find { it.eventType == Events.ProcessStarted }?.az<MediaProcessStartEvent>() ?: return
|
||||
|
||||
active = true
|
||||
|
||||
val streamsParsed = events.findEventOf<MediaFileStreamsParsedEvent>()
|
||||
val streams = streamsParsed?.data
|
||||
if (streams == null) {
|
||||
active = false
|
||||
log.error { "No Streams found for event ${streamsParsed?.metadata?.eventId} with referenceId ${event.metadata.referenceId}" }
|
||||
return
|
||||
}
|
||||
|
||||
val mediaInfoEvent = events.findEventOf<MediaOutInformationConstructedEvent>()
|
||||
val mediaInfo = mediaInfoEvent?.data
|
||||
val mediaInfoData = mediaInfo?.toValueObject()
|
||||
if (mediaInfo == null) {
|
||||
active = false
|
||||
log.error { "No media info data was provided for event ${mediaInfoEvent?.eventId()} with referenceId ${event.referenceId()}" }
|
||||
return
|
||||
} else if (mediaInfoData == null) {
|
||||
active = false
|
||||
log.error { "Media info data was provided but could not be converted to proper value object for event ${mediaInfoEvent?.eventId()} with referenceId ${event.referenceId()}" }
|
||||
return
|
||||
}
|
||||
|
||||
val inputFile = started.data?.file
|
||||
if (inputFile == null) {
|
||||
active = false
|
||||
log.error { "No input file was provided for the start event ${started.metadata.eventId} with referenceId ${event.referenceId()}" }
|
||||
return
|
||||
}
|
||||
val mapper = EncodeWorkArgumentsMapping(
|
||||
inputFile = inputFile,
|
||||
outFileFullName = mediaInfoData.fullName,
|
||||
streams = streams,
|
||||
preference = preference.encodePreference
|
||||
)
|
||||
|
||||
val result = mapper.getArguments()
|
||||
if (result == null) {
|
||||
onProduceEvent(EncodeArgumentCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName())
|
||||
))
|
||||
} else {
|
||||
onProduceEvent(EncodeArgumentCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = result
|
||||
))
|
||||
}
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(EncodeArgumentCreatedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.derivedFromEventId
|
||||
import no.iktdev.eventi.data.eventId
|
||||
import no.iktdev.eventi.data.referenceId
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.taskManager
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.implementations.WorkTaskListener
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class EncodeWorkTaskListener : WorkTaskListener() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
override val produceEvent: Events = Events.EncodeTaskCreated
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.EncodeParameterCreated,
|
||||
Events.WorkProceedPermitted
|
||||
)
|
||||
|
||||
override fun canProduceMultipleEvents(): Boolean {
|
||||
return true
|
||||
}
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
val encodeArguments = if (event.eventType == Events.EncodeParameterCreated) {
|
||||
event.az<EncodeArgumentCreatedEvent>()?.data
|
||||
} else {
|
||||
events.find { it.eventType == Events.EncodeParameterCreated }
|
||||
?.az<EncodeArgumentCreatedEvent>()?.data
|
||||
}
|
||||
if (encodeArguments == null) {
|
||||
log.error { "No Encode arguments found.. referenceId: ${event.referenceId()}" }
|
||||
active = false
|
||||
return
|
||||
}
|
||||
EncodeWorkCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = encodeArguments
|
||||
).also { event ->
|
||||
val taskCreatedSuccessfully = taskManager.createTask(
|
||||
referenceId = event.referenceId(),
|
||||
eventId = event.eventId(),
|
||||
derivedFromEventId = event.derivedFromEventId(),
|
||||
task = TaskType.Encode,
|
||||
data = WGson.gson.toJson(event.data!!),
|
||||
inputFile = event.data!!.inputFile
|
||||
)
|
||||
if (!taskCreatedSuccessfully) {
|
||||
log.error { "Failed to create task for events on referenceId: ${event.referenceId()}" }
|
||||
} else {
|
||||
onProduceEvent(event)
|
||||
}
|
||||
}
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(EncodeWorkCreatedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -1,107 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.eventId
|
||||
import no.iktdev.eventi.data.referenceId
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.ExtractWorkArgumentsMapping
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class ExtractWorkArgumentsTaskListener: CoordinatorEventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
override val produceEvent: Events = Events.ExtractParameterCreated
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.StreamParsed,
|
||||
Events.ReadOutNameAndType
|
||||
)
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val startEvent = events.findFirstEventOf<MediaProcessStartEvent>()
|
||||
val hasExtract = startEvent?.data?.operations?.contains(OperationEvents.EXTRACT) ?: false
|
||||
|
||||
val state = super.shouldIProcessAndHandleEvent(incomingEvent, events)
|
||||
val eventType = events.map { it.eventType }
|
||||
return hasExtract && state && eventType.containsAll(listensForEvents)
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
val started = events.find { it.eventType == Events.ProcessStarted }?.az<MediaProcessStartEvent>() ?: return
|
||||
|
||||
active = true
|
||||
|
||||
val streamsParsed = events.findEventOf<MediaFileStreamsParsedEvent>()
|
||||
val streams = streamsParsed?.data
|
||||
if (streams == null) {
|
||||
active = false
|
||||
log.error { "No Streams found for event ${streamsParsed?.metadata?.eventId} with referenceId ${event.metadata.referenceId}" }
|
||||
return
|
||||
}
|
||||
|
||||
val mediaInfoEvent = events.findEventOf<MediaOutInformationConstructedEvent>()
|
||||
val mediaInfo = mediaInfoEvent?.data
|
||||
val mediaInfoData = mediaInfo?.toValueObject()
|
||||
if (mediaInfo == null) {
|
||||
active = false
|
||||
log.error { "No media info data was provided for event ${mediaInfoEvent?.eventId()} with referenceId ${event.referenceId()}" }
|
||||
return
|
||||
} else if (mediaInfoData == null) {
|
||||
active = false
|
||||
log.error { "Media info data was provided but could not be converted to proper value object for event ${mediaInfoEvent?.eventId()} with referenceId ${event.referenceId()}" }
|
||||
return
|
||||
}
|
||||
|
||||
val inputFile = started.data?.file
|
||||
if (inputFile == null) {
|
||||
active = false
|
||||
log.error { "No input file was provided for the start event ${started.metadata.eventId} with referenceId ${event.referenceId()}" }
|
||||
return
|
||||
}
|
||||
|
||||
val mapper = ExtractWorkArgumentsMapping(
|
||||
inputFile = inputFile,
|
||||
outFileFullName = mediaInfoData.fullName,
|
||||
streams = streams
|
||||
)
|
||||
|
||||
val result = mapper.getArguments()
|
||||
if (result.isEmpty()) {
|
||||
onProduceEvent(ExtractArgumentCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Skipped, getProducerName())
|
||||
))
|
||||
} else {
|
||||
onProduceEvent(ExtractArgumentCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = result
|
||||
))
|
||||
}
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(ExtractArgumentCreatedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -1,93 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.derivedFromEventId
|
||||
import no.iktdev.eventi.data.eventId
|
||||
import no.iktdev.eventi.data.referenceId
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.taskManager
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.implementations.WorkTaskListener
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class ExtractWorkTaskListener: WorkTaskListener() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
override val produceEvent: Events = Events.ExtractTaskCreated
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.ExtractParameterCreated,
|
||||
Events.WorkProceedPermitted
|
||||
)
|
||||
|
||||
override fun canProduceMultipleEvents(): Boolean {
|
||||
return true
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
active = false
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
val arguments = if (event.eventType == Events.ExtractParameterCreated) {
|
||||
event.az<ExtractArgumentCreatedEvent>()?.data
|
||||
} else {
|
||||
events.find { it.eventType == Events.ExtractParameterCreated }
|
||||
?.az<ExtractArgumentCreatedEvent>()?.data
|
||||
}
|
||||
if (arguments == null) {
|
||||
log.error { "No Extract arguments found.. referenceId: ${event.referenceId()}" }
|
||||
active = false
|
||||
return
|
||||
}
|
||||
if (arguments.isEmpty()) {
|
||||
active = false
|
||||
return
|
||||
}
|
||||
|
||||
arguments.mapNotNull {
|
||||
ExtractWorkCreatedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = it
|
||||
)
|
||||
}.forEach { event ->
|
||||
val taskCreatedSuccessfully = taskManager.createTask(
|
||||
referenceId = event.referenceId(),
|
||||
eventId = event.eventId(),
|
||||
derivedFromEventId = event.derivedFromEventId(),
|
||||
task = TaskType.Extract,
|
||||
data = WGson.gson.toJson(event.data!!),
|
||||
inputFile = event.data!!.inputFile
|
||||
)
|
||||
if (!taskCreatedSuccessfully) {
|
||||
log.error { "Failed to create task for events on referenceId: ${event.referenceId()}" }
|
||||
} else {
|
||||
onProduceEvent(event)
|
||||
}
|
||||
}
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(ExtractWorkCreatedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -1,192 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import com.google.gson.JsonObject
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.coordinator.log
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.FileNameDeterminate
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.NameHelper
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.EpisodeInfo
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.MovieInfo
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.pyMetadata
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.FileFilter
|
||||
|
||||
|
||||
@Service
|
||||
class MediaOutInformationTaskListener: CoordinatorEventListener() {
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
override val produceEvent: Events = Events.ReadOutNameAndType
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.MetadataSearchPerformed
|
||||
)
|
||||
|
||||
override fun shouldIHandleFailedEvents(incomingEvent: Event): Boolean {
|
||||
return (incomingEvent.eventType in listensForEvents)
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
val metadataResult = event.az<MediaMetadataReceivedEvent>()
|
||||
val mediaBaseInfo = events.findLast { it.eventType == Events.BaseInfoRead }?.az<BaseInfoEvent>()?.data
|
||||
if (mediaBaseInfo == null) {
|
||||
log.error { "Required event ${Events.BaseInfoRead} is not present" }
|
||||
coordinator?.produceNewEvent(
|
||||
MediaOutInformationConstructedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName())
|
||||
)
|
||||
)
|
||||
active = false
|
||||
return
|
||||
}
|
||||
val pm = ProcessMediaInfoAndMetadata(mediaBaseInfo, metadataResult?.data)
|
||||
val vi = pm.getVideoPayload()
|
||||
|
||||
val result = if (vi != null) {
|
||||
MediaInfoReceived(
|
||||
info = vi
|
||||
).let { MediaOutInformationConstructedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = it
|
||||
) }
|
||||
} else {
|
||||
MediaOutInformationConstructedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName())
|
||||
)
|
||||
}
|
||||
onProduceEvent(result)
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(MediaOutInformationConstructedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
|
||||
class ProcessMediaInfoAndMetadata(val baseInfo: BaseInfo, val metadata: pyMetadata? = null) {
|
||||
var metadataDeterminedContentType: FileNameDeterminate.ContentType = metadata?.type?.let { contentType ->
|
||||
when (contentType) {
|
||||
"serie", "tv" -> FileNameDeterminate.ContentType.SERIE
|
||||
"movie" -> FileNameDeterminate.ContentType.MOVIE
|
||||
else -> FileNameDeterminate.ContentType.UNDEFINED
|
||||
}
|
||||
} ?: FileNameDeterminate.ContentType.UNDEFINED
|
||||
|
||||
fun getTitlesFromMetadata(): List<String> {
|
||||
val titles: MutableList<String> = mutableListOf()
|
||||
metadata?.title?.let { titles.add(it) }
|
||||
metadata?.altTitle?.let { titles.addAll(it) }
|
||||
return titles
|
||||
}
|
||||
fun getExistingCollections() = SharedConfig.outgoingContent.listFiles(FileFilter { it.isDirectory })?.map { it.name } ?: emptyList()
|
||||
|
||||
fun getUsedCollectionForTitleOrNull(): String? {
|
||||
val exisiting = getExistingCollections()
|
||||
if (exisiting.isEmpty()) {
|
||||
return null
|
||||
}
|
||||
// Contains will cause mayhem!, ex: Collection with a single letter will make it messy
|
||||
val existingMatch = exisiting.find { it.lowercase() == baseInfo.title.lowercase() }
|
||||
return existingMatch
|
||||
}
|
||||
|
||||
fun getCollection(): String {
|
||||
val usedCollection = getUsedCollectionForTitleOrNull()
|
||||
if (usedCollection != null) {
|
||||
return usedCollection
|
||||
}
|
||||
|
||||
val metaTitles = getTitlesFromMetadata()
|
||||
val existingCollection = getExistingCollections()
|
||||
|
||||
// Contains will cause mayhem!, ex: Collection with a single letter will make it messy
|
||||
val ecList = existingCollection.filter { ec -> metaTitles.any { it.lowercase() == ec.lowercase() } }
|
||||
if (ecList.isNotEmpty()) {
|
||||
return ecList.first()
|
||||
}
|
||||
|
||||
|
||||
return NameHelper.cleanup(baseInfo.title)
|
||||
}
|
||||
|
||||
fun getTitle(): String {
|
||||
val metaTitles = getTitlesFromMetadata()
|
||||
val collection = getCollection()
|
||||
|
||||
val filteredMetaTitles = metaTitles.filter { it.lowercase().contains(baseInfo.title.lowercase()) || NameHelper.normalize(it).lowercase().contains(baseInfo.title.lowercase()) }
|
||||
|
||||
|
||||
return if (collection == baseInfo.title) {
|
||||
collection
|
||||
} else {
|
||||
NameHelper.cleanup (filteredMetaTitles.firstOrNull() ?: baseInfo.title)
|
||||
}
|
||||
}
|
||||
|
||||
fun getVideoPayload(): JsonObject? {
|
||||
val defaultFnd = FileNameDeterminate(getTitle(), baseInfo.sanitizedName, FileNameDeterminate.ContentType.UNDEFINED)
|
||||
|
||||
val determinedContentType = defaultFnd.getDeterminedVideoInfo()
|
||||
.let {
|
||||
when (it) {
|
||||
is EpisodeInfo -> FileNameDeterminate.ContentType.SERIE
|
||||
is MovieInfo -> FileNameDeterminate.ContentType.MOVIE
|
||||
else -> FileNameDeterminate.ContentType.UNDEFINED
|
||||
}
|
||||
}
|
||||
return if (determinedContentType == metadataDeterminedContentType && determinedContentType == FileNameDeterminate.ContentType.MOVIE) {
|
||||
FileNameDeterminate(getTitle(), getTitle(), FileNameDeterminate.ContentType.MOVIE).getDeterminedVideoInfo()?.toJsonObject()
|
||||
} else {
|
||||
FileNameDeterminate(getTitle(), baseInfo.sanitizedName, metadataDeterminedContentType).getDeterminedVideoInfo()?.toJsonObject()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
fun findNearestValue(list: List<String>, target: String): String? {
|
||||
return list.minByOrNull { it.distanceTo(target) }
|
||||
}
|
||||
|
||||
fun String.distanceTo(other: String): Int {
|
||||
val distance = Array(length + 1) { IntArray(other.length + 1) }
|
||||
for (i in 0..length) {
|
||||
distance[i][0] = i
|
||||
}
|
||||
for (j in 0..other.length) {
|
||||
distance[0][j] = j
|
||||
}
|
||||
for (i in 1..length) {
|
||||
for (j in 1..other.length) {
|
||||
distance[i][j] = minOf(
|
||||
distance[i - 1][j] + 1,
|
||||
distance[i][j - 1] + 1,
|
||||
distance[i - 1][j - 1] + if (this[i - 1] == other[j - 1]) 0 else 1
|
||||
)
|
||||
}
|
||||
}
|
||||
return distance[length][other.length]
|
||||
}
|
||||
}
|
||||
@ -1,161 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import kotlinx.coroutines.*
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.*
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.eventi.database.toEpochSeconds
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.lastOrSuccessOf
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.stereotype.Service
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.LocalDateTime
|
||||
import java.time.ZoneOffset
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.*
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
val metadataTimeoutMinutes: Int = System.getenv("METADATA_TIMEOUT")?.toIntOrNull() ?: -1
|
||||
|
||||
|
||||
@Service
|
||||
class MetadataWaitOrDefaultTaskListener() : CoordinatorEventListener() {
|
||||
|
||||
override fun onReady() {
|
||||
super.onReady()
|
||||
if (metadataTimeoutMinutes == 0) {
|
||||
log.warn { "Metadata timeout is set to 0 minutes.. This will block proceeding until metadata is found.." }
|
||||
}
|
||||
}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
|
||||
override val produceEvent: Events = Events.MetadataSearchPerformed
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.BaseInfoRead,
|
||||
Events.MetadataSearchPerformed,
|
||||
Events.ProcessCompleted
|
||||
)
|
||||
|
||||
val metadataTimeout = metadataTimeoutMinutes * 60
|
||||
|
||||
private val timeoutScope = CoroutineScope(Dispatchers.Default)
|
||||
val timeoutJobs = ConcurrentHashMap<String, Job>()
|
||||
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
if (!isOfEventsIListenFor(incomingEvent))
|
||||
return false
|
||||
|
||||
val childOf = events.filter { it.derivedFromEventId() == incomingEvent.eventId() }
|
||||
val haveListenerProduced = childOf.any { it.eventType == produceEvent }
|
||||
if (haveListenerProduced)
|
||||
return false
|
||||
|
||||
val metadataEvent = events.findEventOf<MediaMetadataReceivedEvent>()
|
||||
val metadataSource = metadataEvent?.metadata?.source
|
||||
|
||||
if (events.any { it.eventType == produceEvent } && !canProduceMultipleEvents() && metadataSource == getProducerName()) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!havProducedDerivedEventOnIncomingEvent(incomingEvent, events) && canProduceMultipleEvents()) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (haveProducedExpectedMessageBasedOnEvent(incomingEvent, events))
|
||||
return false
|
||||
|
||||
return (events.any { it.eventType == Events.BaseInfoRead })
|
||||
}
|
||||
|
||||
/**
|
||||
* This one gets special treatment, since it will only produce a timeout it does not need to use the incoming event
|
||||
*/
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
if (metadataTimeoutMinutes <= -1) {
|
||||
log.info { "Metadata has no timeout, a timeout will be created.." }
|
||||
val meta = incomingEvent.metadata()
|
||||
onProduceEvent(MediaMetadataReceivedEvent(
|
||||
metadata = meta.copy(
|
||||
status = EventStatus.Failed,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = null
|
||||
))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
val searchPerformedEvent: MediaMetadataReceivedEvent? = events.findEventOf<MediaMetadataReceivedEvent>()
|
||||
|
||||
if (searchPerformedEvent != null) {
|
||||
if (timeoutJobs.containsKey(searchPerformedEvent.referenceId())) {
|
||||
val job = timeoutJobs.remove(searchPerformedEvent.referenceId())
|
||||
job?.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
val baseInfo = events.findFirstEventOf<BaseInfoEvent>()
|
||||
if (baseInfo?.isSuccessful() != true) {
|
||||
return
|
||||
}
|
||||
|
||||
if (incomingEvent.isOfEvent(Events.BaseInfoRead)) {
|
||||
if (timeoutJobs.containsKey(incomingEvent.metadata().referenceId))
|
||||
return
|
||||
val ttsc = timeoutScope.launch {
|
||||
createTimeout(incomingEvent.metadata().referenceId, incomingEvent.metadata().eventId, baseInfo)
|
||||
}
|
||||
timeoutJobs[incomingEvent.metadata().referenceId] = ttsc
|
||||
}
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(MediaMetadataReceivedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
))
|
||||
}
|
||||
|
||||
suspend fun createTimeout(referenceId: String, eventId: String, baseInfo: BaseInfoEvent) {
|
||||
val expiryTime = (Instant.now().epochSecond + metadataTimeout)
|
||||
val dateTime = LocalDateTime.ofEpochSecond(expiryTime, 0, ZoneOffset.UTC)
|
||||
val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm", Locale.ENGLISH)
|
||||
log.info { "Sending ${baseInfo.data?.title} to waiting queue. Expiry ${dateTime.format(formatter)}" }
|
||||
delay(Duration.ofSeconds(metadataTimeout.toLong()).toMillis())
|
||||
if (!this.isActive()) {
|
||||
return
|
||||
}
|
||||
coordinator!!.produceNewEvent(
|
||||
MediaMetadataReceivedEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = referenceId,
|
||||
derivedFromEventId = eventId,
|
||||
status = EventStatus.Skipped,
|
||||
source = getProducerName()
|
||||
)
|
||||
)
|
||||
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,119 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.JsonObject
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.dataAs
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaFileStreamsParsedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaMetadataReceivedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.AudioStream
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.ParsedMediaStreams
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.SubtitleStream
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.VideoStream
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class ParseMediaFileStreamsTaskListener() : CoordinatorEventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
|
||||
override val produceEvent: Events = Events.StreamParsed
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.StreamRead
|
||||
)
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
return super.shouldIProcessAndHandleEvent(incomingEvent, events)
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
|
||||
val readData = event.dataAs<JsonObject>()
|
||||
val result = try {
|
||||
MediaFileStreamsParsedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = parseStreams(readData)
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
MediaFileStreamsParsedEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName())
|
||||
)
|
||||
}
|
||||
onProduceEvent(result)
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(
|
||||
MediaFileStreamsParsedEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fun parseStreams(data: JsonObject?): ParsedMediaStreams {
|
||||
val ignoreCodecs = listOf("png", "mjpeg")
|
||||
val gson = Gson()
|
||||
return try {
|
||||
val jStreams = data!!.getAsJsonArray("streams")
|
||||
|
||||
val videoStreams = mutableListOf<VideoStream>()
|
||||
val audioStreams = mutableListOf<AudioStream>()
|
||||
val subtitleStreams = mutableListOf<SubtitleStream>()
|
||||
|
||||
for (streamJson in jStreams) {
|
||||
val streamObject = streamJson.asJsonObject
|
||||
if (!streamObject.has("codec_name")) continue
|
||||
val codecName = streamObject.get("codec_name").asString
|
||||
val codecType = streamObject.get("codec_type").asString
|
||||
|
||||
if (codecName in ignoreCodecs) continue
|
||||
|
||||
when (codecType) {
|
||||
"video" -> videoStreams.add(gson.fromJson(streamObject, VideoStream::class.java))
|
||||
"audio" -> audioStreams.add(gson.fromJson(streamObject, AudioStream::class.java))
|
||||
"subtitle" -> subtitleStreams.add(gson.fromJson(streamObject, SubtitleStream::class.java))
|
||||
}
|
||||
}
|
||||
|
||||
val parsedStreams = ParsedMediaStreams(
|
||||
videoStream = videoStreams,
|
||||
audioStream = audioStreams,
|
||||
subtitleStream = subtitleStreams
|
||||
)
|
||||
parsedStreams
|
||||
|
||||
} catch (e: Exception) {
|
||||
"Failed to parse data, its either not a valid json structure or expected and required fields are not present.".also {
|
||||
log.error { it }
|
||||
}
|
||||
throw e
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,190 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.data.*
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store.*
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.validator.CompletionValidator
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.NameHelper
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.reader.*
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class PersistContentTaskListener : CoordinatorEventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
var doNotProduceComplete = System.getenv("DISABLE_COMPLETE").toBoolean() ?: false
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
override fun onReady() {
|
||||
super.onReady()
|
||||
if (doNotProduceComplete) {
|
||||
log.warn { "DoNotProduceComplete is set!\n\tNo complete event will be triggered!\n\tTo enable production of complete vents, remove this line in your environment: \"DISABLE_COMPLETE\"" }
|
||||
}
|
||||
}
|
||||
|
||||
override fun shouldIHandleFailedEvents(incomingEvent: Event): Boolean {
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
override val produceEvent: Events = Events.PersistContent
|
||||
override val listensForEvents: List<Events> = listOf(
|
||||
Events.CoverDownloaded,
|
||||
Events.ConvertTaskCompleted,
|
||||
Events.EncodeTaskCompleted,
|
||||
Events.ExtractTaskCompleted
|
||||
)
|
||||
|
||||
|
||||
override fun isPrerequisitesFulfilled(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val started = events.find { it.eventType == Events.ProcessStarted }?.az<MediaProcessStartEvent>()
|
||||
if (started == null) {
|
||||
log.info { "No Start event" }
|
||||
return false
|
||||
}
|
||||
val viableEvents = events.filter { it.isSuccessful() }
|
||||
|
||||
if (!CompletionValidator.req1(started, events)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!CompletionValidator.req2(started.data?.operations ?: emptyList(), viableEvents)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!CompletionValidator.req3(started.data?.operations ?: emptyList(), events)) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!CompletionValidator.req4(events)) {
|
||||
return false
|
||||
}
|
||||
|
||||
return super.isPrerequisitesFulfilled(incomingEvent, events)
|
||||
}
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
if (doNotProduceComplete) {
|
||||
return false
|
||||
}
|
||||
val result = super.shouldIProcessAndHandleEvent(incomingEvent, events)
|
||||
return result
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume() ?: return
|
||||
active = true
|
||||
|
||||
if (doNotProduceComplete) {
|
||||
return
|
||||
}
|
||||
|
||||
val mediaInfo: ComposedMediaInfo = composeMediaInfo(events) ?: run {
|
||||
log.error { "Unable to compose media info for ${event.referenceId()}" }
|
||||
return
|
||||
}
|
||||
|
||||
val existingTitles = ContentTitleStore.findMasterTitles(mediaInfo.titles)
|
||||
|
||||
val usableCollection: String = if (existingTitles.isNotEmpty())
|
||||
ContentCatalogStore.getCollectionByTitleAndType(mediaInfo.type, existingTitles) ?: run {
|
||||
log.warn { "Did not receive collection based on titles provided in list ${existingTitles.joinToString(",")}, falling back to fallbackCollection: ${mediaInfo.fallbackCollection}" }
|
||||
mediaInfo.fallbackCollection
|
||||
} else mediaInfo.fallbackCollection
|
||||
|
||||
val mover = ContentCompletionMover(usableCollection, events)
|
||||
|
||||
val newCoverPath = mover.moveCover()
|
||||
val newVideoPath = mover.moveVideo()
|
||||
val newSubtitles = mover.moveSubtitles()
|
||||
|
||||
|
||||
val contentEvent = PersistedContent(
|
||||
cover = newCoverPath?.let { PersistedItem(it.first, it.second) },
|
||||
video = newVideoPath?.let { PersistedItem(it.first, it.second) },
|
||||
subtitles = newSubtitles?.map { PersistedItem(it.source, it.destination) } ?: emptyList()
|
||||
)
|
||||
|
||||
onProduceEvent(PersistedContentEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = contentEvent
|
||||
))
|
||||
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(
|
||||
PersistedContentEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
internal data class ComposedMediaInfo(
|
||||
val title: String,
|
||||
val fallbackCollection: String,
|
||||
|
||||
val titles: List<String>,
|
||||
val type: String,
|
||||
val summaries: List<SummaryInfo>,
|
||||
val genres: List<String>
|
||||
)
|
||||
|
||||
private fun composeMediaInfo(events: List<Event>): ComposedMediaInfo? {
|
||||
val baseInfo =
|
||||
events.find { it.eventType == Events.BaseInfoRead }?.az<BaseInfoEvent>()?.let {
|
||||
it.data
|
||||
} ?: run {
|
||||
log.info { "Cant find BaseInfoEvent on ${Events.BaseInfoRead}" }
|
||||
return null
|
||||
}
|
||||
val metadataInfo =
|
||||
events.find { it.eventType == Events.MetadataSearchPerformed }?.az<MediaMetadataReceivedEvent>()?.data
|
||||
?: run {
|
||||
log.info { "Cant find MediaMetadataReceivedEvent on ${Events.MetadataSearchPerformed}" }
|
||||
null
|
||||
}
|
||||
val mediaInfo: MediaInfo = events.find { it.eventType == Events.ReadOutNameAndType }
|
||||
?.az<MediaOutInformationConstructedEvent>()?.let {
|
||||
it.data?.toValueObject()
|
||||
} ?: run {
|
||||
log.info { "Cant find MediaOutInformationConstructedEvent on ${Events.ReadOutNameAndType}" }
|
||||
return null
|
||||
}
|
||||
|
||||
val summaries = metadataInfo?.summary?.filter { it.summary != null }
|
||||
?.map { SummaryInfo(language = it.language, summary = it.summary!!) } ?: emptyList()
|
||||
|
||||
val titles: MutableList<String> = mutableListOf(mediaInfo.title)
|
||||
metadataInfo?.let {
|
||||
titles.addAll(it.altTitle)
|
||||
titles.add(it.title)
|
||||
titles.add(NameHelper.normalize(it.title))
|
||||
}
|
||||
|
||||
return ComposedMediaInfo(
|
||||
title = NameHelper.normalize(metadataInfo?.title ?: mediaInfo.title),
|
||||
fallbackCollection = baseInfo.title,
|
||||
titles = titles,
|
||||
type = metadataInfo?.type ?: mediaInfo.type,
|
||||
summaries = summaries,
|
||||
genres = metadataInfo?.genres ?: emptyList()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,114 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.listeners
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.JsonObject
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.ConsumableEvent
|
||||
import no.iktdev.eventi.core.WGson
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.dataAs
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEventListener
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.runner.CodeToOutput
|
||||
import no.iktdev.mediaprocessing.shared.common.runner.getOutputUsing
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
|
||||
@Service
|
||||
class ReadMediaFileStreamsTaskListener() : CoordinatorEventListener() {
|
||||
|
||||
override fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
@Autowired
|
||||
override var coordinator: Coordinator? = null
|
||||
|
||||
val log = KotlinLogging.logger {}
|
||||
val requiredOperations = listOf(OperationEvents.ENCODE, OperationEvents.EXTRACT)
|
||||
|
||||
override val produceEvent: Events = Events.StreamRead
|
||||
override val listensForEvents: List<Events> = listOf(Events.ProcessStarted)
|
||||
|
||||
override fun shouldIProcessAndHandleEvent(incomingEvent: Event, events: List<Event>): Boolean {
|
||||
val status = super.shouldIProcessAndHandleEvent(incomingEvent, events)
|
||||
val permittedOperations = events.findFirstEventOf<MediaProcessStartEvent>()?.data?.operations ?: return false
|
||||
return if (permittedOperations.any { it in requiredOperations }) {
|
||||
status
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
override fun onEventsReceived(incomingEvent: ConsumableEvent<Event>, events: List<Event>) {
|
||||
val event = incomingEvent.consume()
|
||||
if (event == null) {
|
||||
log.error { "Event is null and should not be available! ${WGson.gson.toJson(incomingEvent.metadata())}" }
|
||||
return
|
||||
}
|
||||
active = true
|
||||
|
||||
val startEvent = event.dataAs<StartEventData>()
|
||||
if (startEvent == null || !startEvent.operations.any { it in requiredOperations }) {
|
||||
log.info { "${event.metadata.referenceId} does not contain a operation in ${requiredOperations.joinToString(",") { it.name }}" }
|
||||
active = false
|
||||
return
|
||||
}
|
||||
val result = runBlocking {
|
||||
try {
|
||||
val data = fileReadStreams(startEvent, event.metadata.eventId)
|
||||
MediaFileStreamsReadEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Success, getProducerName()),
|
||||
data = data
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
MediaFileStreamsReadEvent(
|
||||
metadata = event.makeDerivedEventInfo(EventStatus.Failed, getProducerName())
|
||||
)
|
||||
}
|
||||
}
|
||||
onProduceEvent(result)
|
||||
active = false
|
||||
}
|
||||
|
||||
override fun produceFailure(incomingEvent: Event) {
|
||||
onProduceEvent(
|
||||
MediaFileStreamsReadEvent(
|
||||
metadata = incomingEvent.makeDerivedEventInfo(EventStatus.Failed, getProducerName()),
|
||||
data = null
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
suspend fun fileReadStreams(started: StartEventData, eventId: String): JsonObject? {
|
||||
val file = File(started.file)
|
||||
return if (file.exists() && file.isFile) {
|
||||
val result = readStreams(file)
|
||||
val joined = result.output.joinToString(" ")
|
||||
Gson().fromJson(joined, JsonObject::class.java)
|
||||
} else {
|
||||
val message = "File in data is not a file or does not exist: ${file.absolutePath}".also {
|
||||
log.error { it }
|
||||
}
|
||||
throw RuntimeException(message)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun readStreams(file: File): CodeToOutput {
|
||||
val result = getOutputUsing(
|
||||
SharedConfig.ffprobe,
|
||||
"-v", "quiet", "-print_format", "json", "-show_streams", file.absolutePath
|
||||
)
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping
|
||||
|
||||
import no.iktdev.exfl.using
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams.AudioArguments
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams.VideoArguments
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.EncodeArgumentData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.AudioStream
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.EncodingPreference
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.ParsedMediaStreams
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.VideoStream
|
||||
import java.io.File
|
||||
|
||||
class EncodeWorkArgumentsMapping(
|
||||
val inputFile: String,
|
||||
val outFileFullName: String,
|
||||
val streams: ParsedMediaStreams,
|
||||
val preference: EncodingPreference
|
||||
) {
|
||||
|
||||
fun getArguments(): EncodeArgumentData? {
|
||||
val vaas = VideoAndAudioSelector(streams, preference)
|
||||
val vArg = vaas.getVideoStream()
|
||||
?.let { VideoArguments(it, streams, preference.video).getVideoArguments() }
|
||||
val aArg = vaas.getAudioStream()
|
||||
?.let { AudioArguments(it, streams, preference.audio).getAudioArguments() }
|
||||
|
||||
val vaArgs = toFfmpegWorkerArguments(vArg, aArg)
|
||||
return if (vaArgs.isEmpty()) {
|
||||
null
|
||||
} else {
|
||||
EncodeArgumentData(
|
||||
inputFile = inputFile,
|
||||
outputFileName = "${outFileFullName}.mp4",
|
||||
arguments = vaArgs
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private class VideoAndAudioSelector(val mediaStreams: ParsedMediaStreams, val preference: EncodingPreference) {
|
||||
private var defaultVideoSelected: VideoStream? = mediaStreams.videoStream
|
||||
.filter { (it.duration_ts ?: 0) > 0 }
|
||||
.maxByOrNull { it.duration_ts ?: 0 } ?: mediaStreams.videoStream.minByOrNull { it.index }
|
||||
private var defaultAudioSelected: AudioStream? = mediaStreams.audioStream
|
||||
.filter { (it.duration_ts ?: 0) > 0 }
|
||||
.maxByOrNull { it.duration_ts ?: 0 } ?: mediaStreams.audioStream.minByOrNull { it.index }
|
||||
|
||||
fun getVideoStream(): VideoStream? {
|
||||
return defaultVideoSelected
|
||||
}
|
||||
|
||||
fun getAudioStream(): AudioStream? {
|
||||
val languageFiltered = mediaStreams.audioStream.filter { it.tags.language == preference.audio.language }
|
||||
val channeledAndCodec = languageFiltered.find {
|
||||
it.channels >= (preference.audio.channels ?: 2) && it.codec_name == preference.audio.codec.lowercase()
|
||||
}
|
||||
return channeledAndCodec ?: return languageFiltered.minByOrNull { it.index } ?: defaultAudioSelected
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,68 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping
|
||||
|
||||
import no.iktdev.eventi.data.dataAs
|
||||
import no.iktdev.eventi.data.isSuccessful
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventSummary
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationsSummary
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OutputFiles
|
||||
import no.iktdev.mediaprocessing.shared.common.getChecksum
|
||||
|
||||
class EventsSummaryMapping {
|
||||
|
||||
fun map(events: List<Event>): EventSummary {
|
||||
val startOperations = events.find { it.eventType == Events.ProcessStarted }?.dataAs<StartEventData>() ?: throw RuntimeException("No start event found")
|
||||
val successOperations = listOfNotNull(
|
||||
if (isEncodedSuccessful(events)) OperationEvents.ENCODE else null,
|
||||
if (isExtractedSuccessful(events)) OperationEvents.EXTRACT else null,
|
||||
if (isConvertedSuccessful(events)) OperationEvents.CONVERT else null
|
||||
)
|
||||
|
||||
|
||||
return EventSummary(
|
||||
inputFile = startOperations.file,
|
||||
inputFileChecksum = getChecksum(startOperations.file),
|
||||
operationsSummary = OperationsSummary(
|
||||
requestedOperations = startOperations.operations,
|
||||
completedOperations = successOperations
|
||||
),
|
||||
outputFiles = getProducesFiles(events)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
fun isEncodedSuccessful(events: List<Event>) = events.filter { it.eventType == Events.EncodeTaskCompleted }.any { it.isSuccessful() }
|
||||
fun isExtractedSuccessful(events: List<Event>) = events.filter { it.eventType == Events.ExtractTaskCompleted }.any { it.isSuccessful() }
|
||||
fun isConvertedSuccessful(events: List<Event>) = events.filter { it.eventType == Events.ConvertTaskCompleted }.any { it.isSuccessful() }
|
||||
|
||||
fun getProducesFiles(events: List<Event>): OutputFiles {
|
||||
val encoded = if (isEncodedSuccessful(events)) {
|
||||
events.filter { it.eventType == Events.EncodeTaskCompleted }
|
||||
.filter { it.isSuccessful() }
|
||||
.mapNotNull { it.dataAs<EncodedData>()?.outputFile }
|
||||
} else emptyList()
|
||||
|
||||
val extracted = if (isExtractedSuccessful(events)) {
|
||||
events.filter { it.eventType == Events.ExtractTaskCompleted }
|
||||
.filter { it.isSuccessful() }
|
||||
.mapNotNull { it.dataAs<ExtractedData>() }
|
||||
.map { it.outputFile }
|
||||
} else emptyList()
|
||||
|
||||
val converted = if (isConvertedSuccessful(events)) {
|
||||
events.filter { it.eventType == Events.ConvertTaskCompleted }
|
||||
.filter { it.isSuccessful() }
|
||||
.mapNotNull { it.dataAs<ConvertedData>() }
|
||||
.flatMap { it.outputFiles }
|
||||
} else emptyList()
|
||||
|
||||
return OutputFiles(
|
||||
encoded = encoded,
|
||||
extracted = extracted,
|
||||
converted = converted
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping
|
||||
|
||||
import no.iktdev.exfl.using
|
||||
import no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams.SubtitleArguments
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ExtractArgumentData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.ParsedMediaStreams
|
||||
import java.io.File
|
||||
|
||||
class ExtractWorkArgumentsMapping(
|
||||
val inputFile: String,
|
||||
val outFileFullName: String,
|
||||
val streams: ParsedMediaStreams
|
||||
) {
|
||||
|
||||
fun getArguments(): List<ExtractArgumentData> {
|
||||
val sArg = SubtitleArguments(streams.subtitleStream).getSubtitleArguments()
|
||||
|
||||
val entries = sArg.map {
|
||||
ExtractArgumentData(
|
||||
inputFile = inputFile,
|
||||
language = it.language,
|
||||
arguments = it.codecParameters + it.optionalParameters + listOf("-map", "0:s:${it.index}"),
|
||||
outputFileName = "${outFileFullName}.${it.language}.${it.format}",
|
||||
storeFileName = outFileFullName
|
||||
)
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.AudioArgumentsDto
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.VideoArgumentsDto
|
||||
|
||||
fun toFfmpegWorkerArguments(
|
||||
videoArguments: VideoArgumentsDto?,
|
||||
audioArguments: AudioArgumentsDto?
|
||||
): List<String> {
|
||||
val arguments = mutableListOf<String>(
|
||||
*videoArguments?.codecParameters?.toTypedArray() ?: arrayOf(),
|
||||
*videoArguments?.optionalParameters?.toTypedArray() ?: arrayOf(),
|
||||
*audioArguments?.codecParameters?.toTypedArray() ?: arrayOf(),
|
||||
*audioArguments?.optionalParameters?.toTypedArray() ?: arrayOf()
|
||||
)
|
||||
videoArguments?.index?.let {
|
||||
arguments.addAll(listOf("-map", "0:v:$it"))
|
||||
}
|
||||
audioArguments?.index?.let {
|
||||
arguments.addAll(listOf("-map", "0:a:$it"))
|
||||
}
|
||||
return arguments
|
||||
}
|
||||
@ -1,178 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.database.withTransaction
|
||||
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.reader.VideoDetails
|
||||
import no.iktdev.streamit.library.db.executeWithStatus
|
||||
import no.iktdev.streamit.library.db.insertWithSuccess
|
||||
import no.iktdev.streamit.library.db.query.MovieQuery
|
||||
import no.iktdev.streamit.library.db.tables.catalog
|
||||
import no.iktdev.streamit.library.db.tables.serie
|
||||
import org.jetbrains.exposed.sql.*
|
||||
|
||||
object ContentCatalogStore {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
/**
|
||||
* Given a list of titles and type,
|
||||
* the codes purpose is to find the matching collection in the catalog by title
|
||||
*/
|
||||
fun getCollectionByTitleAndType(type: String, titles: List<String>): String? {
|
||||
return withTransaction(getStoreDatabase()) {
|
||||
catalog.select {
|
||||
(catalog.type eq type) and
|
||||
((catalog.title inList titles) or
|
||||
(catalog.collection inList titles))
|
||||
}.map {
|
||||
it[catalog.collection]
|
||||
}.firstOrNull()
|
||||
}
|
||||
}
|
||||
|
||||
private fun getCover(collection: String, type: String): String? {
|
||||
return withTransaction(getStoreDatabase()) {
|
||||
catalog.select {
|
||||
(catalog.collection eq collection) and
|
||||
(catalog.type eq type)
|
||||
}.map { it[catalog.cover] }.firstOrNull()
|
||||
}
|
||||
}
|
||||
|
||||
fun storeCatalog(title: String, titles: List<String>, collection: String, type: String, cover: String?, genres: String?): Int? {
|
||||
val status = executeWithStatus(getStoreDatabase().database, block = {
|
||||
val existingRow = catalog.select {
|
||||
(catalog.collection eq collection) and
|
||||
(catalog.type eq type)
|
||||
}.firstOrNull()
|
||||
|
||||
if (existingRow == null) {
|
||||
log.info { "$collection does not exist, and will be created" }
|
||||
catalog.insert {
|
||||
it[catalog.title] = title
|
||||
it[catalog.cover] = cover
|
||||
it[catalog.type] = type
|
||||
it[catalog.collection] = collection
|
||||
it[catalog.genres] = genres
|
||||
}
|
||||
} else {
|
||||
val id = existingRow[catalog.id]
|
||||
val storedTitle = existingRow[catalog.title]
|
||||
val useCover = existingRow[catalog.cover] ?: cover
|
||||
val useGenres = existingRow[catalog.genres] ?: genres
|
||||
|
||||
catalog.update({
|
||||
(catalog.id eq id) and
|
||||
(catalog.collection eq collection)
|
||||
}) {
|
||||
it[catalog.cover] = useCover
|
||||
it[catalog.genres] = useGenres
|
||||
}
|
||||
}
|
||||
}, {
|
||||
log.error { "Failed to store catalog $collection: ${it.message}" }
|
||||
})
|
||||
if (status) {
|
||||
log.info { "$collection was successfully stored!" }
|
||||
} else {
|
||||
log.error { "Unable to store catalog $collection..." }
|
||||
}
|
||||
return getId(title, titles, collection, type)
|
||||
}
|
||||
|
||||
private fun storeMovie(catalogId: Int, videoDetails: VideoDetails) {
|
||||
val iid = MovieQuery(videoDetails.fileName).insertAndGetId() ?: run {
|
||||
log.error { "Movie id was not returned!" }
|
||||
return
|
||||
}
|
||||
val status = executeWithStatus(getStoreDatabase().database, block = {
|
||||
catalog.update({
|
||||
(catalog.id eq catalogId)
|
||||
}) {
|
||||
it[catalog.iid] = iid
|
||||
}
|
||||
}, {
|
||||
log.error { "Failed to store movie ${videoDetails.fileName}: ${it.message}" }
|
||||
})
|
||||
if (status) {
|
||||
log.info { "${videoDetails.fileName} was successfully stored in movies!" }
|
||||
} else {
|
||||
log.error { "Unable to store catalog ${videoDetails.fileName} in movies..." }
|
||||
}
|
||||
}
|
||||
|
||||
private fun storeSerie(collection: String, videoDetails: VideoDetails) {
|
||||
log.info { "Attempting to store $collection!" }
|
||||
val serieInfo = videoDetails.serieInfo ?: run {
|
||||
log.error { "serieInfo in videoDetails is null!" }
|
||||
return
|
||||
}
|
||||
val status = insertWithSuccess(getStoreDatabase().database, block = {
|
||||
serie.insert {
|
||||
it[title] = serieInfo.episodeTitle
|
||||
it[episode] = serieInfo.episodeNumber
|
||||
it[season] = serieInfo.seasonNumber
|
||||
it[video] = videoDetails.fileName
|
||||
it[serie.collection] = collection
|
||||
}
|
||||
}, onError = {
|
||||
log.error { "Failed to store serie ${videoDetails.fileName}: ${it.message}" }
|
||||
})
|
||||
if (!status) {
|
||||
log.error { "Failed to insert ${videoDetails.fileName} with episode: ${serieInfo.episodeNumber} and season ${serieInfo.seasonNumber}" }
|
||||
val finalStatus = insertWithSuccess(getStoreDatabase().database, block = {
|
||||
serie.insert {
|
||||
it[title] = serieInfo.episodeTitle
|
||||
it[episode] = serieInfo.episodeNumber
|
||||
it[season] = 0
|
||||
it[video] = videoDetails.fileName
|
||||
it[serie.collection] = collection
|
||||
}
|
||||
}, { log.error { "Failed to store serie: ${it.message}" } })
|
||||
if (!finalStatus) {
|
||||
log.error { "Failed to insert ${videoDetails.fileName} with fallback season 0" }
|
||||
} else {
|
||||
log.info { "${videoDetails.fileName} was successfully stored in movies with fallback season 0!" }
|
||||
}
|
||||
} else {
|
||||
log.info { "${videoDetails.fileName} was successfully stored in series!" }
|
||||
}
|
||||
}
|
||||
|
||||
fun storeMedia(title: String, titles: List<String>, collection: String, type: String, videoDetails: VideoDetails) {
|
||||
val catalogId = getId(title, titles, collection, type) ?: run {
|
||||
log.warn { "Could not find id for $title with type $type" }
|
||||
return
|
||||
}
|
||||
log.info { "$title is identified as $type" }
|
||||
when (type) {
|
||||
"movie" -> storeMovie(catalogId, videoDetails)
|
||||
"serie" -> storeSerie(collection, videoDetails)
|
||||
else -> {
|
||||
log.error { "$type was provided for the function storeMedia, thus failing" }
|
||||
throw RuntimeException("Illegal type provided")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun getId(title: String, titles: List<String>, collection: String, type: String): Int? {
|
||||
val ids = withTransaction(getStoreDatabase().database) {
|
||||
catalog.select {
|
||||
((catalog.title eq title)
|
||||
or (catalog.collection eq collection)
|
||||
or (catalog.title inList titles)) and
|
||||
(catalog.type eq type)
|
||||
}.map { it[catalog.id].value }
|
||||
} ?: run {
|
||||
log.warn { "No values found on $title with type $type" }
|
||||
return null
|
||||
}
|
||||
if (ids.size > 1) {
|
||||
log.info { "Found ids: ${ids.joinToString(",")}" }
|
||||
}
|
||||
return ids.firstOrNull()
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@ -1,132 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.data.dataAs
|
||||
import no.iktdev.exfl.using
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.getCRC32
|
||||
import no.iktdev.mediaprocessing.shared.common.moveTo
|
||||
import no.iktdev.mediaprocessing.shared.common.notExist
|
||||
import java.io.File
|
||||
|
||||
class ContentCompletionMover(val collection: String, val events: List<Event>) {
|
||||
val log = KotlinLogging.logger {}
|
||||
val storeFolder = SharedConfig.outgoingContent.using(collection)
|
||||
|
||||
init {
|
||||
if (storeFolder.notExist()) {
|
||||
log.info { "Creating missing folders for path ${storeFolder.absolutePath}" }
|
||||
storeFolder.mkdirs()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return Pair<OldPath, NewPath> or null if no file found
|
||||
*/
|
||||
fun moveVideo(): Pair<String, String>? {
|
||||
val encodedFile = events.find { it.eventType == Events.EncodeTaskCompleted }?.dataAs<EncodedData>()?.outputFile?.let {
|
||||
File(it)
|
||||
} ?: return null
|
||||
if (!encodedFile.exists()) {
|
||||
log.error { "Provided file ${encodedFile.absolutePath} does not exist at the given location" }
|
||||
return null
|
||||
}
|
||||
val storeFile = storeFolder.using(encodedFile.name)
|
||||
val result = encodedFile.moveTo(storeFile) {
|
||||
}
|
||||
return if (result) run {
|
||||
log.info { "Moved ${encodedFile.absolutePath} to ${storeFile.absolutePath} for permanent storage and usage" }
|
||||
Pair(encodedFile.absolutePath, storeFile.absolutePath)
|
||||
} else throw RuntimeException("Unable to movie file ${encodedFile.absolutePath} to ${storeFile.absolutePath}")
|
||||
}
|
||||
|
||||
fun moveCover(): Pair<String, String>? {
|
||||
val coverFile = events.find { it.eventType == Events.CoverDownloaded }?.
|
||||
az<MediaCoverDownloadedEvent>()?.data?.absoluteFilePath?.let {
|
||||
File(it)
|
||||
} ?: return null
|
||||
|
||||
if (coverFile.notExist()) {
|
||||
log.error { "Provided file ${coverFile.absolutePath} does not exist at the given location" }
|
||||
return null
|
||||
}
|
||||
val storeFile = storeFolder.using(coverFile.name)
|
||||
if (storeFile.exists() && storeFile.getCRC32() == coverFile.getCRC32()) {
|
||||
return Pair(coverFile.absolutePath, storeFile.absolutePath)
|
||||
}
|
||||
val result = coverFile.moveTo(storeFile)
|
||||
return if (result) {
|
||||
log.info { "Moved ${coverFile.absolutePath} to ${storeFile.absolutePath} for permanent storage and usage" }
|
||||
Pair(coverFile.absolutePath, storeFile.absolutePath)
|
||||
} else null
|
||||
}
|
||||
|
||||
|
||||
|
||||
data class MovableSubtitle(
|
||||
val language: String,
|
||||
val cachedFile: File,
|
||||
val storeFileName: String
|
||||
)
|
||||
|
||||
fun getMovableSubtitles(): List<MovableSubtitle> {
|
||||
val extracted =
|
||||
events.filter { it.eventType == Events.ExtractTaskCompleted }.mapNotNull { it.dataAs<ExtractedData>() }
|
||||
val converted =
|
||||
events.filter { it.eventType == Events.ConvertTaskCompleted }.mapNotNull { it.dataAs<ConvertedData>() }
|
||||
|
||||
val items = mutableListOf<MovableSubtitle>()
|
||||
|
||||
extracted.map { MovableSubtitle(
|
||||
language = it.language,
|
||||
cachedFile = File(it.outputFile),
|
||||
storeFileName = it.storeFileName
|
||||
) }.also { items.addAll(it) }
|
||||
|
||||
converted.flatMap { it.outputFiles.map { outFile ->
|
||||
MovableSubtitle(
|
||||
language = it.language,
|
||||
cachedFile = File(outFile),
|
||||
storeFileName = it.baseName
|
||||
)
|
||||
} }.also { items.addAll(it) }
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
data class MovedSubtitle(
|
||||
val language: String,
|
||||
val source: String,
|
||||
val destination: String
|
||||
)
|
||||
|
||||
fun moveSubtitles(): List<MovedSubtitle>? {
|
||||
val subtitleFolder = storeFolder.using("sub")
|
||||
val moved: MutableList<MovedSubtitle> = mutableListOf()
|
||||
|
||||
val subtitles = getMovableSubtitles()
|
||||
if (subtitles.isEmpty()) {
|
||||
return null
|
||||
}
|
||||
for (movable in subtitles) {
|
||||
val languageFolder = subtitleFolder.using(movable.language).also {
|
||||
if (it.notExist()) {
|
||||
it.mkdirs()
|
||||
}
|
||||
}
|
||||
val storeFile = languageFolder.using("${movable.storeFileName}.${movable.cachedFile.extension}")
|
||||
val success = movable.cachedFile.moveTo(storeFile)
|
||||
if (success) {
|
||||
log.info { "Moved ${movable.cachedFile.absolutePath} to ${storeFile.absolutePath} for permanent storage and usage" }
|
||||
moved.add(MovedSubtitle(movable.language, movable.cachedFile.absolutePath, storeFile.absolutePath))
|
||||
}
|
||||
}
|
||||
|
||||
return moved
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
|
||||
|
||||
import no.iktdev.eventi.database.withTransaction
|
||||
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
|
||||
import no.iktdev.streamit.library.db.query.GenreQuery
|
||||
|
||||
object ContentGenresStore {
|
||||
fun storeAndGetIds(genres: List<String>): String? {
|
||||
return try {
|
||||
withTransaction(getStoreDatabase()) {
|
||||
val gq = GenreQuery( *genres.toTypedArray() )
|
||||
gq.insertAndGetIds()
|
||||
gq.getIds().joinToString(",")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.reader.SummaryInfo
|
||||
import no.iktdev.streamit.library.db.executeOrException
|
||||
import no.iktdev.streamit.library.db.query.SummaryQuery
|
||||
|
||||
object ContentMetadataStore {
|
||||
|
||||
fun storeSummary(catalogId: Int, summaryInfo: SummaryInfo) {
|
||||
val result = executeOrException(getStoreDatabase().database, block = {
|
||||
SummaryQuery(
|
||||
cid = catalogId,
|
||||
language = summaryInfo.language,
|
||||
description = summaryInfo.summary
|
||||
).insert()
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
|
||||
import no.iktdev.streamit.library.db.executeWithStatus
|
||||
import no.iktdev.streamit.library.db.query.SubtitleQuery
|
||||
import no.iktdev.streamit.library.db.tables.subtitle
|
||||
import org.jetbrains.exposed.sql.insert
|
||||
import java.io.File
|
||||
|
||||
object ContentSubtitleStore {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
fun storeSubtitles(collection: String, destinationFile: File): Boolean {
|
||||
return executeWithStatus (getStoreDatabase().database, block = {
|
||||
subtitle.insert {
|
||||
it[this.associatedWithVideo] = destinationFile.nameWithoutExtension
|
||||
it[this.language] = destinationFile.parentFile.nameWithoutExtension
|
||||
it[this.collection] = collection
|
||||
it[this.format] = destinationFile.extension.uppercase()
|
||||
it[this.subtitle] = destinationFile.name
|
||||
}
|
||||
}, onError = {
|
||||
log.error { "Failed to store subtitle $destinationFile: ${it.message}" }
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.getStoreDatabase
|
||||
import no.iktdev.mediaprocessing.shared.common.parsing.NameHelper
|
||||
import no.iktdev.streamit.library.db.tables.titles
|
||||
import no.iktdev.streamit.library.db.withTransaction
|
||||
import org.jetbrains.exposed.sql.insertIgnore
|
||||
import org.jetbrains.exposed.sql.or
|
||||
import org.jetbrains.exposed.sql.select
|
||||
|
||||
object ContentTitleStore {
|
||||
|
||||
fun store(mainTitle: String, otherTitles: List<String>) {
|
||||
try {
|
||||
withTransaction(getStoreDatabase().database, block = {
|
||||
val titlesToUse = otherTitles + listOf(
|
||||
NameHelper.normalize(mainTitle)
|
||||
).filter { it != mainTitle }
|
||||
|
||||
titlesToUse.forEach { t ->
|
||||
titles.insertIgnore {
|
||||
it[masterTitle] = mainTitle
|
||||
it[alternativeTitle] = t
|
||||
}
|
||||
}
|
||||
}, {
|
||||
|
||||
})
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
|
||||
fun findMasterTitles(titleList: List<String>): List<String> {
|
||||
return withTransaction(getStoreDatabase().database, block = {
|
||||
titles.select {
|
||||
(titles.alternativeTitle inList titleList) or
|
||||
(titles.masterTitle inList titleList)
|
||||
}.map {
|
||||
it[titles.masterTitle]
|
||||
}.distinctBy { it }
|
||||
}, {
|
||||
|
||||
}) ?: emptyList()
|
||||
}
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.store
|
||||
|
||||
import com.google.gson.Gson
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.coordinator.eventDatabase
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventSummary
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.filesProcessed
|
||||
import no.iktdev.mediaprocessing.shared.common.getChecksum
|
||||
import no.iktdev.streamit.library.db.withTransaction
|
||||
import org.jetbrains.exposed.sql.insert
|
||||
|
||||
object ProcessedFileStore {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
fun store(title: String, events: List<Event>, summary: EventSummary) {
|
||||
val inputFilePath = events.findFirstEventOf<MediaProcessStartEvent>()?.data?.file ?: return
|
||||
val checksum = getChecksum(inputFilePath)
|
||||
|
||||
|
||||
withTransaction(eventDatabase.database.database, block = {
|
||||
filesProcessed.insert {
|
||||
it[this.title] = title
|
||||
it[this.inputFile] = inputFilePath
|
||||
it[this.data] = Gson().toJson(summary)
|
||||
it[this.checksum] = checksum
|
||||
}
|
||||
}) {
|
||||
it.printStackTrace()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,81 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.AudioArgumentsDto
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.AudioPreference
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.AudioStream
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.ParsedMediaStreams
|
||||
|
||||
class AudioArguments(
|
||||
val audioStream: AudioStream,
|
||||
val allStreams: ParsedMediaStreams,
|
||||
val preference: AudioPreference
|
||||
) {
|
||||
fun isAudioCodecEqual() = audioStream.codec_name.lowercase() == preference.codec.lowercase()
|
||||
|
||||
/**
|
||||
* Checks whether its ac3 or eac3, as google cast supports this and most other devices.
|
||||
*/
|
||||
fun isOfSupportedSurroundCodec(): Boolean {
|
||||
return audioStream.codec_name.lowercase() in listOf("eac3", "ac3")
|
||||
}
|
||||
|
||||
|
||||
fun isSurround(): Boolean {
|
||||
return audioStream.channels > 2
|
||||
}
|
||||
|
||||
|
||||
fun getAudioArguments(): AudioArgumentsDto {
|
||||
val optionalParams = mutableListOf<String>()
|
||||
|
||||
if (!isSurround()) {
|
||||
return if (isAudioCodecEqual()) {
|
||||
asPassthrough()
|
||||
} else {
|
||||
asStereo()
|
||||
}
|
||||
} else {
|
||||
if (preference.forceStereo) {
|
||||
return asStereo()
|
||||
}
|
||||
if (preference.passthroughOnGenerallySupportedSurroundSound && isOfSupportedSurroundCodec()) {
|
||||
return asPassthrough()
|
||||
} else if (preference.passthroughOnGenerallySupportedSurroundSound && !isOfSupportedSurroundCodec()) {
|
||||
return asSurround()
|
||||
}
|
||||
if (preference.convertToEac3OnUnsupportedSurround ) {
|
||||
return asSurround()
|
||||
}
|
||||
|
||||
|
||||
return asStereo()
|
||||
}
|
||||
}
|
||||
|
||||
fun index(): Int {
|
||||
return allStreams.audioStream.indexOf(audioStream)
|
||||
}
|
||||
|
||||
fun asPassthrough(): AudioArgumentsDto {
|
||||
return AudioArgumentsDto(
|
||||
index = index()
|
||||
)
|
||||
}
|
||||
fun asStereo(): AudioArgumentsDto {
|
||||
return AudioArgumentsDto(
|
||||
index = index(),
|
||||
codecParameters = listOf(
|
||||
"-c:a", preference.codec,
|
||||
"-ac", "2"
|
||||
), optionalParameters = emptyList()
|
||||
)
|
||||
}
|
||||
fun asSurround(): AudioArgumentsDto {
|
||||
return AudioArgumentsDto(
|
||||
index = index(),
|
||||
codecParameters = listOf(
|
||||
"-c:a", "eac3"
|
||||
), optionalParameters = emptyList()
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,119 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.SubtitleArgumentsDto
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.SubtitleStream
|
||||
import kotlin.math.sqrt
|
||||
|
||||
class SubtitleArguments(val subtitleStreams: List<SubtitleStream>) {
|
||||
/**
|
||||
* @property DEFAULT is default subtitle as dialog
|
||||
* @property CC is Closed-Captions
|
||||
* @property SHD is Hard of hearing
|
||||
* @property NON_DIALOGUE is for Signs or Song (as in lyrics)
|
||||
*/
|
||||
private enum class SubtitleType {
|
||||
DEFAULT,
|
||||
CC,
|
||||
SHD,
|
||||
NON_DIALOGUE
|
||||
}
|
||||
|
||||
private fun SubtitleStream.isCC(): Boolean {
|
||||
if ((this.disposition?.captions ?: 0) > 0) {
|
||||
return true
|
||||
}
|
||||
val title = this.tags.title?.lowercase() ?: return false
|
||||
val keywords = listOf("cc", "closed caption")
|
||||
return keywords.any { title.contains(it) }
|
||||
}
|
||||
|
||||
private fun SubtitleStream.isSHD(): Boolean {
|
||||
if ((this.disposition?.hearing_impaired ?: 0) > 0) {
|
||||
return true
|
||||
}
|
||||
val title = this.tags.title?.lowercase() ?: return false
|
||||
val keywords = listOf("shd", "hh", "Hard-of-Hearing", "Hard of Hearing")
|
||||
return keywords.any { title.contains(it) }
|
||||
}
|
||||
|
||||
private fun SubtitleStream.isSignOrSong(): Boolean {
|
||||
if ((this.disposition?.lyrics ?: 0) > 0) {
|
||||
return true
|
||||
}
|
||||
val title = this.tags.title?.lowercase() ?: return false
|
||||
val keywords = listOf("song", "songs", "sign", "signs")
|
||||
return keywords.any { title.contains(it) }
|
||||
}
|
||||
|
||||
private fun SubtitleStream.isNonDialog(): Boolean {
|
||||
val title = this.tags.title?.lowercase() ?: return false
|
||||
val keywords = listOf("commentary")
|
||||
return keywords.any { title.contains(it) }
|
||||
}
|
||||
|
||||
private fun getSubtitleType(stream: SubtitleStream): SubtitleType {
|
||||
return if (stream.isSignOrSong() || stream.isNonDialog())
|
||||
SubtitleType.NON_DIALOGUE
|
||||
else if (stream.isSHD()) {
|
||||
SubtitleType.SHD
|
||||
} else if (stream.isCC()) {
|
||||
SubtitleType.CC
|
||||
} else SubtitleType.DEFAULT
|
||||
}
|
||||
|
||||
fun getSubtitleArguments(): List<SubtitleArgumentsDto> {
|
||||
val acceptable = subtitleStreams.filter { !it.isSignOrSong() }
|
||||
val codecFiltered = acceptable.filter { getFormatToCodec(it.codec_name) != null }
|
||||
|
||||
val languageGrouped = codecFiltered.groupBy { it.tags.language ?: "eng" }
|
||||
|
||||
val streamsToExtract = languageGrouped.mapNotNull { item ->
|
||||
val itemToType = item.value.map { it to getSubtitleType(it) }
|
||||
val usableSubtitles = itemToType.filter { it.second == SubtitleType.DEFAULT }.ifEmpty { itemToType }
|
||||
val excludedLowFrameCount = excludeLowFrameCount(usableSubtitles.map { it.first }).sortedByDescending { it.tags.NUMBER_OF_FRAMES }
|
||||
excludedLowFrameCount.firstOrNull() ?: run {
|
||||
usableSubtitles.map { it.first }.firstOrNull { it.disposition?.default == 1 } ?: usableSubtitles.firstOrNull()?.first
|
||||
}
|
||||
}
|
||||
|
||||
return streamsToExtract.mapNotNull { stream ->
|
||||
getFormatToCodec(stream.codec_name)?.let { format ->
|
||||
SubtitleArgumentsDto(
|
||||
mediaIndex = stream.index,
|
||||
index = subtitleStreams.indexOf(stream),
|
||||
language = stream.tags.language ?: "eng",
|
||||
format = format
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fun excludeLowFrameCount(streams: List<SubtitleStream>): List<SubtitleStream> {
|
||||
val usable = streams.filter { (it.tags.NUMBER_OF_FRAMES ?: 0) > 0 }
|
||||
val mean = usable.mapNotNull { it.tags.NUMBER_OF_FRAMES }.average()
|
||||
val variance = usable.map { (it.tags.NUMBER_OF_FRAMES!! - mean) * (it.tags.NUMBER_OF_FRAMES!! - mean) }.average()
|
||||
val standardDeviation = sqrt(variance)
|
||||
|
||||
// Definer intervallet for "normale" rammer: mean ± 2 * standard deviation
|
||||
val lowerBound = mean - 2 * standardDeviation
|
||||
val upperBound = mean + 2 * standardDeviation
|
||||
|
||||
return usable.filter {
|
||||
val frameCount = it.tags.NUMBER_OF_FRAMES ?: 0
|
||||
frameCount.toDouble() in lowerBound..upperBound
|
||||
}
|
||||
}
|
||||
|
||||
fun getFormatToCodec(codecName: String): String? {
|
||||
return when (codecName) {
|
||||
"ass" -> "ass"
|
||||
"subrip" -> "srt"
|
||||
"webvtt", "vtt" -> "vtt"
|
||||
"smi" -> "smi"
|
||||
"hdmv_pgs_subtitle" -> null
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,110 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.mapping.streams
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.log
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.ParsedMediaStreams
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.VideoArgumentsDto
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.VideoPreference
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.VideoStream
|
||||
|
||||
class VideoArguments(
|
||||
val videoStream: VideoStream,
|
||||
val allStreams: ParsedMediaStreams,
|
||||
val preference: VideoPreference
|
||||
) {
|
||||
fun isVideoCodecEqual() = getCodec(videoStream.codec_name) == getCodec(preference.codec.lowercase())
|
||||
fun getCodec(name: String): String {
|
||||
return when (name.lowercase()) {
|
||||
"hevc", "hevec", "h265", "h.265", "libx265" -> "libx265"
|
||||
"h.264", "h264", "libx264" -> "libx264"
|
||||
"vp9", "vp-9", "libvpx-vp9" -> "libvpx-vp9"
|
||||
"av1", "libaom-av1" -> "libaom-av1"
|
||||
"mpeg4", "mp4", "libxvid" -> "libxvid"
|
||||
"vvc", "h.266", "libvvc" -> "libvvc"
|
||||
"vp8", "libvpx" -> "libvpx"
|
||||
else -> name
|
||||
}
|
||||
}
|
||||
|
||||
fun getCodec() = getCodec(videoStream.codec_name)
|
||||
|
||||
|
||||
fun getVideoArguments(): VideoArgumentsDto {
|
||||
val codecParams = if (isVideoCodecEqual()) {
|
||||
if (getCodec() == "libx265") {
|
||||
composeHevcArguments(getCodec())
|
||||
} else {
|
||||
mutableListOf("-c:v", "copy")
|
||||
}
|
||||
} else {
|
||||
when (getCodec(preference.codec.lowercase())) {
|
||||
"libx265" -> composeHevcArguments(getCodec())
|
||||
"libx264" -> composeH264Arguments(getCodec())
|
||||
else -> run {
|
||||
val codec = getCodec(preference.codec.lowercase())
|
||||
log.info { "Unsupported codec found ${codec}, making best effort..." }
|
||||
listOf("-c:v", codec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return VideoArgumentsDto(
|
||||
index = allStreams.videoStream.indexOf(videoStream),
|
||||
codecParameters = codecParams,
|
||||
optionalParameters = composeOptionalArguments()
|
||||
)
|
||||
}
|
||||
|
||||
private fun composeOptionalArguments(): List<String> {
|
||||
|
||||
val pixelFormat: List<String> = if (preference.pixelFormatPassthrough.none { it == videoStream.pix_fmt }) {
|
||||
listOf("-pix_fmt", preference.pixelFormat)
|
||||
} else emptyList()
|
||||
|
||||
val crfParam = if (pixelFormat.isNotEmpty() || !isVideoCodecEqual()) {
|
||||
listOf("-crf", preference.threshold.toString())
|
||||
} else emptyList()
|
||||
|
||||
val defaultCodecParams = listOf("-movflags", "+faststart")
|
||||
|
||||
return pixelFormat + crfParam + defaultCodecParams
|
||||
}
|
||||
|
||||
private fun composeH264Arguments(codec: String): List<String> {
|
||||
return listOf(
|
||||
"-c:v", "libx264",
|
||||
"-profile:v", "high",
|
||||
"-level:v", preference.h264Level.toString(),
|
||||
"preset", "slow",
|
||||
)
|
||||
}
|
||||
|
||||
private fun composeHevcArguments(codec: String): List<String> {
|
||||
val targetProfile = if (videoStream.pix_fmt.contains("10")) "main10" else "main"
|
||||
|
||||
val unsetCodecMetadata = videoStream.codec_tag_string == "[0][0][0][0]" || videoStream.codec_tag == "0x0000"
|
||||
|
||||
// Map level til en streng – her forenklet
|
||||
val targetLevel = when (videoStream.level) {
|
||||
150 -> "5.0"
|
||||
153 -> "5.1"
|
||||
else -> "5.0" // Default hvis vi ikke har en eksplisitt mapping
|
||||
}
|
||||
|
||||
return if (codec != "libx265" || (unsetCodecMetadata && preference.reencodeOnIncorrectMetadataForChromecast)) {
|
||||
// Konverter (eller reenkode) til HEVC med x265 med riktige parametere
|
||||
listOf(
|
||||
"-c:v", "libx265", "-preset", "slow",
|
||||
"-x265-params", "\"profile=$targetProfile:level=$targetLevel\"",
|
||||
"-tag:v", "hev1"
|
||||
)
|
||||
} else {
|
||||
// Dersom vi mener at vi kun trenger å remuxe, kan vi gjøre
|
||||
listOf(
|
||||
"-c:v", "copy", "-tag:v", "hev1"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,119 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.tasksV2.validator
|
||||
|
||||
import no.iktdev.eventi.data.dataAs
|
||||
import no.iktdev.eventi.data.isSkipped
|
||||
import no.iktdev.eventi.data.isSuccessful
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.SubtitleFormats
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.isOnly
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* Validates whether all the required work has been created and processed in accordance with expected behaviour and sequence
|
||||
*/
|
||||
object CompletionValidator {
|
||||
|
||||
/**
|
||||
* Checks whether it requires encode or extract or both, and it has created events with args
|
||||
*/
|
||||
fun req1(started: MediaProcessStartEvent, events: List<Event>): Boolean {
|
||||
val encodeFulfilledOrSkipped = if (started.data?.operations?.contains(OperationEvents.ENCODE) == true) {
|
||||
events.any { it.eventType == Events.EncodeParameterCreated }
|
||||
} else true
|
||||
|
||||
val extractFulfilledOrSkipped = if (started.data?.operations?.contains(OperationEvents.EXTRACT) == true) {
|
||||
events.any { it.eventType == Events.ExtractParameterCreated }
|
||||
} else true
|
||||
|
||||
if (!encodeFulfilledOrSkipped || !extractFulfilledOrSkipped) {
|
||||
return false
|
||||
} else return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether work that was supposed to be created has been created.
|
||||
* Checks if all subtitles that can be processed has been created if convert is set.
|
||||
*/
|
||||
fun req2(operations: List<OperationEvents>, events: List<Event>): Boolean {
|
||||
if (OperationEvents.ENCODE in operations) {
|
||||
val encodeParamter = events.find { it.eventType == Events.EncodeParameterCreated }?.az<EncodeArgumentCreatedEvent>()
|
||||
val encodeWork = events.find { it.eventType == Events.EncodeTaskCreated }
|
||||
if (encodeParamter?.isSuccessful() == true && (encodeWork == null))
|
||||
return false
|
||||
}
|
||||
|
||||
val extractParamter = events.find { it.eventType == Events.ExtractParameterCreated }?.az<ExtractArgumentCreatedEvent>()
|
||||
val extractWork = events.filter { it.eventType == Events.ExtractTaskCreated }
|
||||
if (OperationEvents.EXTRACT in operations) {
|
||||
if (extractParamter?.isSuccessful() == true && extractParamter.data?.size != extractWork.size)
|
||||
return false
|
||||
}
|
||||
|
||||
if (OperationEvents.CONVERT in operations) {
|
||||
val convertWork = events.filter { it.eventType == Events.ConvertTaskCreated }
|
||||
|
||||
val supportedSubtitleFormats = SubtitleFormats.entries.map { it.name }
|
||||
val eventsSupportsConvert = extractWork.filter { it.data is ExtractArgumentData }
|
||||
.filter { (it.dataAs<ExtractArgumentData>()?.outputFileName?.let { f -> File(f).extension.uppercase() } in supportedSubtitleFormats) }
|
||||
|
||||
if (convertWork.size != eventsSupportsConvert.size)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether all work that has been created has been completed
|
||||
*/
|
||||
fun req3(operations: List<OperationEvents>, events: List<Event>): Boolean {
|
||||
if (OperationEvents.ENCODE in operations) {
|
||||
val encodeWork = events.filter { it.eventType == Events.EncodeTaskCreated }
|
||||
val encodePerformed = events.filter { it.eventType == Events.EncodeTaskCompleted }
|
||||
if (encodePerformed.size < encodeWork.size)
|
||||
return false
|
||||
}
|
||||
|
||||
if (OperationEvents.EXTRACT in operations) {
|
||||
val extractWork = events.filter { it.eventType == Events.ExtractTaskCreated }
|
||||
val extractPerformed = events.filter { it.eventType == Events.ExtractTaskCompleted }
|
||||
if (extractPerformed.size < extractWork.size)
|
||||
return false
|
||||
}
|
||||
|
||||
if (OperationEvents.CONVERT in operations) {
|
||||
val convertWork = events.filter { it.eventType == Events.ConvertTaskCreated }
|
||||
val convertPerformed = events.filter { it.eventType == Events.ConvertTaskCompleted }
|
||||
if (convertPerformed.size < convertWork.size || (operations.isOnly(OperationEvents.CONVERT) && convertWork.isEmpty()))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if metadata has cover, if so, 2 events are expected
|
||||
*/
|
||||
fun req4(events: List<Event>): Boolean {
|
||||
val metadata = events.find { it.eventType == Events.MetadataSearchPerformed }
|
||||
if (metadata?.isSkipped() == true) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (metadata?.isSuccessful() != true) {
|
||||
return true
|
||||
}
|
||||
|
||||
val hasCover = metadata.dataAs<pyMetadata>()?.cover != null
|
||||
if (hasCover == false) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (events.any { it.eventType == Events.ReadOutCover } && events.any { it.eventType == Events.CoverDownloaded }) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@ -1,83 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.watcher
|
||||
|
||||
import kotlinx.coroutines.channels.Channel
|
||||
import kotlinx.coroutines.delay
|
||||
import no.iktdev.mediaprocessing.coordinator.defaultCoroutine
|
||||
import no.iktdev.mediaprocessing.shared.common.isFileAvailable
|
||||
import java.io.File
|
||||
import java.util.UUID
|
||||
|
||||
data class PendingFile(val id: String = UUID.randomUUID().toString(), val file: File, var time: Long = 0)
|
||||
class FileWatcherQueue {
|
||||
private val fileChannel = Channel<PendingFile>()
|
||||
|
||||
fun addToQueue(file: File, onFilePending: (PendingFile) -> Unit, onFileAccessible: (PendingFile) -> Unit) {
|
||||
// Check if the file is accessible
|
||||
if (isFileAvailable(file)) {
|
||||
// If accessible, run the function immediately and return
|
||||
onFileAccessible(PendingFile(file = file))
|
||||
return
|
||||
}
|
||||
|
||||
// Add the file to the channel for processing
|
||||
fileChannel.trySend(PendingFile(file = file))
|
||||
|
||||
// Coroutine to process the file and remove it from the queue when accessible
|
||||
defaultCoroutine.launch {
|
||||
while (true) {
|
||||
delay(500)
|
||||
val currentFile = fileChannel.receive()
|
||||
if (isFileAvailable(currentFile.file)) {
|
||||
onFileAccessible(currentFile)
|
||||
// File is accessible, remove it from the queue
|
||||
removeFromQueue(currentFile.file) { /* Do nothing here as the operation is not intended to be performed here */ }
|
||||
} else {
|
||||
// File is not accessible, put it back in the channel for later processing
|
||||
fileChannel.send(currentFile.apply { time += 500 })
|
||||
onFilePending(currentFile)
|
||||
}
|
||||
}
|
||||
} // https://chat.openai.com/share/f3c8f6ea-603a-40d6-a811-f8fea5067501
|
||||
}
|
||||
|
||||
fun removeFromQueue(file: File, onFileRemoved: (PendingFile) -> Unit) {
|
||||
val currentItems = fileChannel.list()
|
||||
val toRemove = currentItems.filter {
|
||||
if (it.file.isDirectory) it.file.name == file.name else it.file.name == file.name && it.file.parent == file.parent
|
||||
}
|
||||
|
||||
toRemove.let {
|
||||
it.forEach { file -> onFileRemoved(file) }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Extension function to find and remove an element from the channel
|
||||
fun <T> Channel<T>.findAndRemove(predicate: (T) -> Boolean): List<T> {
|
||||
val forRemoved = mutableListOf<T>()
|
||||
val items = mutableListOf<T>()
|
||||
while (true) {
|
||||
val item = tryReceive().getOrNull() ?: break
|
||||
if (predicate(item)) {
|
||||
forRemoved.add(item)
|
||||
}
|
||||
items.add(item)
|
||||
}
|
||||
for (item in items) {
|
||||
trySend(item).isSuccess
|
||||
}
|
||||
return forRemoved
|
||||
}
|
||||
|
||||
fun <T> Channel<T>.list(): List<T> {
|
||||
val items = mutableListOf<T>()
|
||||
while (true) {
|
||||
val item = tryReceive().getOrNull() ?: break
|
||||
items.add(item)
|
||||
trySend(item).isSuccess
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,195 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.watcher
|
||||
|
||||
import dev.vishna.watchservice.KWatchEvent.Kind.Deleted
|
||||
import dev.vishna.watchservice.KWatchEvent.Kind.Initialized
|
||||
import dev.vishna.watchservice.asWatchChannel
|
||||
import jakarta.annotation.PreDestroy
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.channels.consumeEach
|
||||
import kotlinx.coroutines.delay
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.database.executeWithResult
|
||||
import no.iktdev.eventi.database.withTransaction
|
||||
import no.iktdev.mediaprocessing.coordinator.*
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.files
|
||||
import no.iktdev.mediaprocessing.shared.common.extended.isSupportedVideoFile
|
||||
import no.iktdev.mediaprocessing.shared.common.ifNotEmpty
|
||||
import no.iktdev.mediaprocessing.shared.common.md5
|
||||
import no.iktdev.streamit.library.db.executeOrException
|
||||
import no.iktdev.streamit.library.db.withTransaction
|
||||
import org.jetbrains.exposed.sql.insertIgnore
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
|
||||
|
||||
interface FileWatcherEvents {
|
||||
fun onFileAvailable(file: PendingFile)
|
||||
|
||||
/**
|
||||
* If the file is being copied or incomplete, or in case a process currently owns the file, pending should be issued
|
||||
*/
|
||||
fun onFilePending(file: PendingFile)
|
||||
|
||||
/**
|
||||
* If the file is either removed or is not a valid file
|
||||
*/
|
||||
fun onFileFailed(file: PendingFile)
|
||||
|
||||
|
||||
fun onFileRemoved(file: PendingFile)
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Service
|
||||
class InputDirectoryWatcher(@Autowired var coordinator: Coordinator): FileWatcherEvents {
|
||||
|
||||
private val logger = KotlinLogging.logger {}
|
||||
val watchDirectories = SharedConfig.incomingContent
|
||||
val queue = FileWatcherQueue()
|
||||
|
||||
@Volatile
|
||||
private var isStopping: Boolean = false
|
||||
@PreDestroy
|
||||
fun setStop() {
|
||||
isStopping = true
|
||||
}
|
||||
|
||||
|
||||
suspend fun watchFiles() {
|
||||
log.info { "Starting Watcher" }
|
||||
val dirs = watchDirectories.filter { it.exists() && it.isDirectory }
|
||||
if (dirs.isNotEmpty()) {
|
||||
val paths = dirs.joinToString("\n\t") { it.absolutePath }
|
||||
log.info { "Watching directories:\n\t$paths" }
|
||||
|
||||
for (dir in dirs) {
|
||||
val files = dir.listFiles()?.map { it.name }?.joinToString { "\n\t $it" } ?: "No files present.."
|
||||
log.info { "Content present for path ${dir.absolutePath} \n$files" }
|
||||
}
|
||||
}
|
||||
|
||||
//val errorConfiguredDirs = watchDirectories.filter { !it.isDirectory || !it.exists()}.joinToString("\n\t") { it.absolutePath }
|
||||
watchDirectories.filter { !it.isDirectory || !it.exists()}.ifNotEmpty {
|
||||
val errorConfiguredDirs = it.joinToString("\n\t") { it.absolutePath }
|
||||
log.error { "Failed to initialize watcher for the following: \n\t $errorConfiguredDirs" }
|
||||
}
|
||||
for (folder in watchDirectories) {
|
||||
startWatchOnDirectory(folder)
|
||||
}
|
||||
}
|
||||
|
||||
val activeWatchers: MutableList<FileWatcher> = mutableListOf()
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
private suspend fun startWatchOnDirectory(file: File) {
|
||||
if (activeWatchers.any {it -> it.file.absolutePath == file.absolutePath}) {
|
||||
log.error { "Attempting to start a watcher on an already watched directory ${file.absolutePath}" }
|
||||
return
|
||||
}
|
||||
val watcher = file.asWatcher { watcher ->
|
||||
try {
|
||||
watcher.consumeEach {
|
||||
if (it.file == SharedConfig.incomingContent) {
|
||||
logger.info { "IO Watcher ${it.kind} on ${it.file.absolutePath}" }
|
||||
} else {
|
||||
logger.info { "IO Event: ${it.kind}: ${it.file.name}" }
|
||||
}
|
||||
try {
|
||||
when (it.kind) {
|
||||
Deleted -> removeFile(it.file)
|
||||
Initialized -> { /* Do nothing */ }
|
||||
else -> {
|
||||
val added = addFile(it.file)
|
||||
if (!added) {
|
||||
logger.info { "Ignoring event kind: ${it.kind.name} for file ${it.file.name} as it is not a supported video file" }
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
log.error { "Consume failed ${e.message}" }
|
||||
e.printStackTrace()
|
||||
watcher.close()
|
||||
}
|
||||
}.also { watcher ->
|
||||
watcher.watcher.invokeOnClose {
|
||||
it?.printStackTrace()
|
||||
log.warn { "Watcher stopped for ${watcher.file}" }
|
||||
if (!isStopping) {
|
||||
log.info { "Determined that the program is not in a termination stage.. Restarting watcher" }
|
||||
activeWatchers.remove(watcher)
|
||||
ioCoroutine.launch {
|
||||
log.info { "Waiting 500ms before restarting watcher.." }
|
||||
delay(500)
|
||||
startWatchOnDirectory(watcher.file)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
log.info { "Now watching ${file.absolutePath} for files" }
|
||||
activeWatchers.add(watcher)
|
||||
}
|
||||
|
||||
|
||||
init {
|
||||
ioCoroutine.launch {
|
||||
watchFiles()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun addFile(file: File): Boolean {
|
||||
return if (file.isFile && file.isSupportedVideoFile()) {
|
||||
log.info { "Adding ${file.name} to queue" }
|
||||
queue.addToQueue(file, this@InputDirectoryWatcher::onFilePending, this@InputDirectoryWatcher::onFileAvailable)
|
||||
true
|
||||
} else if (file.isDirectory) {
|
||||
log.info { "Searching for files in ${file.name}" }
|
||||
val supportedFiles = file.walkTopDown().filter { f -> f.isFile && f.isSupportedVideoFile() }
|
||||
supportedFiles.forEach { sf ->
|
||||
log.info { "Adding ${sf.name} to queue from folder" }
|
||||
queue.addToQueue(sf, this@InputDirectoryWatcher::onFilePending, this@InputDirectoryWatcher::onFileAvailable)
|
||||
}
|
||||
true
|
||||
} else false
|
||||
}
|
||||
|
||||
private fun removeFile(file: File) {
|
||||
log.info { "Removing file from Queue ${file.name}" }
|
||||
queue.removeFromQueue(file, this@InputDirectoryWatcher::onFileRemoved)
|
||||
}
|
||||
|
||||
override fun onFileAvailable(file: PendingFile) {
|
||||
logger.info { "File available ${file.file.name}" }
|
||||
|
||||
// This sends it to coordinator to start the process
|
||||
executeWithResult(eventDatabase.database.database) {
|
||||
files.insertIgnore {
|
||||
it[baseName] = file.file.nameWithoutExtension
|
||||
it[folder] = file.file.parentFile.absolutePath
|
||||
it[fileName] = file.file.absolutePath
|
||||
it[checksum] = file.file.md5()
|
||||
}
|
||||
}
|
||||
coordinator.startProcess(file.file, ProcessType.FLOW)
|
||||
}
|
||||
|
||||
override fun onFilePending(file: PendingFile) {
|
||||
logger.info { "File pending availability ${file.file.name}" }
|
||||
}
|
||||
|
||||
override fun onFileFailed(file: PendingFile) {
|
||||
logger.warn { "File failed availability ${file.file.name}" }
|
||||
}
|
||||
|
||||
override fun onFileRemoved(file: PendingFile) {
|
||||
logger.info { "File removed ${file.file.name} was removed" }
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.Defaults
|
||||
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.context.annotation.Import
|
||||
import org.springframework.web.client.RestTemplate
|
||||
|
||||
|
||||
|
||||
@Configuration
|
||||
public class DefaultProcesserConfiguration: Defaults() {
|
||||
}
|
||||
|
||||
@Configuration
|
||||
class SocketImplemented: SocketImplementation() {
|
||||
override var additionalOrigins: List<String> = ProcesserEnv.wsAllowedOrigins.split(",")
|
||||
}
|
||||
@ -1,92 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||
import no.iktdev.exfl.observable.Observables
|
||||
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||
import no.iktdev.eventi.database.MySqlDataSource
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.runners
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.tasks
|
||||
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
||||
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||
import org.jetbrains.exposed.sql.transactions.TransactionManager
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||
import org.springframework.boot.runApplication
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import javax.annotation.PreDestroy
|
||||
|
||||
|
||||
private val logger = KotlinLogging.logger {}
|
||||
val ioCoroutine = CoroutinesIO()
|
||||
val defaultCoroutine = CoroutinesDefault()
|
||||
|
||||
|
||||
@SpringBootApplication
|
||||
class ProcesserApplication {
|
||||
|
||||
@PreDestroy
|
||||
fun onShutdown() {
|
||||
doTransactionalCleanup()
|
||||
}
|
||||
|
||||
@Transactional
|
||||
fun doTransactionalCleanup() {
|
||||
runnerManager.unlist()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private lateinit var eventsDatabase: MySqlDataSource
|
||||
fun getEventsDatabase(): MySqlDataSource {
|
||||
return eventsDatabase
|
||||
}
|
||||
|
||||
|
||||
lateinit var taskManager: TasksManager
|
||||
lateinit var runnerManager: RunnerManager
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
defaultCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
|
||||
eventsDatabase = DatabaseEnvConfig.toEventsDatabase()
|
||||
eventsDatabase.createDatabase()
|
||||
eventsDatabase.createTables(tasks, runners)
|
||||
|
||||
taskManager = TasksManager(eventsDatabase)
|
||||
|
||||
runnerManager = RunnerManager(dataSource = getEventsDatabase(), applicationName = ProcesserApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
|
||||
runApplication<ProcesserApplication>(*args)
|
||||
log.info { "App Version: ${getAppVersion()}" }
|
||||
|
||||
}
|
||||
|
||||
@EnableScheduling
|
||||
class DatabaseReconnect() {
|
||||
var lostConnectionCount = 0
|
||||
@Scheduled(fixedDelay = (100_000))
|
||||
fun checkIfConnected() {
|
||||
if (TransactionManager.currentOrNull() == null) {
|
||||
lostConnectionCount++
|
||||
eventsDatabase.toDatabase()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer
|
||||
|
||||
import no.iktdev.exfl.using
|
||||
import java.io.File
|
||||
|
||||
class ProcesserEnv {
|
||||
companion object {
|
||||
val wsAllowedOrigins: String = System.getenv("AllowedOriginsWebsocket")?.takeIf { it.isNotBlank() } ?: ""
|
||||
|
||||
val ffmpeg: String = System.getenv("SUPPORTING_EXECUTABLE_FFMPEG") ?: "ffmpeg"
|
||||
val allowOverwrite = System.getenv("ALLOW_OVERWRITE").toBoolean() ?: false
|
||||
|
||||
val logDirectory = if (!System.getenv("LOG_DIR").isNullOrBlank()) File(System.getenv("LOG_DIR")) else
|
||||
File("data").using("logs")
|
||||
|
||||
val encodeLogDirectory = logDirectory.using("encode")
|
||||
val extractLogDirectory = logDirectory.using("extract")
|
||||
|
||||
val fullLogging = System.getenv("FullLogging").toBoolean()
|
||||
}
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.ProcesserEventInfo
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.tryPost
|
||||
import no.iktdev.mediaprocessing.shared.common.trySend
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.stereotype.Service
|
||||
import org.springframework.web.client.RestTemplate
|
||||
|
||||
@Service
|
||||
class Reporter() {
|
||||
@Autowired
|
||||
lateinit var restTemplate: RestTemplate
|
||||
@Autowired
|
||||
lateinit var messageTemplate: SimpMessagingTemplate
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
|
||||
fun encodeTaskAssigned(task: Task) {
|
||||
messageTemplate.trySend("/topic/encode/assigned", task)
|
||||
}
|
||||
|
||||
fun extractTaskAssigned(task: Task) {
|
||||
messageTemplate.trySend("/topic/extract/assigned", task)
|
||||
}
|
||||
|
||||
fun sendEncodeProgress(progress: ProcesserEventInfo) {
|
||||
restTemplate.tryPost<String>(SharedConfig.uiUrl + "/encode/progress", progress)
|
||||
messageTemplate.trySend("/topic/encode/progress", progress)
|
||||
}
|
||||
|
||||
fun sendExtractProgress(progress: ProcesserEventInfo) {
|
||||
restTemplate.tryPost<String>(SharedConfig.uiUrl + "/extract/progress", progress)
|
||||
messageTemplate.trySend("/topic/extract/progress", progress)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,98 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.*
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.ActiveMode
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
class TaskCoordinator(): TaskCoordinatorBase() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
lateinit var runnerManager: RunnerManager
|
||||
|
||||
override fun onCoordinatorReady() {
|
||||
super.onCoordinatorReady()
|
||||
runnerManager = RunnerManager(dataSource = getEventsDatabase(), applicationName = ProcesserApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
}
|
||||
|
||||
|
||||
override val taskAvailabilityEventListener: MutableMap<TaskType, MutableList<TaskQueueListener>> = mutableMapOf(
|
||||
TaskType.Encode to mutableListOf(),
|
||||
TaskType.Extract to mutableListOf()
|
||||
)
|
||||
|
||||
private val taskListeners: MutableSet<TaskEvents> = mutableSetOf()
|
||||
fun getTaskListeners(): List<TaskEvents> {
|
||||
return taskListeners.toList()
|
||||
}
|
||||
fun addTaskEventListener(listener: TaskEvents) {
|
||||
taskListeners.add(listener)
|
||||
}
|
||||
|
||||
fun addEncodeTaskListener(listener: TaskQueueListener) {
|
||||
addTaskListener(TaskType.Encode, listener)
|
||||
}
|
||||
fun addExtractTaskListener(listener: TaskQueueListener) {
|
||||
addTaskListener(TaskType.Extract, listener)
|
||||
}
|
||||
|
||||
override fun addTaskListener(type: TaskType, listener: TaskQueueListener) {
|
||||
super.addTaskListener(type, listener)
|
||||
pullForAvailableTasks()
|
||||
}
|
||||
|
||||
|
||||
override fun pullForAvailableTasks() {
|
||||
if (runnerManager.iAmSuperseded()) {
|
||||
// This will let the application complete but not consume new
|
||||
val prevState = taskMode
|
||||
taskMode = ActiveMode.Passive
|
||||
if (taskMode != prevState && taskMode == ActiveMode.Passive) {
|
||||
log.warn { "A newer version has been detected. Changing mode to $taskMode, no new tasks will be processed" }
|
||||
}
|
||||
return
|
||||
}
|
||||
val available = taskManager.getClaimableTasks().asClaimable()
|
||||
available.forEach { (type, list) ->
|
||||
taskAvailabilityEventListener[type]?.forEach { listener ->
|
||||
list.foreachOrUntilClaimed {
|
||||
listener.onTaskAvailable(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onProduceEvent(event: Event) {
|
||||
taskManager.produceEvent(event)
|
||||
}
|
||||
|
||||
override fun getEnabledState(): Boolean {
|
||||
return runnerManager.amIEnabled()
|
||||
}
|
||||
|
||||
override fun clearExpiredClaims() {
|
||||
val expiredClaims = taskManager.getTasksWithExpiredClaim().filter { it.task in listOf(TaskType.Encode, TaskType.Extract) }
|
||||
expiredClaims.forEach {
|
||||
log.info { "Found event with expired claim: ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
}
|
||||
expiredClaims.forEach {
|
||||
val result = taskManager.deleteTaskClaim(referenceId = it.referenceId, eventId = it.eventId)
|
||||
if (result) {
|
||||
log.info { "Released claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
} else {
|
||||
log.error { "Failed to release claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface TaskEvents {
|
||||
fun onCancelOrStopProcess(eventId: String)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.processer.TaskCoordinator
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.bind.annotation.RequestBody
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
|
||||
@Controller
|
||||
class CancelController(@Autowired var task: TaskCoordinator) {
|
||||
|
||||
@RequestMapping(path = ["/cancel"])
|
||||
fun cancelProcess(@RequestBody eventId: String? = null): ResponseEntity<String> {
|
||||
if (eventId.isNullOrBlank()) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("No eventId provided!")
|
||||
}
|
||||
task.getTaskListeners().forEach { it.onCancelOrStopProcess(eventId) }
|
||||
return ResponseEntity.ok(null)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.ffmpeg
|
||||
|
||||
class FfmpegArgumentsBuilder() {
|
||||
private val defaultArguments = listOf(
|
||||
"-nostdin",
|
||||
"-nostats",
|
||||
"-hide_banner"
|
||||
)
|
||||
private var inputFile: String? = null
|
||||
private var outputFile: String? = null
|
||||
private var overwrite: Boolean = false
|
||||
private var progress: Boolean = false
|
||||
private var suppliedArgs: List<String> = emptyList()
|
||||
|
||||
fun inputFile(inputFile: String) = apply {
|
||||
this.inputFile = inputFile
|
||||
}
|
||||
|
||||
fun outputFile(outputFile: String) = apply {
|
||||
this.outputFile = outputFile
|
||||
}
|
||||
|
||||
fun allowOverwrite(allowOverwrite: Boolean) = apply {
|
||||
this.overwrite = allowOverwrite
|
||||
}
|
||||
|
||||
fun withProgress(withProgress: Boolean) = apply {
|
||||
this.progress = withProgress
|
||||
}
|
||||
|
||||
fun args(args: List<String>) = apply {
|
||||
this.suppliedArgs = args
|
||||
}
|
||||
|
||||
fun build(): List<String> {
|
||||
val args = mutableListOf<String>()
|
||||
val inFile = if (inputFile == null || inputFile?.isBlank() == true) {
|
||||
throw RuntimeException("Inputfile is required")
|
||||
} else this.inputFile!!
|
||||
val outFile: String = if (outputFile == null || outputFile?.isBlank() == true) {
|
||||
throw RuntimeException("Outputfile is required")
|
||||
} else this.outputFile!!
|
||||
|
||||
if (overwrite) {
|
||||
args.add("-y")
|
||||
}
|
||||
args.addAll(defaultArguments)
|
||||
args.addAll(listOf("-i", inFile))
|
||||
args.addAll(suppliedArgs)
|
||||
args.add(outFile)
|
||||
if (progress) {
|
||||
args.addAll(listOf("-progress", "pipe:1"))
|
||||
}
|
||||
|
||||
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.ffmpeg
|
||||
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.progress.FfmpegDecodedProgress
|
||||
|
||||
interface FfmpegListener {
|
||||
fun onStarted(inputFile: String)
|
||||
fun onCompleted(inputFile: String, outputFile: String)
|
||||
fun onProgressChanged(inputFile: String, progress: FfmpegDecodedProgress)
|
||||
fun onError(inputFile: String, message: String) {}
|
||||
}
|
||||
@ -1,135 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.ffmpeg
|
||||
|
||||
import com.fasterxml.jackson.core.io.UTF8Writer
|
||||
import com.github.pgreze.process.Redirect
|
||||
import com.github.pgreze.process.process
|
||||
import kotlinx.coroutines.*
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.exfl.using
|
||||
import no.iktdev.mediaprocessing.processer.ProcesserEnv
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.progress.FfmpegDecodedProgress
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.progress.FfmpegProgressDecoder
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.time.LocalDateTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.*
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
class FfmpegRunner(
|
||||
val inputFile: String,
|
||||
val outputFile: String,
|
||||
val arguments: List<String>,
|
||||
private val listener: FfmpegListener,
|
||||
val logDir: File
|
||||
) {
|
||||
val workOutputFile = File(outputFile).let {
|
||||
File(it.parentFile.absoluteFile, "${it.nameWithoutExtension}.work.${it.extension}")
|
||||
}.absolutePath
|
||||
|
||||
|
||||
val currentDateTime = LocalDateTime.now()
|
||||
val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd.HH.mm")
|
||||
val formattedDateTime = currentDateTime.format(formatter)
|
||||
|
||||
val logFile = logDir.using("$formattedDateTime-${File(inputFile).nameWithoutExtension}.log")
|
||||
|
||||
val scope = CoroutineScope(Dispatchers.Unconfined + Job())
|
||||
private var job: Job? = null
|
||||
|
||||
val decoder = FfmpegProgressDecoder()
|
||||
private val outputCache = mutableListOf<String>()
|
||||
|
||||
fun isWorking(): Boolean {
|
||||
return job != null && (job?.isCompleted != true) && scope.isActive
|
||||
}
|
||||
|
||||
fun run(progress: Boolean = false) {
|
||||
log.info { "Work file can be found at $workOutputFile" }
|
||||
val args = FfmpegArgumentsBuilder()
|
||||
.inputFile(inputFile)
|
||||
.outputFile(workOutputFile)
|
||||
.args(arguments)
|
||||
.allowOverwrite(ProcesserEnv.allowOverwrite)
|
||||
.withProgress(progress)
|
||||
.build()
|
||||
log.info { "Starting ffmpeg on file $inputFile with arguments:\n\t ${args.joinToString(" ")}" }
|
||||
|
||||
job = scope.launch {
|
||||
execute(args)
|
||||
}
|
||||
}
|
||||
|
||||
fun isAlive(): Boolean {
|
||||
return scope.isActive && job?.isCompleted != true
|
||||
}
|
||||
|
||||
private suspend fun execute(args: List<String>) {
|
||||
withContext(Dispatchers.IO) {
|
||||
logFile.createNewFile()
|
||||
}
|
||||
listener.onStarted(inputFile)
|
||||
val processOp = process(
|
||||
ProcesserEnv.ffmpeg, *args.toTypedArray(),
|
||||
stdout = Redirect.CAPTURE,
|
||||
stderr = Redirect.CAPTURE,
|
||||
consumer = {
|
||||
if (ProcesserEnv.fullLogging) {
|
||||
log.info { it }
|
||||
}
|
||||
onOutputChanged(it)
|
||||
},
|
||||
destroyForcibly = true
|
||||
)
|
||||
|
||||
val result = processOp
|
||||
onOutputChanged("Received exit code: ${result.resultCode}")
|
||||
if (result.resultCode != 0) {
|
||||
log.warn { "Work outputfile is orphaned and could be found using this path:\n$workOutputFile" }
|
||||
listener.onError(inputFile, result.output.joinToString("\n"))
|
||||
} else {
|
||||
log.info { "Converting work file to output file: $workOutputFile -> $outputFile" }
|
||||
val success = File(workOutputFile).renameTo(File(outputFile))
|
||||
if (!success) {
|
||||
val outMessage = "Could not convert file $workOutputFile -> $outputFile"
|
||||
log.error { outMessage }
|
||||
listener.onError(inputFile, outMessage)
|
||||
} else {
|
||||
listener.onCompleted(inputFile, outputFile)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
fun cancel(message: String = "Work was interrupted as requested") {
|
||||
job?.cancel()
|
||||
scope.cancel(message)
|
||||
listener.onError(inputFile, message)
|
||||
}
|
||||
|
||||
private var progress: FfmpegDecodedProgress? = null
|
||||
fun onOutputChanged(line: String) {
|
||||
outputCache.add(line)
|
||||
writeToLog(line)
|
||||
decoder.defineDuration(line)
|
||||
decoder.parseVideoProgress(outputCache.toList())?.let { decoded ->
|
||||
try {
|
||||
val _progress = decoder.getProgress(decoded)
|
||||
if (progress == null || _progress.progress > (progress?.progress ?: -1)) {
|
||||
progress = _progress
|
||||
listener.onProgressChanged(inputFile, _progress)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fun writeToLog(line: String) {
|
||||
FileOutputStream(logFile, true).bufferedWriter(Charsets.UTF_8).use {
|
||||
it.appendLine(line)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,73 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.ffmpeg
|
||||
|
||||
import kotlinx.coroutines.cancel
|
||||
import mu.KLogger
|
||||
import no.iktdev.exfl.using
|
||||
import no.iktdev.mediaprocessing.processer.taskManager
|
||||
import no.iktdev.mediaprocessing.shared.common.ClaimableTask
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.TaskQueueListener
|
||||
import no.iktdev.mediaprocessing.shared.common.getComputername
|
||||
import no.iktdev.mediaprocessing.shared.common.services.TaskService
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
import javax.annotation.PostConstruct
|
||||
|
||||
abstract class FfmpegTaskService: TaskService(), FfmpegListener {
|
||||
abstract override val logDir: File
|
||||
abstract override val log: KLogger
|
||||
|
||||
protected var runner: FfmpegRunner? = null
|
||||
|
||||
fun getTemporaryStoreFile(fileName: String): File {
|
||||
return SharedConfig.cachedContent.using(fileName)
|
||||
}
|
||||
|
||||
override fun onTaskAvailable(data: ClaimableTask) {
|
||||
if (runner?.isWorking() == true) {
|
||||
//log.info { "Worker is already running.., will not consume" }
|
||||
return
|
||||
}
|
||||
|
||||
if (assignedTask != null) {
|
||||
log.info { "Assigned task is not unassigned.., will not consume" }
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
val task = data.consume()
|
||||
if (task == null) {
|
||||
log.error { "Task is already consumed!" }
|
||||
return
|
||||
}
|
||||
val isAlreadyClaimed = taskManager.isTaskClaimed(referenceId = task.referenceId, eventId = task.eventId)
|
||||
if (isAlreadyClaimed) {
|
||||
log.warn { "Process is already claimed!" }
|
||||
return
|
||||
}
|
||||
task.let {
|
||||
this.assignedTask = it
|
||||
onTaskAssigned(it)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@PostConstruct
|
||||
private fun onCreated() {
|
||||
log.info { "Starting with id: $serviceId" }
|
||||
onAttachListener()
|
||||
}
|
||||
|
||||
fun clearWorker() {
|
||||
this.runner?.scope?.cancel()
|
||||
this.assignedTask = null
|
||||
this.runner = null
|
||||
}
|
||||
|
||||
fun cancelWorkIfRunning(eventId: String) {
|
||||
if (assignedTask?.eventId == eventId) {
|
||||
runner?.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.ffmpeg.progress
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.ProcesserProgress
|
||||
|
||||
data class FfmpegDecodedProgress(
|
||||
val progress: Int = -1,
|
||||
val time: String,
|
||||
val duration: String,
|
||||
val speed: String,
|
||||
val estimatedCompletionSeconds: Long = -1,
|
||||
val estimatedCompletion: String = "Unknown",
|
||||
) {
|
||||
fun toProcessProgress(): ProcesserProgress {
|
||||
return ProcesserProgress(
|
||||
progress = this.progress,
|
||||
speed = this.speed,
|
||||
timeWorkedOn = this.time,
|
||||
timeLeft = this.estimatedCompletion
|
||||
)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
data class ECT(val day: Int = 0, val hour: Int = 0, val minute: Int = 0, val second: Int = 0)
|
||||
@ -1,176 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.ffmpeg.progress
|
||||
|
||||
import mu.KotlinLogging
|
||||
import java.lang.StringBuilder
|
||||
import java.time.LocalTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.math.floor
|
||||
|
||||
class FfmpegProgressDecoder {
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
data class DecodedProgressData(
|
||||
val frame: Int?,
|
||||
val fps: Double?,
|
||||
val stream_0_0_q: Double?,
|
||||
val bitrate: String?,
|
||||
val total_size: Int?,
|
||||
val out_time_us: Long?,
|
||||
val out_time_ms: Long?,
|
||||
val out_time: String?,
|
||||
val dup_frames: Int?,
|
||||
val drop_frames: Int?,
|
||||
val speed: Double?,
|
||||
val progress: String?
|
||||
)
|
||||
|
||||
val expectedKeys = listOf<String>(
|
||||
"frame=",
|
||||
"fps=",
|
||||
"stream_0_0_q=",
|
||||
"bitrate=",
|
||||
"total_size=",
|
||||
"out_time_us=",
|
||||
"out_time_ms=",
|
||||
"out_time=",
|
||||
"dup_frames=",
|
||||
"drop_frames=",
|
||||
"speed=",
|
||||
"progress="
|
||||
)
|
||||
var duration: Int? = null
|
||||
set(value) {
|
||||
if (field == null || field == 0)
|
||||
field = value
|
||||
}
|
||||
var durationTime: String = "NA"
|
||||
fun parseVideoProgress(lines: List<String>): DecodedProgressData? {
|
||||
var frame: Int? = null
|
||||
var progress: String? = null
|
||||
val metadataMap = mutableMapOf<String, String>()
|
||||
|
||||
try {
|
||||
val eqValue = Regex("=")
|
||||
for (line in lines) {
|
||||
val keyValuePairs = Regex("=\\s*").replace(line, "=").split(" ").filter { it.isNotBlank() }.filter { eqValue.containsMatchIn(it) }
|
||||
for (keyValuePair in keyValuePairs) {
|
||||
val (key, value) = keyValuePair.split("=")
|
||||
metadataMap[key] = value
|
||||
}
|
||||
|
||||
if (frame == null) {
|
||||
frame = metadataMap["frame"]?.toIntOrNull()
|
||||
}
|
||||
|
||||
progress = metadataMap["progress"]
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return if (progress != null) {
|
||||
// When "progress" is found, build and return the VideoMetadata object
|
||||
DecodedProgressData(
|
||||
frame, metadataMap["fps"]?.toDoubleOrNull(), metadataMap["stream_0_0_q"]?.toDoubleOrNull(),
|
||||
metadataMap["bitrate"], metadataMap["total_size"]?.toIntOrNull(), metadataMap["out_time_us"]?.toLongOrNull(),
|
||||
metadataMap["out_time_ms"]?.toLongOrNull(), metadataMap["out_time"], metadataMap["dup_frames"]?.toIntOrNull(),
|
||||
metadataMap["drop_frames"]?.toIntOrNull(), metadataMap["speed"]?.replace("x", "", ignoreCase = true)?.toDoubleOrNull(), progress
|
||||
)
|
||||
} else {
|
||||
null // If "progress" is not found, return null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
val parsedDurations: MutableList<String> = mutableListOf()
|
||||
var hasReadContinue: Boolean = false
|
||||
fun defineDuration(value: String) {
|
||||
if (hasReadContinue) {
|
||||
return
|
||||
}
|
||||
if (value.contains(Regex("progress=continue"))) {
|
||||
hasReadContinue = true
|
||||
}
|
||||
|
||||
val results = Regex("Duration:\\s*([^,]+),").find(value)?.groupValues?.firstOrNull() ?: return
|
||||
log.info { "Identified duration for estimation $results" }
|
||||
|
||||
val parsedDuration = Regex("[0-9]+:[0-9]+:[0-9]+.[0-9]+").find(results.toString())?.value ?: return
|
||||
|
||||
if (!parsedDurations.contains(parsedDuration)) {
|
||||
parsedDurations.add(parsedDuration)
|
||||
}
|
||||
|
||||
val longestDuration = parsedDurations.mapNotNull {
|
||||
timeSpanToSeconds(it)
|
||||
}.maxOrNull() ?: return
|
||||
|
||||
duration = longestDuration
|
||||
}
|
||||
|
||||
private fun timeSpanToSeconds(time: String?): Int?
|
||||
{
|
||||
time ?: return null
|
||||
val timeString = Regex("[0-9]+:[0-9]+:[0-9]+.[0-9]+").find(time) ?: return null
|
||||
val strippedMS = Regex("[0-9]+:[0-9]+:[0-9]+").find(timeString.value) ?: return null
|
||||
val outTime = LocalTime.parse(strippedMS.value, DateTimeFormatter.ofPattern("HH:mm:ss"))
|
||||
return outTime.toSecondOfDay()
|
||||
}
|
||||
|
||||
|
||||
fun getProgress(decoded: DecodedProgressData): FfmpegDecodedProgress {
|
||||
if (duration == null)
|
||||
return FfmpegDecodedProgress(duration = durationTime, time = "NA", speed = "NA")
|
||||
val progressTime = timeSpanToSeconds(decoded.out_time) ?: 0
|
||||
val progress = floor((progressTime.toDouble() / duration!!.toDouble()) *100).toInt()
|
||||
|
||||
val ect = getEstimatedTimeRemaining(decoded)
|
||||
|
||||
return FfmpegDecodedProgress(
|
||||
progress = progress,
|
||||
estimatedCompletionSeconds = ect,
|
||||
estimatedCompletion = getETA(ect),
|
||||
duration = durationTime,
|
||||
time = decoded.out_time ?: "NA",
|
||||
speed = decoded.speed?.toString() ?: "NA"
|
||||
)
|
||||
}
|
||||
|
||||
fun getEstimatedTimeRemaining(decoded: DecodedProgressData): Long {
|
||||
val position = timeSpanToSeconds(decoded.out_time) ?: 0
|
||||
return if(duration == null || decoded.speed == null) -1 else
|
||||
Math.round(Math.round(duration!!.toDouble() - position.toDouble()) / decoded.speed)
|
||||
}
|
||||
|
||||
fun getECT(time: Long): ECT {
|
||||
var seconds = time
|
||||
val day = TimeUnit.SECONDS.toDays(seconds)
|
||||
seconds -= java.util.concurrent.TimeUnit.DAYS.toSeconds(day)
|
||||
|
||||
val hour = TimeUnit.SECONDS.toHours(seconds)
|
||||
seconds -= java.util.concurrent.TimeUnit.HOURS.toSeconds(hour)
|
||||
|
||||
val minute = TimeUnit.SECONDS.toMinutes(seconds)
|
||||
seconds -= java.util.concurrent.TimeUnit.MINUTES.toSeconds(minute)
|
||||
|
||||
return ECT(day.toInt(), hour.toInt(), minute.toInt(), seconds.toInt())
|
||||
}
|
||||
private fun getETA(time: Long): String {
|
||||
val etc = getECT(time) ?: return "Unknown"
|
||||
val str = StringBuilder()
|
||||
if (etc.day > 0) {
|
||||
str.append("${etc.day}d").append(" ")
|
||||
}
|
||||
if (etc.hour > 0) {
|
||||
str.append("${etc.hour}h").append(" ")
|
||||
}
|
||||
if (etc.day == 0 && etc.minute > 0) {
|
||||
str.append("${etc.minute}m").append(" ")
|
||||
}
|
||||
if (etc.hour == 0 && etc.second > 0) {
|
||||
str.append("${etc.second}s").append(" ")
|
||||
}
|
||||
return str.toString().trim()
|
||||
}
|
||||
}
|
||||
@ -1,229 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.services
|
||||
|
||||
import com.google.gson.Gson
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.mediaprocessing.processer.ProcesserEnv
|
||||
import no.iktdev.mediaprocessing.processer.Reporter
|
||||
import no.iktdev.mediaprocessing.processer.TaskCoordinator
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.FfmpegRunner
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.FfmpegTaskService
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.progress.FfmpegDecodedProgress
|
||||
import no.iktdev.mediaprocessing.processer.taskManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.Status
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.EncodeArgumentData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.EncodeWorkPerformedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.EncodedData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.ProcesserEventInfo
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.WorkStatus
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
import java.time.Duration
|
||||
|
||||
@Service
|
||||
class EncodeService(
|
||||
@Autowired var tasks: TaskCoordinator,
|
||||
@Autowired private val reporter: Reporter
|
||||
) : FfmpegTaskService(), TaskCoordinator.TaskEvents {
|
||||
|
||||
fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
override val log = KotlinLogging.logger {}
|
||||
override val logDir = ProcesserEnv.encodeLogDirectory
|
||||
|
||||
override fun getServiceId(serviceName: String): String {
|
||||
return super.getServiceId(this::class.java.simpleName)
|
||||
}
|
||||
|
||||
override fun onAttachListener() {
|
||||
tasks.addEncodeTaskListener(this)
|
||||
tasks.addTaskEventListener(this)
|
||||
}
|
||||
|
||||
override fun isReadyToConsume(): Boolean {
|
||||
return runner?.isWorking() == false
|
||||
}
|
||||
|
||||
override fun isTaskClaimable(task: Task): Boolean {
|
||||
return !taskManager.isTaskClaimed(referenceId = task.referenceId, eventId = task.eventId)
|
||||
}
|
||||
|
||||
override fun onTaskAssigned(task: Task) {
|
||||
reporter.encodeTaskAssigned(task)
|
||||
startEncode(task)
|
||||
}
|
||||
|
||||
|
||||
fun startEncode(event: Task) {
|
||||
val ffwrc = event.data as EncodeArgumentData
|
||||
val outFile = getTemporaryStoreFile(ffwrc.outputFileName)
|
||||
outFile.parentFile.mkdirs()
|
||||
if (!logDir.exists()) {
|
||||
logDir.mkdirs()
|
||||
}
|
||||
|
||||
val setClaim =
|
||||
taskManager.markTaskAsClaimed(referenceId = event.referenceId, eventId = event.eventId, claimer = serviceId)
|
||||
|
||||
if (setClaim) {
|
||||
log.info { "Claim successful for ${event.referenceId} encode" }
|
||||
runner = FfmpegRunner(
|
||||
inputFile = ffwrc.inputFile,
|
||||
outputFile = outFile.absolutePath,
|
||||
arguments = ffwrc.arguments,
|
||||
logDir = logDir, listener = this
|
||||
)
|
||||
if (outFile.exists()) {
|
||||
val reason = "${this::class.java.simpleName} identified the file as already existing, either allow overwrite or delete the offending file: ${outFile.absolutePath}"
|
||||
if (ffwrc.arguments.firstOrNull() != "-y") {
|
||||
this.onError(
|
||||
ffwrc.inputFile,
|
||||
reason
|
||||
)
|
||||
// Setting consumed to prevent spamming
|
||||
taskManager.markTaskAsCompleted(event.referenceId, event.eventId, Status.ERROR, reason)
|
||||
return
|
||||
}
|
||||
}
|
||||
runner?.run(true)
|
||||
|
||||
} else {
|
||||
log.error { "Failed to set claim on referenceId: ${event.referenceId} on event ${event.task}" }
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStarted(inputFile: String) {
|
||||
val task = assignedTask ?: return
|
||||
taskManager.markTaskAsClaimed(task.referenceId, task.eventId, serviceId)
|
||||
sendProgress(
|
||||
task.referenceId, task.eventId, status = WorkStatus.Started, FfmpegDecodedProgress(
|
||||
progress = 0,
|
||||
time = "Unkown",
|
||||
duration = "Unknown",
|
||||
speed = "0",
|
||||
)
|
||||
)
|
||||
log.info { "Encode started for ${task.referenceId}" }
|
||||
|
||||
runner?.scope?.launch {
|
||||
log.info { "Encode keep-alive started for ${task.referenceId}" }
|
||||
while (runner?.isAlive() == true) {
|
||||
delay(Duration.ofMinutes(5).toMillis())
|
||||
taskManager.refreshTaskClaim(task.referenceId, task.eventId, serviceId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCompleted(inputFile: String, outputFile: String) {
|
||||
val task = assignedTask ?: return
|
||||
log.info { "Encode completed for ${task.referenceId}" }
|
||||
val claimSuccessful = taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||
runBlocking {
|
||||
delay(1000)
|
||||
if (!claimSuccessful) {
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||
delay(1000)
|
||||
}
|
||||
var readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||
while (!readbackIsSuccess) {
|
||||
delay(1000)
|
||||
readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||
}
|
||||
|
||||
tasks.onProduceEvent(
|
||||
EncodeWorkPerformedEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = EncodedData(
|
||||
outputFile
|
||||
)
|
||||
)
|
||||
)
|
||||
sendProgress(
|
||||
task.referenceId, task.eventId, status = WorkStatus.Completed, FfmpegDecodedProgress(
|
||||
progress = 100,
|
||||
time = "",
|
||||
duration = "",
|
||||
speed = "0",
|
||||
)
|
||||
)
|
||||
clearWorker()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onError(inputFile: String, message: String) {
|
||||
val task = assignedTask ?: return
|
||||
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId, Status.ERROR, message)
|
||||
|
||||
log.error { "Encode failed for ${task.referenceId}\n$message" }
|
||||
tasks.onProduceEvent(EncodeWorkPerformedEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Failed,
|
||||
source = getProducerName()
|
||||
)
|
||||
))
|
||||
sendProgress(
|
||||
task.referenceId, task.eventId, status = WorkStatus.Failed, progress = FfmpegDecodedProgress(
|
||||
progress = 0,
|
||||
time = "",
|
||||
duration = "",
|
||||
speed = "0",
|
||||
)
|
||||
)
|
||||
clearWorker()
|
||||
}
|
||||
|
||||
|
||||
override fun onProgressChanged(inputFile: String, progress: FfmpegDecodedProgress) {
|
||||
val task = assignedTask ?: return
|
||||
sendProgress(task.referenceId, task.eventId, WorkStatus.Working, progress)
|
||||
}
|
||||
|
||||
|
||||
fun sendProgress(
|
||||
referenceId: String,
|
||||
eventId: String,
|
||||
status: WorkStatus,
|
||||
progress: FfmpegDecodedProgress? = null
|
||||
) {
|
||||
val runner = runner ?: return
|
||||
|
||||
val processerEventInfo = ProcesserEventInfo(
|
||||
referenceId = referenceId,
|
||||
eventId = eventId,
|
||||
status = status,
|
||||
inputFile = runner.inputFile,
|
||||
outputFiles = listOf(runner.outputFile),
|
||||
progress = progress?.toProcessProgress()
|
||||
)
|
||||
try {
|
||||
log.info { "Reporting encode progress ${Gson().toJson(processerEventInfo)}" }
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
try {
|
||||
reporter.sendEncodeProgress(processerEventInfo)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCancelOrStopProcess(eventId: String) {
|
||||
cancelWorkIfRunning(eventId)
|
||||
}
|
||||
}
|
||||
@ -1,204 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.processer.services
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.mediaprocessing.processer.ProcesserEnv
|
||||
import no.iktdev.mediaprocessing.processer.Reporter
|
||||
import no.iktdev.mediaprocessing.processer.TaskCoordinator
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.FfmpegRunner
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.FfmpegTaskService
|
||||
import no.iktdev.mediaprocessing.processer.ffmpeg.progress.FfmpegDecodedProgress
|
||||
import no.iktdev.mediaprocessing.processer.taskManager
|
||||
import no.iktdev.mediaprocessing.shared.common.limitedWhile
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.Status
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ExtractArgumentData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ExtractWorkCreatedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ExtractWorkPerformedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ExtractedData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.ProcesserEventInfo
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.WorkStatus
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
|
||||
@Service
|
||||
class ExtractService(
|
||||
@Autowired var tasks: TaskCoordinator,
|
||||
@Autowired private val reporter: Reporter
|
||||
) : FfmpegTaskService(), TaskCoordinator.TaskEvents {
|
||||
|
||||
fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
override val log = KotlinLogging.logger {}
|
||||
override val logDir = ProcesserEnv.encodeLogDirectory
|
||||
|
||||
override fun getServiceId(serviceName: String): String {
|
||||
return super.getServiceId(this::class.java.simpleName)
|
||||
}
|
||||
|
||||
override fun onAttachListener() {
|
||||
tasks.addExtractTaskListener(this)
|
||||
tasks.addTaskEventListener(this)
|
||||
}
|
||||
|
||||
override fun isReadyToConsume(): Boolean {
|
||||
return runner?.isWorking() == false
|
||||
}
|
||||
|
||||
override fun isTaskClaimable(task: Task): Boolean {
|
||||
return !taskManager.isTaskClaimed(referenceId = task.referenceId, eventId = task.eventId)
|
||||
}
|
||||
|
||||
override fun onTaskAssigned(task: Task) {
|
||||
reporter.extractTaskAssigned(task)
|
||||
startExtract(task)
|
||||
}
|
||||
|
||||
|
||||
fun startExtract(event: Task) {
|
||||
val ffwrc = event.data as ExtractArgumentData
|
||||
val outputFile = getTemporaryStoreFile(ffwrc.outputFileName)
|
||||
if (!logDir.exists()) {
|
||||
logDir.mkdirs()
|
||||
}
|
||||
|
||||
val setClaim =
|
||||
taskManager.markTaskAsClaimed(referenceId = event.referenceId, eventId = event.eventId, claimer = serviceId)
|
||||
if (setClaim) {
|
||||
log.info { "Claim successful for ${event.referenceId} extract" }
|
||||
runner = FfmpegRunner(
|
||||
inputFile = ffwrc.inputFile,
|
||||
outputFile = outputFile.absolutePath,
|
||||
arguments = ffwrc.arguments,
|
||||
logDir = logDir,
|
||||
listener = this
|
||||
)
|
||||
if (outputFile.exists()) {
|
||||
val reason =
|
||||
"${this::class.java.simpleName} identified the file as already existing, either allow overwrite or delete the offending file: ${outputFile.absolutePath}"
|
||||
if (ffwrc.arguments.firstOrNull() != "-y") {
|
||||
this.onError(
|
||||
ffwrc.inputFile,
|
||||
reason
|
||||
)
|
||||
// Setting consumed to prevent spamming
|
||||
taskManager.markTaskAsCompleted(event.referenceId, event.eventId, Status.ERROR, reason)
|
||||
return
|
||||
}
|
||||
}
|
||||
runner?.run()
|
||||
} else {
|
||||
log.error { "Failed to set claim on referenceId: ${event.referenceId} on event ${event.task}" }
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStarted(inputFile: String) {
|
||||
val task = assignedTask ?: return
|
||||
taskManager.markTaskAsClaimed(task.referenceId, task.eventId, serviceId)
|
||||
sendProgress(task.referenceId, task.eventId, WorkStatus.Started)
|
||||
}
|
||||
|
||||
override fun onCompleted(inputFile: String, outputFile: String) {
|
||||
val task = assignedTask ?: return
|
||||
assert(task.data is ExtractArgumentData) { "Wrong data type found!" }
|
||||
val taskData = task.data as ExtractArgumentData
|
||||
log.info { "Extract completed for ${task.referenceId}" }
|
||||
runBlocking {
|
||||
var successfulComplete = false
|
||||
limitedWhile({ !successfulComplete }, 1000 * 10, 1000) {
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||
successfulComplete = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||
}
|
||||
|
||||
tasks.onProduceEvent(
|
||||
ExtractWorkPerformedEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = ExtractedData(
|
||||
language = taskData.language,
|
||||
outputFile = outputFile,
|
||||
storeFileName = taskData.storeFileName
|
||||
)
|
||||
)
|
||||
)
|
||||
sendProgress(
|
||||
task.referenceId, task.eventId, status = WorkStatus.Completed, FfmpegDecodedProgress(
|
||||
progress = 100,
|
||||
time = "",
|
||||
duration = "",
|
||||
speed = "0",
|
||||
)
|
||||
)
|
||||
clearWorker()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onError(inputFile: String, message: String) {
|
||||
val task = assignedTask ?: return
|
||||
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId, Status.ERROR, message)
|
||||
|
||||
log.error { "Extract failed for ${task.referenceId}\n$message" }
|
||||
tasks.onProduceEvent(
|
||||
ExtractWorkPerformedEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Failed,
|
||||
source = getProducerName()
|
||||
)
|
||||
)
|
||||
)
|
||||
sendProgress(
|
||||
task.referenceId, task.eventId, status = WorkStatus.Failed, progress = FfmpegDecodedProgress(
|
||||
progress = 0,
|
||||
time = "",
|
||||
duration = "",
|
||||
speed = "0",
|
||||
)
|
||||
)
|
||||
clearWorker()
|
||||
}
|
||||
|
||||
override fun onProgressChanged(inputFile: String, progress: FfmpegDecodedProgress) {
|
||||
val task = assignedTask ?: return
|
||||
sendProgress(task.referenceId, task.eventId, WorkStatus.Working, progress)
|
||||
}
|
||||
|
||||
|
||||
fun sendProgress(
|
||||
referenceId: String,
|
||||
eventId: String,
|
||||
status: WorkStatus,
|
||||
progress: FfmpegDecodedProgress? = null
|
||||
) {
|
||||
val runner = runner ?: return
|
||||
|
||||
val processerEventInfo = ProcesserEventInfo(
|
||||
referenceId = referenceId,
|
||||
eventId = eventId,
|
||||
status = status,
|
||||
inputFile = runner.inputFile,
|
||||
outputFiles = listOf(runner.outputFile),
|
||||
progress = progress?.toProcessProgress()
|
||||
)
|
||||
try {
|
||||
reporter.sendExtractProgress(processerEventInfo)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCancelOrStopProcess(eventId: String) {
|
||||
cancelWorkIfRunning(eventId)
|
||||
}
|
||||
}
|
||||
@ -1,131 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.Defaults
|
||||
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.beans.factory.annotation.Value
|
||||
import org.springframework.boot.web.client.RestTemplateBuilder
|
||||
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory
|
||||
import org.springframework.boot.web.server.WebServerFactoryCustomizer
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.core.io.Resource
|
||||
import org.springframework.messaging.converter.MappingJackson2MessageConverter
|
||||
import org.springframework.messaging.converter.StringMessageConverter
|
||||
import org.springframework.stereotype.Component
|
||||
import org.springframework.stereotype.Service
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
import org.springframework.web.client.RestTemplate
|
||||
import org.springframework.web.method.HandlerTypePredicate
|
||||
import org.springframework.web.servlet.config.annotation.*
|
||||
import org.springframework.web.servlet.resource.PathResourceResolver
|
||||
import org.springframework.web.socket.CloseStatus
|
||||
import org.springframework.web.socket.TextMessage
|
||||
import org.springframework.web.socket.WebSocketSession
|
||||
import org.springframework.web.socket.handler.TextWebSocketHandler
|
||||
import org.springframework.web.util.DefaultUriBuilderFactory
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
|
||||
@Configuration
|
||||
class WebConfig: WebMvcConfigurer {
|
||||
override fun addCorsMappings(registry: CorsRegistry) {
|
||||
registry.addMapping("/**")
|
||||
.allowedOrigins("localhost", "*://localhost:3000", "localhost:80")
|
||||
.allowCredentials(true)
|
||||
}
|
||||
|
||||
override fun addResourceHandlers(registry: ResourceHandlerRegistry) {
|
||||
|
||||
registry.addResourceHandler("/**")
|
||||
.addResourceLocations("classpath:/static/")
|
||||
.resourceChain(true)
|
||||
.addResolver(object: PathResourceResolver() {
|
||||
override fun getResource(resourcePath: String, location: Resource): Resource? {
|
||||
// Show index.html if no resource was found
|
||||
return if (!location.createRelative(resourcePath).exists() && !location.createRelative(resourcePath).isReadable) {
|
||||
location.createRelative("index.html");
|
||||
} else {
|
||||
location.createRelative(resourcePath);
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
override fun configurePathMatch(configurer: PathMatchConfigurer) {
|
||||
configurer.addPathPrefix("/api", HandlerTypePredicate.forAnnotation(RestController::class.java))
|
||||
}
|
||||
|
||||
|
||||
@Value("\${APP_DEPLOYMENT_PORT:8080}")
|
||||
private val deploymentPort = 8080
|
||||
|
||||
@Bean
|
||||
fun webServerFactoryCustomizer(): WebServerFactoryCustomizer<TomcatServletWebServerFactory>? {
|
||||
return WebServerFactoryCustomizer { factory: TomcatServletWebServerFactory ->
|
||||
factory.port = deploymentPort
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Configuration
|
||||
class ApiCommunicationConfig {
|
||||
|
||||
@Bean
|
||||
fun coordinatorTemplate(builder: RestTemplateBuilder): RestTemplate {
|
||||
try {
|
||||
val url = UIEnv.coordinatorUrl
|
||||
log.info { "CoordinatorUrl: $url" }
|
||||
require(url.isNotBlank()) { "UIEnv.coordinatorUrl er ikke satt!" }
|
||||
return builder
|
||||
.uriTemplateHandler(DefaultUriBuilderFactory(url)) // Bruker den returnerte instansen
|
||||
.build()
|
||||
|
||||
} catch (e: Exception) {
|
||||
throw IllegalStateException("Feil ved opprettelse av coordinatorTemplate: ${e.message}", e)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Configuration
|
||||
class SocketImplemented: SocketImplementation() {
|
||||
override var additionalOrigins: List<String> = UIEnv.wsAllowedOrigins.split(",")
|
||||
}
|
||||
|
||||
@Service
|
||||
class WebSocketMonitoringService() {
|
||||
private val clients = ConcurrentHashMap.newKeySet<WebSocketSession>()
|
||||
fun anyListening() = clients.isNotEmpty()
|
||||
|
||||
fun addClient(session: WebSocketSession) {
|
||||
clients.add(session)
|
||||
}
|
||||
fun removeClient(session: WebSocketSession) {
|
||||
clients.remove(session)
|
||||
}
|
||||
}
|
||||
|
||||
@Component
|
||||
class WebSocketHandler(private val webSocketPollingService: WebSocketMonitoringService) : TextWebSocketHandler() {
|
||||
|
||||
// Kalles når en WebSocket-klient kobler til
|
||||
override fun afterConnectionEstablished(session: WebSocketSession) {
|
||||
webSocketPollingService.addClient(session) // Legg til klienten i service
|
||||
}
|
||||
|
||||
// Kalles når en WebSocket-klient kobler fra
|
||||
override fun afterConnectionClosed(session: WebSocketSession, status: CloseStatus) {
|
||||
webSocketPollingService.removeClient(session) // Fjern klienten fra service
|
||||
}
|
||||
|
||||
// Håndterer meldinger fra WebSocket-klientene hvis nødvendig
|
||||
override fun handleTextMessage(session: WebSocketSession, message: TextMessage) {
|
||||
// Håndter meldinger fra klienten
|
||||
}
|
||||
}
|
||||
|
||||
@Configuration
|
||||
class DefaultConfiguration: Defaults()
|
||||
@ -1,68 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui
|
||||
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.database.MySqlDataSource
|
||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||
import no.iktdev.exfl.observable.ObservableMap
|
||||
import no.iktdev.exfl.observable.Observables
|
||||
import no.iktdev.exfl.observable.observableMapOf
|
||||
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.database.EventsDatabase
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
||||
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||
import no.iktdev.mediaprocessing.ui.dto.explore.ExplorerItem
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||
import org.springframework.boot.runApplication
|
||||
import org.springframework.context.ApplicationContext
|
||||
import org.springframework.context.annotation.Bean
|
||||
|
||||
val log = KotlinLogging.logger {}
|
||||
lateinit var eventDatabase: EventsDatabase
|
||||
lateinit var eventsManager: EventsManager
|
||||
|
||||
|
||||
@SpringBootApplication
|
||||
class UIApplication {
|
||||
|
||||
@Bean
|
||||
fun eventManager(): EventsManager {
|
||||
return eventsManager
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
val ioCoroutine = CoroutinesIO()
|
||||
val defaultCoroutine = CoroutinesDefault()
|
||||
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
|
||||
|
||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
defaultCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
|
||||
eventDatabase = EventsDatabase().also {
|
||||
eventsManager = EventsManager(it.database)
|
||||
}
|
||||
|
||||
|
||||
runApplication<UIApplication>(*args)
|
||||
log.info { "App Version: ${getAppVersion()}" }
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui
|
||||
|
||||
import java.io.File
|
||||
|
||||
object UIEnv {
|
||||
val socketEncoder: String = System.getenv("EncoderWs")?.takeIf { it.isNotBlank() } ?: "ws://encoder:8080/ws"
|
||||
val coordinatorUrl: String = System.getenv("Coordinator")?.takeIf { it.isNotBlank() } ?: "http://coordinator"
|
||||
val wsAllowedOrigins: String = System.getenv("AllowedOriginsWebsocket")?.takeIf { it.isNotBlank() } ?: ""
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.dto
|
||||
|
||||
data class EventHolder(
|
||||
val referenceId: String,
|
||||
val fileName: String?,
|
||||
val events: List<EventChain>,
|
||||
val created: Long
|
||||
)
|
||||
|
||||
data class EventChain(
|
||||
val eventId: String,
|
||||
val eventName: String,
|
||||
val created: Long,
|
||||
val success: Boolean,
|
||||
val failure: Boolean,
|
||||
val skipped: Boolean,
|
||||
val events: MutableList<EventChain> = mutableListOf()
|
||||
)
|
||||
@ -1,87 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.dto
|
||||
|
||||
enum class SimpleEventDataState {
|
||||
NA,
|
||||
QUEUED,
|
||||
STARTED,
|
||||
ENDED,
|
||||
FAILED
|
||||
}
|
||||
|
||||
fun toSimpleEventDataStateFromStatus(state: String?): SimpleEventDataState {
|
||||
return when(state) {
|
||||
"QUEUED" -> SimpleEventDataState.QUEUED
|
||||
"STARTED" -> SimpleEventDataState.STARTED
|
||||
"UPDATED" -> SimpleEventDataState.STARTED
|
||||
"FAILURE" -> SimpleEventDataState.FAILED
|
||||
"ENDED" -> SimpleEventDataState.ENDED
|
||||
else -> SimpleEventDataState.NA
|
||||
}
|
||||
}
|
||||
|
||||
data class SimpleEventDataObject(
|
||||
val id: String,
|
||||
val name: String?,
|
||||
val path: String?,
|
||||
val givenTitle: String? = null,
|
||||
val givenSanitizedName: String? = null,
|
||||
val givenCollection: String? = null,
|
||||
val determinedType: String? = null,
|
||||
val eventEncoded: SimpleEventDataState = SimpleEventDataState.NA,
|
||||
val eventExtracted: SimpleEventDataState = SimpleEventDataState.NA,
|
||||
val eventConverted: SimpleEventDataState = SimpleEventDataState.NA,
|
||||
val eventCollected: SimpleEventDataState = SimpleEventDataState.NA,
|
||||
var encodingProgress: Int? = null,
|
||||
val encodingTimeLeft: Long? = null
|
||||
)
|
||||
|
||||
|
||||
data class EventDataObject(
|
||||
val id: String,
|
||||
var details: Details? = null,
|
||||
var metadata: Metadata? = null,
|
||||
var encode: Encode? = null,
|
||||
var io: IO? = null,
|
||||
var events: List<String> = emptyList()
|
||||
) {
|
||||
fun toSimple() = SimpleEventDataObject(
|
||||
id = id,
|
||||
name = details?.name,
|
||||
path = details?.file,
|
||||
givenTitle = details?.title,
|
||||
givenSanitizedName = details?.sanitizedName,
|
||||
givenCollection = details?.collection,
|
||||
determinedType = details?.type,
|
||||
eventEncoded = toSimpleEventDataStateFromStatus(encode?.state),
|
||||
encodingProgress = encode?.progress,
|
||||
encodingTimeLeft = encode?.timeLeft
|
||||
)
|
||||
}
|
||||
|
||||
data class Details(
|
||||
val name: String,
|
||||
val file: String,
|
||||
var title: String? = null,
|
||||
val sanitizedName: String,
|
||||
var collection: String? = null,
|
||||
var type: String? = null
|
||||
)
|
||||
|
||||
data class Metadata(
|
||||
val source: String
|
||||
)
|
||||
|
||||
interface ProcessableItem {
|
||||
var state: String
|
||||
}
|
||||
|
||||
data class IO(
|
||||
val inputFile: String,
|
||||
val outputFile: String
|
||||
)
|
||||
|
||||
data class Encode(
|
||||
override var state: String,
|
||||
var progress: Int = 0,
|
||||
var timeLeft: Long? = null
|
||||
) : ProcessableItem
|
||||
@ -1,32 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.dto
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
|
||||
data class EventSummary(
|
||||
val referenceId: String,
|
||||
val baseName: String? = null,
|
||||
val collection: String? = null,
|
||||
val events: List<Events> = emptyList(),
|
||||
val status: SummaryState = SummaryState.Started,
|
||||
val activeEvens: Map<String, EventSummarySubItem>
|
||||
)
|
||||
|
||||
data class EventSummarySubItem(
|
||||
val eventId: String,
|
||||
val status: SummaryState,
|
||||
val progress: Int = 0
|
||||
)
|
||||
|
||||
enum class SummaryState {
|
||||
Completed,
|
||||
AwaitingStore,
|
||||
Working,
|
||||
Pending,
|
||||
AwaitingConfirmation,
|
||||
Preparing,
|
||||
Metadata,
|
||||
Analyzing,
|
||||
Read,
|
||||
Started
|
||||
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.dto.explore
|
||||
|
||||
interface ExplorerAttr {
|
||||
val created: Long
|
||||
}
|
||||
|
||||
data class ExplorerAttributes(
|
||||
override val created: Long
|
||||
): ExplorerAttr
|
||||
@ -1,19 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.dto.explore
|
||||
|
||||
data class ExplorerCursor (
|
||||
val name: String,
|
||||
val path: String,
|
||||
val items: List<ExplorerItem>,
|
||||
)
|
||||
enum class ExplorerItemType {
|
||||
FILE,
|
||||
FOLDER
|
||||
}
|
||||
|
||||
data class ExplorerItem(
|
||||
val name: String,
|
||||
val path: String,
|
||||
val extension: String? = null,
|
||||
val created: Long,
|
||||
val type: ExplorerItemType
|
||||
)
|
||||
@ -1,93 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.explorer
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.SharedConfig
|
||||
import no.iktdev.mediaprocessing.ui.dto.explore.ExplorerAttributes
|
||||
import no.iktdev.mediaprocessing.ui.dto.explore.ExplorerCursor
|
||||
import no.iktdev.mediaprocessing.ui.dto.explore.ExplorerItem
|
||||
import no.iktdev.mediaprocessing.ui.dto.explore.ExplorerItemType
|
||||
import java.io.File
|
||||
import java.io.FileFilter
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.attribute.BasicFileAttributeView
|
||||
|
||||
class ExplorerCore {
|
||||
|
||||
fun getCursor(path: String): ExplorerCursor? {
|
||||
val file = File(path)
|
||||
if (!file.exists() || file.isFile) {
|
||||
return null
|
||||
}
|
||||
return ExplorerCursor(
|
||||
name = file.name,
|
||||
path = file.absolutePath,
|
||||
items = getFiles(file) + getFolders(file),
|
||||
)
|
||||
}
|
||||
|
||||
fun fromFile(file: File): ExplorerItem? {
|
||||
if (!file.exists())
|
||||
return null
|
||||
val attr = getAttr(file)
|
||||
return ExplorerItem(
|
||||
path = file.absolutePath,
|
||||
name = file.nameWithoutExtension,
|
||||
extension = file.extension,
|
||||
created = attr.created,
|
||||
type = ExplorerItemType.FILE
|
||||
)
|
||||
}
|
||||
|
||||
private fun getFiles(inDirectory: File): List<ExplorerItem> {
|
||||
return inDirectory.listFiles(FileFilter { it.isFile })?.map {
|
||||
val attr = getAttr(it)
|
||||
ExplorerItem(
|
||||
path = it.absolutePath,
|
||||
name = it.nameWithoutExtension,
|
||||
extension = it.extension,
|
||||
created = attr.created,
|
||||
type = ExplorerItemType.FILE
|
||||
)
|
||||
} ?: emptyList()
|
||||
}
|
||||
|
||||
private fun getFolders(inDirectory: File): List<ExplorerItem> {
|
||||
return inDirectory.listFiles(FileFilter { it.isDirectory })?.map {
|
||||
val attr = getAttr(it)
|
||||
ExplorerItem(
|
||||
name = it.name,
|
||||
path = it.absolutePath,
|
||||
created = attr.created,
|
||||
type = ExplorerItemType.FOLDER
|
||||
)
|
||||
} ?: emptyList()
|
||||
}
|
||||
|
||||
private fun getAttr(item: File): ExplorerAttributes {
|
||||
val attrView = Files.getFileAttributeView(item.toPath(), BasicFileAttributeView::class.java).readAttributes()
|
||||
return ExplorerAttributes(
|
||||
created = attrView.creationTime().toMillis()
|
||||
)
|
||||
}
|
||||
|
||||
fun getHomeCursor(): ExplorerCursor? {
|
||||
return getCursor(SharedConfig.inputRoot.absolutePath)
|
||||
}
|
||||
|
||||
fun getRoots(): ExplorerCursor {
|
||||
return ExplorerCursor(
|
||||
name = "root",
|
||||
path = "",
|
||||
items = File.listRoots().map {
|
||||
val attr = getAttr(it)
|
||||
ExplorerItem(
|
||||
path = it.absolutePath,
|
||||
name = it.absolutePath,
|
||||
extension = it.extension,
|
||||
created = attr.created,
|
||||
type = ExplorerItemType.FOLDER
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.service
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
class CompletedEventsService(
|
||||
@Autowired eventsManager: EventsManager
|
||||
) {
|
||||
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import no.iktdev.eventi.data.*
|
||||
import no.iktdev.eventi.database.toEpochSeconds
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.ui.dto.EventChain
|
||||
import no.iktdev.mediaprocessing.ui.dto.EventHolder
|
||||
import no.iktdev.mediaprocessing.ui.eventsManager
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.handler.annotation.MessageMapping
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.stereotype.Controller
|
||||
import java.io.File
|
||||
|
||||
@Controller
|
||||
class ChainedEventsTopic(
|
||||
@Autowired private val template: SimpMessagingTemplate?,
|
||||
) {
|
||||
@MessageMapping("/chained/all")
|
||||
fun sendAllChainedEvents() {
|
||||
val holders: MutableList<EventHolder> = mutableListOf()
|
||||
eventsManager.getAllEvents().onEach { events ->
|
||||
holders.add(EventHolder(
|
||||
referenceId = events.first().referenceId(),
|
||||
fileName = events.findFirstOf(Events.ProcessStarted)?.dataAs<StartEventData>()?.file?.let { File(it).name },
|
||||
events = events.chained(),
|
||||
created = events.chained().firstOrNull()?.created ?: 0
|
||||
))
|
||||
}
|
||||
template?.convertAndSend("/topic/chained/all", holders)
|
||||
}
|
||||
|
||||
|
||||
fun List<Event>.chained(): List<EventChain> {
|
||||
val eventMap = this.associateBy { it.eventId() }
|
||||
val chains = mutableMapOf<String, EventChain>()
|
||||
val children = mutableSetOf<String>()
|
||||
|
||||
this.forEach { event ->
|
||||
val eventId = event.metadata.eventId
|
||||
val derivedFromEventId = event.metadata.derivedFromEventId
|
||||
val created = event.metadata.created.toEpochSeconds() * 1000L
|
||||
val chain = chains.getOrPut(eventId) {
|
||||
EventChain(eventId, event.eventType.toString(), created, success = event.isSuccessful(), skipped = event.isSkipped(), failure = event.isFailed())
|
||||
}
|
||||
|
||||
if (derivedFromEventId != null && eventMap.containsKey(derivedFromEventId)) {
|
||||
val parentChain = chains.getOrPut(derivedFromEventId) {
|
||||
EventChain(derivedFromEventId, eventMap[derivedFromEventId]!!.eventType.toString(), created, success = event.isSuccessful(), skipped = event.isSkipped(), failure = event.isFailed())
|
||||
}
|
||||
parentChain.events.add(chain)
|
||||
children.add(eventId)
|
||||
}
|
||||
}
|
||||
|
||||
chains.values.forEach { chain -> chain.events.sortBy { it.created }}
|
||||
|
||||
return chains.values.filter { it.eventId !in children }
|
||||
.sortedBy { it.created }
|
||||
}
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import com.google.gson.Gson
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventRequest
|
||||
import no.iktdev.mediaprocessing.ui.UIEnv
|
||||
import no.iktdev.mediaprocessing.ui.explorer.ExplorerCore
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.handler.annotation.MessageMapping
|
||||
import org.springframework.messaging.handler.annotation.Payload
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.client.RestTemplate
|
||||
|
||||
val log = KotlinLogging.logger {}
|
||||
@Controller
|
||||
class ExplorerTopic(
|
||||
@Autowired private val template: SimpMessagingTemplate?,
|
||||
@Autowired private val coordinatorTemplate: RestTemplate,
|
||||
val explorer: ExplorerCore = ExplorerCore()
|
||||
) {
|
||||
|
||||
@MessageMapping("/explorer/home")
|
||||
fun goHome() {
|
||||
explorer.getHomeCursor()?.let {
|
||||
template?.convertAndSend("/topic/explorer/go", it)
|
||||
}
|
||||
}
|
||||
|
||||
@MessageMapping("/explorer/navigate")
|
||||
fun navigateTo(@Payload path: String) {
|
||||
val cursor = explorer.getCursor(path)
|
||||
cursor?.let {
|
||||
template?.convertAndSend("/topic/explorer/go", it)
|
||||
}
|
||||
}
|
||||
|
||||
@MessageMapping("/explorer/root")
|
||||
fun goRoot() {
|
||||
explorer.getRoots()?.let {
|
||||
template?.convertAndSend("/topic/explorer/go", it)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import com.google.gson.Gson
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventRequest
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.handler.annotation.MessageMapping
|
||||
import org.springframework.messaging.handler.annotation.Payload
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.client.RestTemplate
|
||||
|
||||
@Controller
|
||||
class FileRequestTopic(
|
||||
@Autowired private val template: SimpMessagingTemplate?,
|
||||
@Autowired private val coordinatorTemplate: RestTemplate,
|
||||
) {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
@MessageMapping("/request/encode")
|
||||
fun requestEncode(@Payload data: EventRequest) {
|
||||
val req = coordinatorTemplate.postForEntity("/request/encode", data, String::class.java)
|
||||
log.info { req }
|
||||
}
|
||||
|
||||
@MessageMapping("/request/extract")
|
||||
fun requestExtract(@Payload data: EventRequest) {
|
||||
val req = coordinatorTemplate.postForEntity("/request/extract", data, String::class.java)
|
||||
log.info { req }
|
||||
}
|
||||
|
||||
@MessageMapping("/request/convert")
|
||||
fun requestConvert(@Payload data: EventRequest) {
|
||||
val req = coordinatorTemplate.postForEntity("/request/convert", data, String::class.java)
|
||||
log.info { req }
|
||||
}
|
||||
|
||||
@MessageMapping("/request/all")
|
||||
fun requestAllAvailableActions(@Payload data: EventRequest) {
|
||||
log.info { "Sending data to coordinator: ${Gson().toJson(data)}" }
|
||||
val req = coordinatorTemplate.postForEntity("/request/all", data, String::class.java)
|
||||
log.info { req }
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,166 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import com.google.gson.Gson
|
||||
import no.iktdev.eventi.data.referenceId
|
||||
import no.iktdev.eventi.database.toEpochSeconds
|
||||
import no.iktdev.eventi.database.withDirtyRead
|
||||
import no.iktdev.eventi.database.withTransaction
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.ProcesserEventInfo
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.toEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.toTask
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.events
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.tasks
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.ui.WebSocketMonitoringService
|
||||
import no.iktdev.mediaprocessing.ui.eventDatabase
|
||||
import no.iktdev.mediaprocessing.ui.socket.a2a.ProcesserListenerService
|
||||
import org.jetbrains.exposed.sql.selectAll
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.handler.annotation.MessageMapping
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
|
||||
@Service
|
||||
class ProcesserTasksTopic(
|
||||
@Autowired a2AProcesserService: ProcesserListenerService,
|
||||
@Autowired private val webSocketMonitoringService: WebSocketMonitoringService,
|
||||
@Autowired private val message: SimpMessagingTemplate?,
|
||||
): SocketListener(message) {
|
||||
|
||||
private var referenceIds: List<String> = emptyList()
|
||||
|
||||
final val a2a = object : ProcesserListenerService.A2AProcesserListener {
|
||||
override fun onExtractProgress(info: ProcesserEventInfo) {
|
||||
if (referenceIds.none { it == info.referenceId }) {
|
||||
updateTopicWithTasks()
|
||||
}
|
||||
log.info { "Forwarding extract progress ${Gson().toJson(info)}" }
|
||||
message?.convertAndSend("/topic/processer/extract/progress", info)
|
||||
}
|
||||
|
||||
override fun onEncodeProgress(info: ProcesserEventInfo) {
|
||||
if (referenceIds.none { it == info.referenceId }) {
|
||||
updateTopicWithTasks()
|
||||
}
|
||||
log.info { "Forwarding encode progress ${Gson().toJson(info)}" }
|
||||
message?.convertAndSend("/topic/processer/encode/progress", info)
|
||||
}
|
||||
|
||||
override fun onEncodeAssigned(task: Task) {
|
||||
if (referenceIds.none { it == task.referenceId }) {
|
||||
updateTopicWithTasks()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onExtractAssigned(task: Task) {
|
||||
if (referenceIds.none { it == task.referenceId }) {
|
||||
updateTopicWithTasks()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
init {
|
||||
a2AProcesserService.attachListener(a2a)
|
||||
}
|
||||
|
||||
enum class Status {
|
||||
Skipped,
|
||||
Awaiting, // Waiting for tasks to be created
|
||||
NeedsApproval,
|
||||
Pending,
|
||||
InProgress,
|
||||
Completed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
data class ContentEventState(
|
||||
val referenceId: String,
|
||||
val title: String,
|
||||
val encode: Status = Status.Skipped,
|
||||
val extract: Status = Status.Skipped,
|
||||
val convert: Status = Status.Skipped,
|
||||
val created: Long
|
||||
) {}
|
||||
|
||||
|
||||
@MessageMapping("/tasks/all")
|
||||
fun updateTopicWithTasks() {
|
||||
val states = update()
|
||||
template?.convertAndSend("/topic/tasks/all", states)
|
||||
}
|
||||
|
||||
fun getOperationState(tasks: List<Task>, hasOperation: Boolean, canStart: Boolean): Status {
|
||||
if (!hasOperation) return Status.Skipped
|
||||
if (tasks.isEmpty()) return Status.Awaiting
|
||||
|
||||
if (!canStart) return Status.NeedsApproval
|
||||
|
||||
if (tasks.any { it.consumed }) {
|
||||
return Status.Completed
|
||||
}
|
||||
if (tasks.any { it.claimed }) {
|
||||
return Status.InProgress
|
||||
}
|
||||
if (tasks.any{ it.status == "ERROR"}) {
|
||||
return Status.Failed
|
||||
}
|
||||
return Status.Pending
|
||||
|
||||
}
|
||||
|
||||
|
||||
fun update(): MutableList<ContentEventState> {
|
||||
val eventStates: MutableList<ContentEventState> = mutableListOf()
|
||||
|
||||
val tasks = pullAllTasks()
|
||||
val availableEvents = pullAllEvents().also {
|
||||
referenceIds = it.keys.toList()
|
||||
}
|
||||
|
||||
for ((referenceId, events) in availableEvents) {
|
||||
val startEvent = events.findFirstEventOf<MediaProcessStartEvent>() ?: continue
|
||||
val startData = startEvent.data ?: continue
|
||||
val title = events.findFirstEventOf<BaseInfoEvent>()?.data?.sanitizedName ?: startData.file.let { File(it).nameWithoutExtension }
|
||||
val canStart = if (startData.type == ProcessType.FLOW) true else {
|
||||
events.findEventsOf<PermitWorkCreationEvent>().isNotEmpty()
|
||||
}
|
||||
val tasksCreated = tasks[referenceId]
|
||||
val encode = tasksCreated?.filter { it.task == TaskType.Encode } ?: emptyList()
|
||||
val extract = tasksCreated?.filter { it.task == TaskType.Extract } ?: emptyList()
|
||||
val convert = tasksCreated?.filter { it.task == TaskType.Convert } ?: emptyList()
|
||||
|
||||
eventStates.add(ContentEventState(
|
||||
title = title,
|
||||
referenceId = referenceId,
|
||||
encode = getOperationState(encode, startData.operations.contains(OperationEvents.ENCODE), canStart),
|
||||
extract = getOperationState(extract, startData.operations.contains(OperationEvents.EXTRACT), canStart),
|
||||
convert = getOperationState(convert, startData.operations.contains(OperationEvents.CONVERT), canStart),
|
||||
created = startEvent.metadata.created.toEpochSeconds() * 1000L
|
||||
))
|
||||
}
|
||||
return eventStates
|
||||
}
|
||||
|
||||
fun pullAllTasks(): Map<String, List<Task>> {
|
||||
val result = withTransaction(eventDatabase.database) {
|
||||
tasks.selectAll().toTask()
|
||||
.groupBy { it.referenceId }
|
||||
} ?: emptyMap()
|
||||
return result
|
||||
}
|
||||
|
||||
fun pullAllEvents(): Map<String, List<Event>> {
|
||||
val result = withDirtyRead(eventDatabase.database) {
|
||||
events.selectAll().toEvent()
|
||||
.groupBy { it.referenceId() }
|
||||
} ?: emptyMap()
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,108 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import mu.KotlinLogging
|
||||
import org.springframework.messaging.simp.stomp.StompCommand
|
||||
import org.springframework.messaging.simp.stomp.StompHeaders
|
||||
import org.springframework.messaging.simp.stomp.StompSession
|
||||
import org.springframework.messaging.simp.stomp.StompSessionHandlerAdapter
|
||||
import org.springframework.web.socket.client.standard.StandardWebSocketClient
|
||||
import org.springframework.web.socket.messaging.WebSocketStompClient
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.ScheduledFuture
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
class SocketClient(private val url: String, val listener: SocketEvents? = null) {
|
||||
private val logger = KotlinLogging.logger {}
|
||||
|
||||
private val client = WebSocketStompClient(StandardWebSocketClient())
|
||||
private val subscriptions: MutableList<SocketSubscription> = mutableListOf()
|
||||
private var session: StompSession? = null
|
||||
private val scheduler = Executors.newSingleThreadScheduledExecutor()
|
||||
private var reconnectFuture: ScheduledFuture<*>? = null
|
||||
|
||||
|
||||
|
||||
fun subscribe(topic: String, handler: StompSessionHandlerAdapter) {
|
||||
subscriptions.add(SocketSubscription(topic, handler))
|
||||
session?.subscribe(topic, handler)
|
||||
}
|
||||
|
||||
|
||||
private fun reconnect() {
|
||||
if (reconnectFuture != null && !reconnectFuture!!.isDone) return // Allerede reconnecting
|
||||
|
||||
reconnectFuture?.cancel(true) // Kansellerer tidligere reconnect-task hvis den kjører
|
||||
|
||||
logger.info { "Scheduling reconnect in 30 seconds" }
|
||||
|
||||
reconnectFuture = scheduler.scheduleWithFixedDelay({
|
||||
try {
|
||||
logger.info { "Attempting to reconnect... $url" }
|
||||
listener?.onReconnecting()
|
||||
connect()
|
||||
} catch (e: Exception) {
|
||||
logger.error(e) { "Reconnect attempt failed" }
|
||||
}
|
||||
}, 5, 30, TimeUnit.SECONDS) // Starter etter 5 sekunder, med 30 sekunders intervall
|
||||
}
|
||||
|
||||
private fun resetReconnector() {
|
||||
reconnectFuture?.cancel(true)
|
||||
reconnectFuture = null
|
||||
}
|
||||
|
||||
fun disconnect() {
|
||||
if (!scheduler.isShutdown) {
|
||||
try {
|
||||
reconnectFuture?.cancel(true)
|
||||
scheduler.shutdownNow()
|
||||
} catch (e: Exception) {}
|
||||
}
|
||||
session?.disconnect()
|
||||
listener?.onDisconnected()
|
||||
}
|
||||
|
||||
private val connectAdapter = object: StompSessionHandlerAdapter() {
|
||||
override fun afterConnected(session: StompSession, connectedHeaders: StompHeaders) {
|
||||
super.afterConnected(session, connectedHeaders)
|
||||
resetReconnector()
|
||||
listener?.onConnected()
|
||||
subscriptions.forEach {
|
||||
session.subscribe(it.destination, it.handler)
|
||||
}
|
||||
}
|
||||
|
||||
override fun handleTransportError(session: StompSession, exception: Throwable) {
|
||||
super.handleTransportError(session, exception)
|
||||
listener?.onException(exception)
|
||||
this@SocketClient.session = null
|
||||
reconnect()
|
||||
}
|
||||
|
||||
override fun handleException(
|
||||
session: StompSession,
|
||||
command: StompCommand?,
|
||||
headers: StompHeaders,
|
||||
payload: ByteArray,
|
||||
exception: Throwable
|
||||
) {
|
||||
super.handleException(session, command, headers, payload, exception)
|
||||
listener?.onException(exception)
|
||||
}
|
||||
}
|
||||
|
||||
fun connect() {
|
||||
client.connect(url, connectAdapter)
|
||||
}
|
||||
|
||||
interface SocketEvents {
|
||||
fun onConnected(): Unit {}
|
||||
fun onReconnecting(): Unit {}
|
||||
fun onDisconnected(): Unit {}
|
||||
fun onException(e: Throwable) {}
|
||||
}
|
||||
data class SocketSubscription(
|
||||
val destination: String,
|
||||
val handler: StompSessionHandlerAdapter
|
||||
)
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.stereotype.Controller
|
||||
|
||||
@Controller
|
||||
class SocketListener(
|
||||
@Autowired protected val template: SimpMessagingTemplate?,
|
||||
) {
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import org.springframework.messaging.simp.stomp.StompHeaders
|
||||
import org.springframework.messaging.simp.stomp.StompSessionHandlerAdapter
|
||||
import java.lang.reflect.Type
|
||||
|
||||
open class SocketMessageHandler: StompSessionHandlerAdapter() {
|
||||
override fun getPayloadType(headers: StompHeaders): Type {
|
||||
return ByteArray::class.java
|
||||
}
|
||||
|
||||
override fun handleFrame(headers: StompHeaders, payload: Any?) {
|
||||
super.handleFrame(headers, payload)
|
||||
if (payload is ByteArray) {
|
||||
onMessage(String(payload))
|
||||
} else if (payload is String) {
|
||||
onMessage(payload)
|
||||
}
|
||||
}
|
||||
|
||||
open fun onMessage(socketMessage: String) {
|
||||
}
|
||||
}
|
||||
@ -1,109 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket
|
||||
|
||||
import no.iktdev.eventi.database.withTransaction
|
||||
import no.iktdev.exfl.observable.ObservableList
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaProcessStartEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.az
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.jsonToEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.events
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.files
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.filesProcessed
|
||||
import no.iktdev.mediaprocessing.ui.WebSocketMonitoringService
|
||||
import no.iktdev.mediaprocessing.ui.eventDatabase
|
||||
import no.iktdev.mediaprocessing.ui.socket.UnprocessedFilesTopic.DatabaseData.filesInProcess
|
||||
import no.iktdev.mediaprocessing.ui.socket.UnprocessedFilesTopic.DatabaseData.pullUnprocessedFiles
|
||||
import no.iktdev.mediaprocessing.ui.socket.UnprocessedFilesTopic.DatabaseData.unprocessedFiles
|
||||
import no.iktdev.mediaprocessing.ui.socket.UnprocessedFilesTopic.DatabaseData.update
|
||||
import org.jetbrains.exposed.sql.select
|
||||
import org.jetbrains.exposed.sql.selectAll
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.handler.annotation.MessageMapping
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.client.RestTemplate
|
||||
import org.springframework.web.socket.WebSocketSession
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
@Controller
|
||||
@EnableScheduling
|
||||
class UnprocessedFilesTopic(
|
||||
@Autowired private val template: SimpMessagingTemplate?,
|
||||
@Autowired private val coordinatorTemplate: RestTemplate,
|
||||
@Autowired private val webSocketMonitoringService: WebSocketMonitoringService
|
||||
) {
|
||||
|
||||
|
||||
object DatabaseData {
|
||||
private const val PULL_MIN_INTERVAL = 10_000 // 10 sekunder
|
||||
private var lastPoll: Long = 0
|
||||
var unprocessedFiles: List<FileInfo> = emptyList()
|
||||
private set
|
||||
var filesInProcess: List<FileInfo> = emptyList()
|
||||
private set
|
||||
|
||||
// Funksjon som oppdaterer dataen hvis det har gått mer enn PULL_MIN_INTERVAL siden forrige oppdatering
|
||||
fun update() {
|
||||
val currentTime = System.currentTimeMillis()
|
||||
|
||||
// Sjekk om det har gått mer enn 10 sekunder (10000 ms) siden siste oppdatering
|
||||
if (currentTime - lastPoll >= PULL_MIN_INTERVAL) {
|
||||
// Oppdater tidspunktet for siste poll
|
||||
lastPoll = currentTime
|
||||
|
||||
// Oppdater dataene ved å hente nye verdier
|
||||
val filesNotCompleted = pullUnprocessedFiles()
|
||||
filesInProcess = pullUncompletedFiles()
|
||||
unprocessedFiles = filesNotCompleted.filter { u -> !filesInProcess.any { p -> p.checksum == u.checksum } }
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private fun pullUnprocessedFiles(): List<FileInfo> = withTransaction(eventDatabase.database) {
|
||||
val found = files.select {
|
||||
files.checksum notInSubQuery filesProcessed.slice(filesProcessed.checksum).selectAll()
|
||||
}.mapNotNull {
|
||||
FileInfo(
|
||||
it[files.baseName],
|
||||
it[files.fileName],
|
||||
it[files.checksum],
|
||||
)
|
||||
}
|
||||
unprocessedFiles = found
|
||||
found//.filter { File(it.fileName).exists() }
|
||||
} ?: emptyList()
|
||||
|
||||
private fun pullUncompletedFiles(): List<FileInfo> = withTransaction(eventDatabase.database) {
|
||||
val eventStartedFiles = events.select {
|
||||
events.event eq "ProcessStarted"
|
||||
}.mapNotNull { it[events.data].jsonToEvent(it[events.event]) }
|
||||
.mapNotNull { it.az<MediaProcessStartEvent>() }
|
||||
.mapNotNull { it.data?.file }
|
||||
unprocessedFiles.filter { it.fileName in eventStartedFiles }.also {
|
||||
filesInProcess = it
|
||||
}
|
||||
} ?: emptyList()
|
||||
}
|
||||
|
||||
data class UnprocessedFiles(
|
||||
val available: List<FileInfo>,
|
||||
val inProcess: List<FileInfo>
|
||||
)
|
||||
|
||||
@MessageMapping("/files/unprocessed")
|
||||
fun postUnProcessedFiles() {
|
||||
update()
|
||||
template?.convertAndSend("/topic/files/unprocessed", UnprocessedFiles(
|
||||
available = unprocessedFiles,
|
||||
inProcess = filesInProcess
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
data class FileInfo(
|
||||
val name: String,
|
||||
val fileName: String,
|
||||
val checksum: String
|
||||
)
|
||||
@ -1,110 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.ui.socket.a2a
|
||||
|
||||
import com.google.gson.Gson
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.ProcesserEventInfo
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.ui.UIEnv
|
||||
import no.iktdev.mediaprocessing.ui.WebSocketMonitoringService
|
||||
import no.iktdev.mediaprocessing.ui.log
|
||||
import no.iktdev.mediaprocessing.ui.socket.SocketClient
|
||||
import no.iktdev.mediaprocessing.ui.socket.SocketMessageHandler
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.messaging.simp.SimpMessagingTemplate
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
class ProcesserListenerService(
|
||||
@Autowired private val webSocketMonitoringService: WebSocketMonitoringService,
|
||||
) {
|
||||
private val logger = KotlinLogging.logger {}
|
||||
private val listeners: MutableList<A2AProcesserListener> = mutableListOf()
|
||||
|
||||
private var socketClient: SocketClient? = null
|
||||
|
||||
fun attachListener(listener: A2AProcesserListener) {
|
||||
listeners.add(listener)
|
||||
}
|
||||
|
||||
val gson = Gson()
|
||||
|
||||
private final val socketEvent = object : SocketClient.SocketEvents {
|
||||
override fun onConnected() {
|
||||
super.onConnected()
|
||||
log.info { "EncoderWsUrl: ${UIEnv.socketEncoder}" }
|
||||
logger.info { "Tilkoblet processer" }
|
||||
|
||||
socketClient?.subscribe("/topic/encode/progress", encodeProcessMessage)
|
||||
socketClient?.subscribe("/topic/extract/progress", extractProcessFrameHandler)
|
||||
socketClient?.subscribe("/topic/encode/assigned", encodeTaskAssignedMessage)
|
||||
socketClient?.subscribe("/topic/extract/assigned", extractTaskAssignedMessage)
|
||||
}
|
||||
}
|
||||
|
||||
init {
|
||||
SocketClient(UIEnv.socketEncoder, socketEvent).also {
|
||||
it.connect()
|
||||
this.socketClient = it
|
||||
}
|
||||
}
|
||||
|
||||
private val encodeTaskAssignedMessage = object: SocketMessageHandler() {
|
||||
override fun onMessage(socketMessage: String) {
|
||||
super.onMessage(socketMessage)
|
||||
val response = gson.fromJson(socketMessage, Task::class.java)
|
||||
listeners.forEach { listener ->
|
||||
run {
|
||||
listener.onEncodeAssigned(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val extractTaskAssignedMessage = object: SocketMessageHandler() {
|
||||
override fun onMessage(socketMessage: String) {
|
||||
super.onMessage(socketMessage)
|
||||
val response = gson.fromJson(socketMessage, Task::class.java)
|
||||
listeners.forEach { listener ->
|
||||
run {
|
||||
listener.onExtractAssigned(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private val encodeProcessMessage = object : SocketMessageHandler() {
|
||||
override fun onMessage(socketMessage: String) {
|
||||
super.onMessage(socketMessage)
|
||||
val response = gson.fromJson(socketMessage, ProcesserEventInfo::class.java)
|
||||
listeners.forEach { listener ->
|
||||
run {
|
||||
listener.onEncodeProgress(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private val extractProcessFrameHandler = object : SocketMessageHandler() {
|
||||
override fun onMessage(socketMessage: String) {
|
||||
super.onMessage(socketMessage)
|
||||
if (webSocketMonitoringService.anyListening()) {
|
||||
}
|
||||
val response = gson.fromJson(socketMessage, ProcesserEventInfo::class.java)
|
||||
listeners.forEach { listener ->
|
||||
run {
|
||||
listener.onEncodeProgress(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface A2AProcesserListener {
|
||||
fun onExtractProgress(info: ProcesserEventInfo)
|
||||
fun onEncodeProgress(info: ProcesserEventInfo)
|
||||
fun onEncodeAssigned(task: Task)
|
||||
fun onExtractAssigned(task: Task)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common
|
||||
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.web.client.RestTemplate
|
||||
|
||||
@Configuration
|
||||
open class Defaults {
|
||||
|
||||
@Bean
|
||||
open fun restTemplate(): RestTemplate {
|
||||
val restTemplate = RestTemplate()
|
||||
return restTemplate
|
||||
}
|
||||
}
|
||||
@ -1,112 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common
|
||||
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.exfl.using
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
|
||||
open class DownloadClient(val url: String, val outDir: File, val baseName: String) {
|
||||
val log = KotlinLogging.logger {}
|
||||
protected val http: HttpURLConnection = openConnection()
|
||||
private val BUFFER_SIZE = 4096
|
||||
|
||||
private fun openConnection(): HttpURLConnection {
|
||||
try {
|
||||
return URL(url).openConnection() as HttpURLConnection
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
throw BadAddressException("Provided url is either not provided (null) or is not a valid http url")
|
||||
}
|
||||
}
|
||||
|
||||
protected fun getLength(): Int {
|
||||
return http.contentLength
|
||||
}
|
||||
|
||||
protected fun getProgress(read: Int, total: Int = getLength()): Int {
|
||||
return ((read * 100) / total)
|
||||
}
|
||||
|
||||
suspend fun getOutFile(): File? = withContext(Dispatchers.IO) {
|
||||
val extension = getExtension()
|
||||
?: throw UnsupportedFormatException("Provided url does not contain a supported file extension")
|
||||
val outFile = outDir.using("$baseName.$extension")
|
||||
if (!outDir.exists()) {
|
||||
log.error { "Unable to create parent folder for ${outFile.name}. Download skipped!" }
|
||||
return@withContext null
|
||||
}
|
||||
return@withContext outFile
|
||||
}
|
||||
|
||||
suspend fun download(outFile: File): File? = withContext(Dispatchers.IO) {
|
||||
if (outFile.exists()) {
|
||||
log.info { "${outFile.name} already exists. Download skipped!" }
|
||||
return@withContext null
|
||||
}
|
||||
|
||||
val inputStream = http.inputStream
|
||||
val fos = FileOutputStream(outFile, false)
|
||||
|
||||
var totalBytesRead = 0
|
||||
val buffer = ByteArray(BUFFER_SIZE)
|
||||
inputStream.apply {
|
||||
fos.use { fout ->
|
||||
run {
|
||||
var bytesRead = read(buffer)
|
||||
while (bytesRead >= 0) {
|
||||
fout.write(buffer, 0, bytesRead)
|
||||
totalBytesRead += bytesRead
|
||||
bytesRead = read(buffer)
|
||||
// System.out.println(getProgress(totalBytesRead))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
inputStream.close()
|
||||
fos.close()
|
||||
return@withContext outFile
|
||||
}
|
||||
|
||||
open fun getExtension(): String? {
|
||||
val possiblyExtension = url.lastIndexOf(".") + 1
|
||||
return if (possiblyExtension > 1) {
|
||||
return url.toString().substring(possiblyExtension)
|
||||
} else {
|
||||
val mimeType = http.contentType ?: null
|
||||
contentTypeToExtension()[mimeType]
|
||||
}
|
||||
}
|
||||
|
||||
open fun contentTypeToExtension(): Map<String, String> {
|
||||
return mapOf(
|
||||
"image/png" to "png",
|
||||
"image/jpeg" to "jpg",
|
||||
"image/webp" to "webp",
|
||||
"image/bmp" to "bmp",
|
||||
"image/tiff" to "tiff"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class BadAddressException : java.lang.Exception {
|
||||
constructor() : super() {}
|
||||
constructor(message: String?) : super(message) {}
|
||||
constructor(message: String?, cause: Throwable?) : super(message, cause) {}
|
||||
}
|
||||
|
||||
class UnsupportedFormatException : Exception {
|
||||
constructor() : super() {}
|
||||
constructor(message: String?) : super(message) {}
|
||||
constructor(message: String?, cause: Throwable?) : super(message, cause) {}
|
||||
}
|
||||
|
||||
class InvalidFileException : Exception {
|
||||
constructor() : super() {}
|
||||
constructor(message: String?) : super(message) {}
|
||||
constructor(message: String?, cause: Throwable?) : super(message, cause) {}
|
||||
}
|
||||
}
|
||||
@ -1,54 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
data class LogKey(
|
||||
val referenceId: String,
|
||||
val listener: String,
|
||||
val eventType: String
|
||||
)
|
||||
|
||||
object EventDeadlockDetector {
|
||||
|
||||
private val stuckEvents = ConcurrentHashMap<LogKey, Int>()
|
||||
private val suppressedEvents = ConcurrentHashMap<LogKey, Boolean>()
|
||||
private val threshold = 5 // Antall ganger før det anses som deadlock
|
||||
private val resetInterval = 10L // Reset telling hvert 10. sekund
|
||||
|
||||
init {
|
||||
Executors.newScheduledThreadPool(1).scheduleAtFixedRate({
|
||||
stuckEvents.clear()
|
||||
}, resetInterval, resetInterval, TimeUnit.SECONDS)
|
||||
}
|
||||
|
||||
fun detect(referenceId: String, listener: String, eventType: String): Boolean {
|
||||
val key = LogKey(referenceId, listener, eventType)
|
||||
|
||||
if (suppressedEvents[key] == true) {
|
||||
return false
|
||||
}
|
||||
|
||||
val count = stuckEvents.merge(key, 1) { old, _ -> old + 1 } ?: 1
|
||||
|
||||
if (count > threshold) {
|
||||
suppressedEvents[key] = true
|
||||
onDeadlockDetected(key)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
fun resolve(referenceId: String, listener: String, eventType: String) {
|
||||
val key = LogKey(referenceId, listener, eventType)
|
||||
stuckEvents.remove(key)
|
||||
suppressedEvents.remove(key)
|
||||
}
|
||||
|
||||
private fun onDeadlockDetected(key: LogKey) {
|
||||
println("🚨 Deadlock detected! ReferenceId=${key.referenceId}, Listener=${key.listener}, EventType=${key.eventType}")
|
||||
// Her kan du f.eks. sende et varsel, restarte prosess, etc.
|
||||
}
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.GsonBuilder
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ffmpeg.PreferenceDto
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
class Preference {
|
||||
|
||||
companion object {
|
||||
private var prevPreference: PreferenceDto? = null
|
||||
fun getPreference(): PreferenceDto {
|
||||
val preference = readOrDefaultPreference()
|
||||
if (preference != prevPreference) {
|
||||
log.info { "[Audio]: Codec = " + preference.encodePreference.audio.codec }
|
||||
log.info { "[Audio]: Language = " + preference.encodePreference.audio.language }
|
||||
log.info { "[Audio]: Channels = " + preference.encodePreference.audio.channels }
|
||||
log.info { "[Audio]: Sample rate = " + preference.encodePreference.audio.sample_rate }
|
||||
log.info { "[Audio]: Use EAC3 for surround = " + preference.encodePreference.audio.convertToEac3OnUnsupportedSurround }
|
||||
|
||||
log.info { "[Video]: Codec = " + preference.encodePreference.video.codec }
|
||||
log.info { "[Video]: Pixel format = " + preference.encodePreference.video.pixelFormat }
|
||||
log.info { "[Video]: Pixel format pass-through = " + preference.encodePreference.video.pixelFormatPassthrough.joinToString(", ") }
|
||||
log.info { "[Video]: Threshold = " + preference.encodePreference.video.threshold }
|
||||
}
|
||||
return preference.also { prevPreference = it }
|
||||
}
|
||||
|
||||
private fun readOrDefaultPreference(): PreferenceDto {
|
||||
val preference = readPreferenceFromFile() ?: PreferenceDto()
|
||||
return preference
|
||||
}
|
||||
|
||||
|
||||
private fun readPreferenceFromFile(): PreferenceDto? {
|
||||
val prefFile = SharedConfig.preference
|
||||
if (!prefFile.exists()) {
|
||||
log.info("Preference file: ${prefFile.absolutePath} does not exists...")
|
||||
log.info("Using default configuration")
|
||||
silentTry {
|
||||
val gson = GsonBuilder().setPrettyPrinting().create()
|
||||
SharedConfig.preference.printWriter().use { out ->
|
||||
out.print(gson.toJson(PreferenceDto()))
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
else {
|
||||
log.info("Preference file: ${prefFile.absolutePath} found")
|
||||
}
|
||||
|
||||
return try {
|
||||
val instr = prefFile.inputStream()
|
||||
val text = instr.bufferedReader().use { it.readText() }
|
||||
Gson().fromJson(text, PreferenceDto::class.java)
|
||||
}
|
||||
catch (e: Exception) {
|
||||
log.error("Failed to read preference file: ${prefFile.absolutePath}.. Will use default configuration")
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common
|
||||
|
||||
import no.iktdev.eventi.database.DatabaseConnectionConfig
|
||||
import no.iktdev.eventi.database.MySqlDataSource
|
||||
import java.io.File
|
||||
|
||||
object SharedConfig {
|
||||
var inputRoot: File = if (!System.getenv("INPUT_ROOT").isNullOrBlank()) File(System.getenv("INPUT_ROOT")) else File("/src/input")
|
||||
var incomingContent: List<File> = if (!System.getenv("DIRECTORY_CONTENT_INCOMING").isNullOrBlank()) {
|
||||
System.getenv("DIRECTORY_CONTENT_INCOMING").split(",")
|
||||
.map { File(it) }
|
||||
} else listOf(File("/src/input"))
|
||||
var cachedContent: File = if (!System.getenv("DIRECTORY_CONTENT_CACHE").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_CACHE")) else File("/src/cache")
|
||||
val outgoingContent: File = if (!System.getenv("DIRECTORY_CONTENT_OUTGOING").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_OUTGOING")) else File("/src/output")
|
||||
|
||||
val ffprobe: String = System.getenv("SUPPORTING_EXECUTABLE_FFPROBE") ?: "ffprobe"
|
||||
val ffmpeg: String = System.getenv("SUPPORTING_EXECUTABLE_FFMPEG") ?: "ffmpeg"
|
||||
val uiUrl: String = System.getenv("APP_URL_UI") ?: "http://ui:8080"
|
||||
|
||||
val preference: File = File("/data/config/preference.json")
|
||||
val verbose: Boolean = System.getenv("VERBOSE")?.let { it.toBoolean() } ?: false
|
||||
}
|
||||
|
||||
object DatabaseEnvConfig {
|
||||
val address: String? = System.getenv("DATABASE_ADDRESS")
|
||||
val port: String? = System.getenv("DATABASE_PORT")
|
||||
val username: String? = System.getenv("DATABASE_USERNAME")
|
||||
val password: String? = System.getenv("DATABASE_PASSWORD")
|
||||
val eventBasedDatabase: String? = System.getenv("DATABASE_NAME_E")
|
||||
val storedDatabase: String? = System.getenv("DATABASE_NAME_S")
|
||||
}
|
||||
|
||||
fun DatabaseEnvConfig.toStoredDatabase(): MySqlDataSource {
|
||||
val config = DatabaseConnectionConfig(
|
||||
databaseName = this.storedDatabase ?: "streamit",
|
||||
address = this.address ?: "localhost",
|
||||
port = this.port,
|
||||
username = this.username ?: "root",
|
||||
password = this.password ?: ""
|
||||
)
|
||||
return MySqlDataSource(config)
|
||||
}
|
||||
|
||||
fun DatabaseEnvConfig.toEventsDatabase(): MySqlDataSource {
|
||||
val config = DatabaseConnectionConfig(
|
||||
databaseName = this.eventBasedDatabase ?: "persistentEvents",
|
||||
address = this.address ?: "localhost",
|
||||
port = this.port,
|
||||
username = this.username ?: "root",
|
||||
password = this.password ?: ""
|
||||
)
|
||||
return MySqlDataSource(config)
|
||||
}
|
||||
@ -1,105 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.ActiveMode
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
import javax.annotation.PostConstruct
|
||||
|
||||
@EnableScheduling
|
||||
abstract class TaskCoordinatorBase() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
var taskMode: ActiveMode = ActiveMode.Active
|
||||
var isEnabled: Boolean = true
|
||||
private var ready: Boolean = false
|
||||
fun isReady() = ready
|
||||
|
||||
abstract fun onProduceEvent(event: Event)
|
||||
|
||||
|
||||
abstract val taskAvailabilityEventListener: MutableMap<TaskType, MutableList<TaskQueueListener>>
|
||||
|
||||
open fun addTaskListener(type: TaskType, listener: TaskQueueListener) {
|
||||
val listeners = taskAvailabilityEventListener[type] ?: mutableListOf<TaskQueueListener>().also { it ->
|
||||
taskAvailabilityEventListener[type] = it
|
||||
}
|
||||
listeners.add(listener)
|
||||
}
|
||||
|
||||
|
||||
open fun onCoordinatorReady() {
|
||||
ready = true
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
fun onInitializationCompleted() {
|
||||
onCoordinatorReady()
|
||||
pullAvailable()
|
||||
}
|
||||
|
||||
abstract fun pullForAvailableTasks()
|
||||
|
||||
@Scheduled(fixedDelay = (5_000))
|
||||
fun pullAvailable() {
|
||||
if (taskMode != ActiveMode.Active || !isEnabled) {
|
||||
return
|
||||
}
|
||||
pullForAvailableTasks()
|
||||
}
|
||||
|
||||
abstract fun clearExpiredClaims()
|
||||
|
||||
abstract fun getEnabledState(): Boolean
|
||||
|
||||
@Scheduled(fixedDelay = 10_000)
|
||||
fun pullEnabledState() {
|
||||
val prevState = isEnabled
|
||||
isEnabled = getEnabledState()
|
||||
if (prevState != isEnabled) {
|
||||
log.info { "State changed for coordinator: $prevState -> $isEnabled" }
|
||||
if (isEnabled) {
|
||||
log.info { "New tasks will now be processed" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = (300_000))
|
||||
fun resetExpiredClaims() {
|
||||
if (taskMode != ActiveMode.Active) {
|
||||
return
|
||||
}
|
||||
clearExpiredClaims()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
class ClaimableTask(private var task: Task) {
|
||||
var isConsumed: Boolean = false
|
||||
private set
|
||||
fun consume(): Task? {
|
||||
return if (!isConsumed) {
|
||||
isConsumed = true
|
||||
task
|
||||
} else null
|
||||
}
|
||||
}
|
||||
|
||||
interface TaskQueueListener {
|
||||
fun onTaskAvailable(data: ClaimableTask)
|
||||
}
|
||||
|
||||
fun Map<TaskType, List<Task>>.asClaimable(): Map<TaskType, List<ClaimableTask>> {
|
||||
return this.mapValues { v -> v.value.map { ClaimableTask(it) } }
|
||||
}
|
||||
|
||||
fun List<ClaimableTask>.foreachOrUntilClaimed(block: (task: ClaimableTask) -> Unit) {
|
||||
this.forEach {
|
||||
if (!it.isConsumed) {
|
||||
block(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,146 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract
|
||||
|
||||
import com.google.gson.*
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.core.LocalDateTimeAdapter
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.*
|
||||
import java.lang.reflect.Type
|
||||
import java.time.LocalDateTime
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
|
||||
enum class Events {
|
||||
ProcessStarted,
|
||||
StreamRead,
|
||||
StreamParsed,
|
||||
BaseInfoRead,
|
||||
MetadataSearchPerformed,
|
||||
ReadOutNameAndType,
|
||||
ReadOutCover,
|
||||
EncodeParameterCreated,
|
||||
ExtractParameterCreated,
|
||||
WorkProceedPermitted,
|
||||
EncodeTaskCreated,
|
||||
ExtractTaskCreated,
|
||||
ConvertTaskCreated,
|
||||
EncodeTaskCompleted,
|
||||
ExtractTaskCompleted,
|
||||
ConvertTaskCompleted,
|
||||
CoverDownloaded,
|
||||
PersistContent,
|
||||
ProcessCompleted,
|
||||
Unknown
|
||||
;
|
||||
|
||||
companion object {
|
||||
fun toEvent(event: String): Events {
|
||||
return Events.entries.find { it.name == event } ?: Unknown
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun Events.toEventClass(): Class<out Event> {
|
||||
return when (this) {
|
||||
Events.ProcessStarted -> MediaProcessStartEvent::class.java
|
||||
|
||||
Events.StreamRead -> MediaFileStreamsReadEvent::class.java
|
||||
Events.StreamParsed -> MediaFileStreamsParsedEvent::class.java
|
||||
Events.BaseInfoRead -> BaseInfoEvent::class.java
|
||||
Events.MetadataSearchPerformed -> MediaMetadataReceivedEvent::class.java
|
||||
Events.ReadOutNameAndType -> MediaOutInformationConstructedEvent::class.java
|
||||
Events.ReadOutCover -> MediaCoverInfoReceivedEvent::class.java
|
||||
|
||||
Events.EncodeParameterCreated -> EncodeArgumentCreatedEvent::class.java
|
||||
Events.ExtractParameterCreated -> ExtractArgumentCreatedEvent::class.java
|
||||
|
||||
Events.WorkProceedPermitted -> PermitWorkCreationEvent::class.java
|
||||
|
||||
Events.EncodeTaskCreated -> EncodeWorkCreatedEvent::class.java
|
||||
Events.ExtractTaskCreated -> ExtractWorkCreatedEvent::class.java
|
||||
Events.ConvertTaskCreated -> ConvertWorkCreatedEvent::class.java
|
||||
|
||||
Events.EncodeTaskCompleted -> EncodeWorkPerformedEvent::class.java
|
||||
Events.ExtractTaskCompleted -> ExtractWorkPerformedEvent::class.java
|
||||
Events.ConvertTaskCompleted -> ConvertWorkPerformed::class.java
|
||||
Events.CoverDownloaded -> MediaCoverDownloadedEvent::class.java
|
||||
|
||||
Events.PersistContent -> PersistedContentEvent::class.java
|
||||
Events.ProcessCompleted -> MediaProcessCompletedEvent::class.java
|
||||
else -> UnknownEvent::class.java
|
||||
}
|
||||
}
|
||||
|
||||
fun String.jsonToEvent(eventType: String): Event {
|
||||
val event = Events.toEvent(eventType)
|
||||
return EventJson.fromJson(this, event)
|
||||
|
||||
/*val clazz = Events.toEvent(eventType)?.toEventClass()
|
||||
clazz.let { eventClass ->
|
||||
try {
|
||||
val type = TypeToken.getParameterized(eventClass).type
|
||||
return WGson.gson.fromJson(this, type)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
try {
|
||||
// Fallback
|
||||
val type = object : TypeToken<Event>() {}.type
|
||||
return WGson.gson.fromJson(this, type)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
// Default
|
||||
val type = object : TypeToken<Event>() {}.type
|
||||
log.error { "Failed to convert event: $eventType and data: $this to proper type!" }
|
||||
return WGson.gson.fromJson(this, type)*/
|
||||
}
|
||||
|
||||
object EventJson {
|
||||
private val gson = GsonBuilder()
|
||||
.registerTypeAdapter(LocalDateTime::class.java, LocalDateTimeAdapter())
|
||||
.create()
|
||||
|
||||
fun fromJson(json: String, event: Events): Event {
|
||||
val gson = GsonBuilder()
|
||||
.registerTypeAdapter(LocalDateTime::class.java, LocalDateTimeAdapter())
|
||||
.registerTypeAdapter(Event::class.java, EventDeserializer(event))
|
||||
.create()
|
||||
return gson.fromJson(json, Event::class.java)
|
||||
}
|
||||
|
||||
fun toJson(data: Any?): String {
|
||||
return gson.toJson(data)
|
||||
}
|
||||
|
||||
class EventDeserializer(private val eventType: Events) : JsonDeserializer<Event> {
|
||||
override fun deserialize(json: JsonElement, typeOfT: Type, context: JsonDeserializationContext): Event {
|
||||
// 🔥 Finn riktig klasse basert på eventType (som kommer eksternt fra databasen)
|
||||
val eventClass = eventType.toEventClass()
|
||||
|
||||
if (eventType == Events.Unknown || eventClass.simpleName == Event::class.java.simpleName) {
|
||||
val fallbackGson = GsonBuilder()
|
||||
.registerTypeAdapter(LocalDateTime::class.java, LocalDateTimeAdapter())
|
||||
.create()
|
||||
|
||||
return fallbackGson.fromJson(json, eventClass)
|
||||
}
|
||||
// Deserialiser objektet til riktig klasse
|
||||
val event = context.deserialize<Event>(json, eventClass)
|
||||
|
||||
// 🔥 Sett eventType eksplisitt etter deserialisering
|
||||
if (event is Event) {
|
||||
event::class.java.getDeclaredField("eventType").apply {
|
||||
isAccessible = true
|
||||
set(event, eventType)
|
||||
}
|
||||
}
|
||||
|
||||
return event
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract
|
||||
|
||||
import no.iktdev.eventi.implementations.EventCoordinator
|
||||
import no.iktdev.eventi.implementations.EventListenerImpl
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
|
||||
abstract class EventsListenerContract<E: EventsManagerContract, C: EventCoordinator<Event, E>>: EventListenerImpl<Event, E>() {
|
||||
abstract override val produceEvent: Events
|
||||
abstract override val listensForEvents: List<Events>
|
||||
abstract override val coordinator: C?
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract
|
||||
|
||||
import no.iktdev.eventi.implementations.EventsManagerImpl
|
||||
import no.iktdev.eventi.database.DataSource
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
|
||||
abstract class EventsManagerContract(dataSource: DataSource) : EventsManagerImpl<Event>(dataSource) {
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.data.EventImpl
|
||||
import no.iktdev.eventi.data.isSuccessful
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
|
||||
fun List<EventImpl>.lastOrSuccess(): EventImpl? {
|
||||
return this.lastOrNull { it.isSuccessful() } ?: this.lastOrNull()
|
||||
}
|
||||
|
||||
fun List<EventImpl>.lastOrSuccessOf(event: no.iktdev.mediaprocessing.shared.common.contract.Events): EventImpl? {
|
||||
val validEvents = this.filter { it.eventType == event }
|
||||
return validEvents.lastOrNull { it.isSuccessful() } ?: validEvents.lastOrNull()
|
||||
}
|
||||
|
||||
fun List<EventImpl>.lastOrSuccessOf(event: no.iktdev.mediaprocessing.shared.common.contract.Events, predicate: (EventImpl) -> Boolean): EventImpl? {
|
||||
val validEvents = this.filter { it.eventType == event && predicate(it) }
|
||||
return validEvents.lastOrNull()
|
||||
}
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract
|
||||
|
||||
enum class ProcessType {
|
||||
FLOW,
|
||||
MANUAL
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract.data
|
||||
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
|
||||
data class BaseInfoEvent(
|
||||
override val metadata: EventMetadata,
|
||||
override val eventType: Events = Events.BaseInfoRead,
|
||||
override val data: BaseInfo? = null
|
||||
) : Event()
|
||||
|
||||
data class BaseInfo(
|
||||
val title: String,
|
||||
val sanitizedName: String,
|
||||
val searchTitles: List<String> = emptyList<String>(),
|
||||
)
|
||||
@ -1,23 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract.data
|
||||
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.SubtitleFormats
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.tasks.TaskData
|
||||
|
||||
data class ConvertWorkCreatedEvent(
|
||||
override val metadata: EventMetadata,
|
||||
override val eventType: Events = Events.ConvertTaskCreated,
|
||||
override val data: ConvertData? = null
|
||||
) : Event() {
|
||||
}
|
||||
|
||||
data class ConvertData(
|
||||
override val inputFile: String,
|
||||
val language: String,
|
||||
val outputDirectory: String,
|
||||
val outputFileName: String,
|
||||
val storeFileName: String,
|
||||
val formats: List<SubtitleFormats> = emptyList(),
|
||||
val allowOverwrite: Boolean
|
||||
): TaskData()
|
||||
@ -1,18 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract.data
|
||||
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
|
||||
class ConvertWorkPerformed(
|
||||
override val metadata: EventMetadata,
|
||||
override val eventType: Events = Events.ConvertTaskCompleted,
|
||||
override val data: ConvertedData? = null,
|
||||
val message: String? = null
|
||||
) : Event() {
|
||||
}
|
||||
|
||||
data class ConvertedData(
|
||||
val language: String,
|
||||
val baseName: String,
|
||||
val outputFiles: List<String>
|
||||
)
|
||||
@ -1,19 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.shared.common.contract.data
|
||||
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.tasks.TaskData
|
||||
|
||||
data class EncodeArgumentCreatedEvent(
|
||||
override val metadata: EventMetadata,
|
||||
override val eventType: Events = Events.EncodeParameterCreated,
|
||||
override val data: EncodeArgumentData? = null
|
||||
) : Event() {
|
||||
|
||||
}
|
||||
|
||||
data class EncodeArgumentData(
|
||||
val arguments: List<String>,
|
||||
val outputFileName: String,
|
||||
override val inputFile: String
|
||||
): TaskData()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user