diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 614499a6..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,250 +0,0 @@ -name: Build Modules - -on: - push: - branches: - - master - pull_request: - branches: - - master - workflow_dispatch: - -jobs: - pre-check: - runs-on: ubuntu-latest - outputs: - pyMetadata: ${{ steps.filter.outputs.pyMetadata }} - commonCode: ${{ steps.filter.outputs.commonCode }} - reader: ${{ steps.filter.outputs.reader }} - encode: ${{ steps.filter.outputs.encode }} - convert: ${{ steps.filter.outputs.convert }} - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - pyMetadata: - - 'pyMetadata/**' - reader: - - 'Reader/**' - encode: - - 'Encode/**' - convert: - - 'Convert/**' - commonCode: - - 'CommonCode/**' - # Step to print the outputs from "pre-check" job - - name: Print Outputs from pre-check job - run: | - echo "pyMetadata: ${{ needs.pre-check.outputs.pyMetadata }}" - echo "commonCode: ${{ needs.pre-check.outputs.commonCode }}" - echo "reader: ${{ needs.pre-check.outputs.reader }}" - echo "encode: ${{ needs.pre-check.outputs.encode }}" - echo "convert: ${{ needs.pre-check.outputs.convert }}" - - build-commoncode: - runs-on: ubuntu-latest - needs: pre-check - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Cache CommonCode Gradle dependencies - id: cache-gradle - uses: actions/cache@v2 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('CommonCode/gradle/wrapper/gradle-wrapper.properties') }} - - - name: Build CommonCode - if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.commonCode == 'true' || github.event_name == 'workflow_dispatch' - run: | - cd CommonCode - chmod +x ./gradlew - ./gradlew build - - build-encode: - needs: build-commoncode - if: ${{ needs.pre-check.outputs.encode == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.commonCode == 'true' }} - runs-on: ubuntu-latest - #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Cache CommonCode Gradle dependencies - id: cache-gradle - uses: actions/cache@v2 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('CommonCode/gradle/wrapper/gradle-wrapper.properties') }} - - - - name: Build Encode module - id: build-encode - run: | - cd Encode - chmod +x ./gradlew - ./gradlew build - echo "Build completed" - - - - name: Generate Docker image tag - id: docker-tag - run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" - - - name: Docker login - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_HUB_NAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: ./Encode - push: true - tags: | - bskjon/mediaprocessing-encoder:latest - bskjon/mediaprocessing-encoder:${{ github.sha }} - bskjon/mediaprocessing-encoder:${{ steps.docker-tag.outputs.tag }} - - build-reader: - needs: build-commoncode - runs-on: ubuntu-latest - if: ${{ needs.pre-check.outputs.reader == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.commonCode == 'true' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Cache CommonCode Gradle dependencies - id: cache-gradle - uses: actions/cache@v2 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('CommonCode/gradle/wrapper/gradle-wrapper.properties') }} - - - name: Build Reader module - id: build-reader - run: | - cd Reader - chmod +x ./gradlew - ./gradlew build - echo "Build completed" - - - name: Generate Docker image tag - id: docker-tag - run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" - - - name: Docker login - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_HUB_NAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: ./Reader - push: true - tags: | - bskjon/mediaprocessing-reader:latest - bskjon/mediaprocessing-reader:${{ github.sha }} - bskjon/mediaprocessing-reader:${{ steps.docker-tag.outputs.tag }} - - - - build-pymetadata: - needs: pre-check - if: ${{ needs.pre-check.outputs.pyMetadata == 'true' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Build pyMetadata module - id: build-pymetadata - run: | - if [[ "${{ steps.check-pymetadata.outputs.changed }}" == "true" || "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then - cd pyMetadata - # Add the necessary build steps for your Python module here - echo "Build completed" - else - echo "pyMetadata has not changed. Skipping pyMetadata module build." - echo "::set-output name=job_skipped::true" - fi - - - name: Generate Docker image tag - id: docker-tag - run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" - - - name: Docker login - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_HUB_NAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: ./pyMetadata - push: true - tags: | - bskjon/mediaprocessing-pymetadata:latest - bskjon/mediaprocessing-pymetadata:${{ github.sha }} - bskjon/mediaprocessing-pymetadata:${{ steps.docker-tag.outputs.tag }} - - - build-convert: - needs: build-commoncode - if: ${{ needs.pre-check.outputs.convert == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.commonCode == 'true' }} - runs-on: ubuntu-latest - #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Cache CommonCode Gradle dependencies - id: cache-gradle - uses: actions/cache@v2 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('CommonCode/gradle/wrapper/gradle-wrapper.properties') }} - - - - name: Build Convert module - id: build-convert - run: | - cd Convert - chmod +x ./gradlew - ./gradlew build - echo "Build completed" - - - - name: Generate Docker image tag - id: docker-tag - run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" - - - name: Docker login - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_HUB_NAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: ./Convert - push: true - tags: | - bskjon/mediaprocessing-converter:latest - bskjon/mediaprocessing-converter:${{ github.sha }} - bskjon/mediaprocessing-converter:${{ steps.docker-tag.outputs.tag }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4ac4a0aa..b63da455 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,6 @@ build/ !**/src/test/**/build/ ### IntelliJ IDEA ### -.idea -**/.idea/* .idea/modules.xml .idea/jarRepositories.xml .idea/compiler.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..26d33521 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/gradle.xml b/.idea/gradle.xml new file mode 100644 index 00000000..9cd68be8 --- /dev/null +++ b/.idea/gradle.xml @@ -0,0 +1,27 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml new file mode 100644 index 00000000..e805548a --- /dev/null +++ b/.idea/kotlinc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..bf8f806e --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 00000000..2b63946d --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/CommonCode/.gitignore b/CommonCode/.gitignore deleted file mode 100644 index b63da455..00000000 --- a/CommonCode/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -.gradle -build/ -!gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/CommonCode/gradle/wrapper/gradle-wrapper.properties b/CommonCode/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index dfb7e10f..00000000 --- a/CommonCode/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,6 +0,0 @@ -#Sat Jul 15 17:55:49 CEST 2023 -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/CommonCode/settings.gradle.kts b/CommonCode/settings.gradle.kts deleted file mode 100644 index 47b4e959..00000000 --- a/CommonCode/settings.gradle.kts +++ /dev/null @@ -1,2 +0,0 @@ -rootProject.name = "CommonCode" - diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/CommonConfig.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/CommonConfig.kt deleted file mode 100644 index b533ee99..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/CommonConfig.kt +++ /dev/null @@ -1,9 +0,0 @@ -package no.iktdev.streamit.content.common - -import java.io.File - -object CommonConfig { - var kafkaTopic: String = System.getenv("KAFKA_TOPIC") ?: "contentEvents" - var incomingContent: File = if (!System.getenv("DIRECTORY_CONTENT_INCOMING").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_INCOMING")) else File("/src/input") - val outgoingContent: File = if (!System.getenv("DIRECTORY_CONTENT_OUTGOING").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_OUTGOING")) else File("/src/output") -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/DefaultKafkaReader.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/DefaultKafkaReader.kt deleted file mode 100644 index 8c723f9d..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/DefaultKafkaReader.kt +++ /dev/null @@ -1,53 +0,0 @@ -package no.iktdev.streamit.content.common - -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.consumers.DefaultConsumer -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import no.iktdev.streamit.library.kafka.producer.DefaultProducer -import java.util.* - -abstract class DefaultKafkaReader(val subId: String = UUID.randomUUID().toString()) { - val messageProducer = DefaultProducer(CommonConfig.kafkaTopic) - val defaultConsumer = DefaultConsumer(subId = subId) - - open fun loadDeserializers(): Map> { - return emptyMap() - } - - fun produceErrorMessage(event: KafkaEvents, baseMessage: Message, reason: String) { - val message = Message( - referenceId = baseMessage.referenceId, - Status(statusType = StatusType.ERROR, message = reason) - ) - messageProducer.sendMessage(event.event, message) - } - - fun produceErrorMessage(event: KafkaEvents, referenceId: String, reason: String) { - val message = Message( - referenceId = referenceId, - Status(statusType = StatusType.ERROR, message = reason) - ) - messageProducer.sendMessage(event.event, message) - } - - fun produceMessage(event: KafkaEvents, baseMessage: Message, data: Any?) { - val message = Message( - referenceId = baseMessage.referenceId, - baseMessage.status, - data = data - ) - messageProducer.sendMessage(event.event, message) - } - fun produceSuccessMessage(event: KafkaEvents, referenceId: String, data: Any? = null) { - val message = Message( - referenceId = referenceId, - status = Status(StatusType.SUCCESS), - data = data - ) - messageProducer.sendMessage(event.event, message) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/FileAccess.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/FileAccess.kt deleted file mode 100644 index 0ed74b82..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/FileAccess.kt +++ /dev/null @@ -1,25 +0,0 @@ -package no.iktdev.streamit.content.common - -import mu.KotlinLogging -import java.io.File -import java.io.RandomAccessFile - -private val logger = KotlinLogging.logger {} -class FileAccess { - companion object { - fun isFileAvailable(file: File): Boolean { - if (!file.exists()) return false - var stream: RandomAccessFile? = null - try { - stream = RandomAccessFile(file, "rw") - stream.close() - logger.info { "File ${file.name} is read and writable" } - return true - } catch (e: Exception) { - stream?.close() - } - return false - } - } - -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/SequentialKafkaReader.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/SequentialKafkaReader.kt deleted file mode 100644 index 53eb2f4b..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/SequentialKafkaReader.kt +++ /dev/null @@ -1,20 +0,0 @@ -package no.iktdev.streamit.content.common - -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.consumers.DefaultConsumer -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import no.iktdev.streamit.library.kafka.listener.sequential.ISequentialMessageEvent -import no.iktdev.streamit.library.kafka.listener.sequential.SequentialMessageListener -import no.iktdev.streamit.library.kafka.producer.DefaultProducer - -abstract class SequentialKafkaReader(subId: String): DefaultKafkaReader(subId), ISequentialMessageEvent { - - abstract val accept: KafkaEvents - abstract val subAccepts: List - - -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ContentOutNameDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ContentOutNameDeserializer.kt deleted file mode 100644 index 322a2909..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ContentOutNameDeserializer.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.ContentOutName -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class ContentOutNameDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): ContentOutName? { - return incomingMessage.dataAs(ContentOutName::class.java) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ConvertWorkDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ConvertWorkDeserializer.kt deleted file mode 100644 index 183f0447..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ConvertWorkDeserializer.kt +++ /dev/null @@ -1,13 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.reader.work.ConvertWork -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class ConvertWorkDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): ConvertWork? { - return incomingMessage.dataAs(ConvertWork::class.java) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/DeserializerRegistry.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/DeserializerRegistry.kt deleted file mode 100644 index 71082dc5..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/DeserializerRegistry.kt +++ /dev/null @@ -1,52 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class DeserializerRegistry { - companion object { - private val _registry = mutableMapOf>( - KafkaEvents.EVENT_READER_RECEIVED_FILE to FileResultDeserializer(), - KafkaEvents.EVENT_READER_RECEIVED_STREAMS to MediaStreamsDeserializer(), - KafkaEvents.EVENT_METADATA_OBTAINED to MetadataResultDeserializer(), - KafkaEvents.EVENT_READER_DETERMINED_SERIE to EpisodeInfoDeserializer(), - KafkaEvents.EVENT_READER_DETERMINED_MOVIE to MovieInfoDeserializer(), - KafkaEvents.EVENT_READER_DETERMINED_FILENAME to ContentOutNameDeserializer(), - - KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO to EncodeWorkDeserializer(), - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_QUEUED to EncodeWorkDeserializer(), - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_STARTED to EncodeWorkDeserializer(), - - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_ENDED to EncodeWorkDeserializer(), - KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE to ExtractWorkDeserializer(), - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_ENDED to ExtractWorkDeserializer(), - KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_ENDED to ConvertWorkDeserializer() - - ) - fun getRegistry(): Map> = _registry.toMap() - fun getEventToDeserializer(vararg keys: KafkaEvents): Map> { - val missingFields = keys.filter { !getRegistry().keys.contains(it) } - - if (missingFields.isNotEmpty()) { - throw MissingDeserializerException("Missing deserializers for: ${missingFields.joinToString(", ")}") - } - return getRegistry().filter { keys.contains(it.key) }.map { it.key.event to it.value }.toMap() - } - - private fun toEvent(event: String): KafkaEvents? { - return KafkaEvents.values().find { it.event == event } - } - - fun getDeserializerForEvent(event: String): IMessageDataDeserialization<*>? { - val deszEvent = toEvent(event) ?: return null - return getEventToDeserializer(deszEvent).values.first() - } - - fun addDeserializer(key: KafkaEvents, deserializer: IMessageDataDeserialization<*>) { - _registry[key] = deserializer - } - - } -} - -class MissingDeserializerException(override val message: String): RuntimeException() \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/EncodeWorkDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/EncodeWorkDeserializer.kt deleted file mode 100644 index 847fa1f5..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/EncodeWorkDeserializer.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class EncodeWorkDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): EncodeWork? { - return incomingMessage.dataAs(EncodeWork::class.java) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/EpisodeInfoDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/EpisodeInfoDeserializer.kt deleted file mode 100644 index 8c6cddf8..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/EpisodeInfoDeserializer.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.reader.EpisodeInfo -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class EpisodeInfoDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): EpisodeInfo? { - return incomingMessage.dataAs(EpisodeInfo::class.java) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ExtractWorkDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ExtractWorkDeserializer.kt deleted file mode 100644 index 155fb56b..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/ExtractWorkDeserializer.kt +++ /dev/null @@ -1,12 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class ExtractWorkDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): ExtractWork? { - return incomingMessage.dataAs(ExtractWork::class.java) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/FileResultDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/FileResultDeserializer.kt deleted file mode 100644 index 97809fe4..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/FileResultDeserializer.kt +++ /dev/null @@ -1,13 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class FileResultDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): FileResult? { - return incomingMessage.dataAs(FileResult::class.java) - } -} diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MediaStreamsDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MediaStreamsDeserializer.kt deleted file mode 100644 index f2248f32..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MediaStreamsDeserializer.kt +++ /dev/null @@ -1,47 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import com.google.gson.Gson -import com.google.gson.JsonObject -import no.iktdev.streamit.content.common.streams.AudioStream -import no.iktdev.streamit.content.common.streams.MediaStreams -import no.iktdev.streamit.content.common.streams.SubtitleStream -import no.iktdev.streamit.content.common.streams.VideoStream -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class MediaStreamsDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): MediaStreams? { - return try { - val gson = Gson() - val jsonObject = if (incomingMessage.data is String) { - gson.fromJson(incomingMessage.data as String, JsonObject::class.java) - } else { - gson.fromJson(incomingMessage.dataAsJson(), JsonObject::class.java) - } - - val streamsJsonArray = jsonObject.getAsJsonArray("streams") - - val rstreams = streamsJsonArray.mapNotNull { streamJson -> - val streamObject = streamJson.asJsonObject - - val codecType = streamObject.get("codec_type").asString - if (streamObject.has("codec_name") && streamObject.get("codec_name").asString == "mjpeg") { - null - } else { - when (codecType) { - "video" -> gson.fromJson(streamObject, VideoStream::class.java) - "audio" -> gson.fromJson(streamObject, AudioStream::class.java) - "subtitle" -> gson.fromJson(streamObject, SubtitleStream::class.java) - else -> null //throw IllegalArgumentException("Unknown stream type: $codecType") - } - } - } - - return MediaStreams(rstreams) - } catch (e: Exception) { - e.printStackTrace() - null - } - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MetadataResultDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MetadataResultDeserializer.kt deleted file mode 100644 index d0ee0b5e..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MetadataResultDeserializer.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.Metadata -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class MetadataResultDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): Metadata? { - return incomingMessage.dataAs(Metadata::class.java) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MovieInfoDeserializer.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MovieInfoDeserializer.kt deleted file mode 100644 index fcc7b2b0..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deserializers/MovieInfoDeserializer.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.common.deserializers - -import no.iktdev.streamit.content.common.dto.reader.MovieInfo -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization - -class MovieInfoDeserializer: IMessageDataDeserialization { - override fun deserialize(incomingMessage: Message): MovieInfo? { - return incomingMessage.dataAs(MovieInfo::class.java) - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/ContentOutName.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/ContentOutName.kt deleted file mode 100644 index 2c3b76ea..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/ContentOutName.kt +++ /dev/null @@ -1,5 +0,0 @@ -package no.iktdev.streamit.content.common.dto - -data class ContentOutName( - val baseName: String -) diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/Metadata.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/Metadata.kt deleted file mode 100644 index 7eaba43a..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/Metadata.kt +++ /dev/null @@ -1,10 +0,0 @@ -package no.iktdev.streamit.content.common.dto - -data class Metadata( - val title: String, - val altTitle: List = emptyList(), - val cover: String? = null, - val type: String, - val summary: String? = null, - val genres: List = emptyList() -) diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/WorkOrderItem.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/WorkOrderItem.kt deleted file mode 100644 index 2bcccb1b..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/WorkOrderItem.kt +++ /dev/null @@ -1,19 +0,0 @@ -package no.iktdev.streamit.content.common.dto - -data class WorkOrderItem( - val id: String, - val inputFile: String, - val outputFile: String, - val collection: String, - val state: State, - val progress: Int = 0, - val remainingTime: Long? = null -) - -enum class State { - QUEUED, - STARTED, - UPDATED, - FAILURE, - ENDED -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/EpisodeInfo.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/EpisodeInfo.kt deleted file mode 100644 index ce8fc0f1..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/EpisodeInfo.kt +++ /dev/null @@ -1,9 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader - -data class EpisodeInfo( - val title: String, - val episode: Int, - val season: Int, - val episodeTitle: String?, - override val fullName: String -): VideoInfo(fullName) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/FileResult.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/FileResult.kt deleted file mode 100644 index 30d4685a..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/FileResult.kt +++ /dev/null @@ -1,7 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader - -data class FileResult( - val file: String, - val title: String = "", - val sanitizedName: String = "" -) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/MovieInfo.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/MovieInfo.kt deleted file mode 100644 index 4c2b6794..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/MovieInfo.kt +++ /dev/null @@ -1,6 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader - -data class MovieInfo( - val title: String, - override val fullName: String -) : VideoInfo(fullName) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/SubtitleInfo.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/SubtitleInfo.kt deleted file mode 100644 index 1883dfb0..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/SubtitleInfo.kt +++ /dev/null @@ -1,9 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader - -import java.io.File - -data class SubtitleInfo( - val inputFile: String, - val collection: String, - val language: String -) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/VideoInfo.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/VideoInfo.kt deleted file mode 100644 index 1bed99bc..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/VideoInfo.kt +++ /dev/null @@ -1,5 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader - -abstract class VideoInfo( - @Transient open val fullName: String -) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/ConvertWork.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/ConvertWork.kt deleted file mode 100644 index 24fca8f9..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/ConvertWork.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader.work - -import java.util.* - -data class ConvertWork( - val workId: String = UUID.randomUUID().toString(), - val collection: String, - val language: String, - val inFile: String, - val outFiles: List -) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/EncodeWork.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/EncodeWork.kt deleted file mode 100644 index cc26a1ab..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/EncodeWork.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader.work - -import java.util.* - -data class EncodeWork( - override val workId: String = UUID.randomUUID().toString(), - override val collection: String, - override val inFile: String, - override val outFile: String, - val arguments: List -) : WorkBase(collection = collection, inFile = inFile, outFile = outFile) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/ExtractWork.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/ExtractWork.kt deleted file mode 100644 index 92585bd4..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/ExtractWork.kt +++ /dev/null @@ -1,13 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader.work - -import java.util.* - -data class ExtractWork( - override val workId: String = UUID.randomUUID().toString(), - override val collection: String, - val language: String, - override val inFile: String, - val arguments: List, - override val outFile: String, - var produceConvertEvent: Boolean = true -) : WorkBase(collection = collection, inFile = inFile, outFile = outFile) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/WorkBase.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/WorkBase.kt deleted file mode 100644 index a162ebcf..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/dto/reader/work/WorkBase.kt +++ /dev/null @@ -1,10 +0,0 @@ -package no.iktdev.streamit.content.common.dto.reader.work - -import java.util.UUID - -abstract class WorkBase( - @Transient open val workId: String = UUID.randomUUID().toString(), - @Transient open val collection: String, - @Transient open val inFile: String, - @Transient open val outFile: String -) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/SubtitleStreamSelector.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/SubtitleStreamSelector.kt deleted file mode 100644 index 1c362194..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/SubtitleStreamSelector.kt +++ /dev/null @@ -1,47 +0,0 @@ -package no.iktdev.streamit.content.common.streams - -class SubtitleStreamSelector(val streams: List) { - - fun getCandidateForConversion(): List { - val languageGrouped = getDesiredStreams().groupBy { it.tags.language ?: "eng" } - val priority = listOf("subrip", "srt", "webvtt", "vtt", "ass") - - val result = mutableListOf() - for ((language, streams) in languageGrouped) { - val selectedStream = streams.firstOrNull { it.codec_name in priority } - if (selectedStream != null) { - result.add(selectedStream) - } - } - return result - } - - fun getDesiredStreams(): List { - val desiredTypes = listOf(SubtitleType.DEFAULT, SubtitleType.CC, SubtitleType.SHD) - val typeGuesser = SubtitleTypeGuesser() - val codecFiltered = streams.filter { getFormatToCodec(it.codec_name) != null } - - val mappedToType = codecFiltered.map { typeGuesser.guessType(it) to it }.filter { it.first in desiredTypes } - .groupBy { it.second.tags.language ?: "eng" } - .mapValues { entry -> - val languageStreams = entry.value - val sortedStreams = languageStreams.sortedBy { desiredTypes.indexOf(it.first) } - sortedStreams.firstOrNull()?.second - }.mapNotNull { it.value } - - - return mappedToType - } - - - fun getFormatToCodec(codecName: String): String? { - return when(codecName) { - "ass" -> "ass" - "subrip" -> "srt" - "webvtt", "vtt" -> "vtt" - "smi" -> "smi" - "hdmv_pgs_subtitle" -> null - else -> null - } - } -} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/SubtitleTypeGuesser.kt b/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/SubtitleTypeGuesser.kt deleted file mode 100644 index 595a48c0..00000000 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/SubtitleTypeGuesser.kt +++ /dev/null @@ -1,56 +0,0 @@ -package no.iktdev.streamit.content.common.streams - -/** - * @property SHD is Hard of hearing - * @property CC is Closed-Captions - * @property NON_DIALOGUE is for Signs or Song (as in lyrics) - * @property DEFAULT is default subtitle as dialog - */ -enum class SubtitleType { - SHD, - CC, - NON_DIALOGUE, - DEFAULT -} - -class SubtitleTypeGuesser { - fun guessType(subtitle: SubtitleStream): SubtitleType { - if (subtitle.tags != null && subtitle.tags.title?.isBlank() == false) { - val title = subtitle.tags.title!! - if (title.lowercase().contains("song") - || title.lowercase().contains("songs") - || title.lowercase().contains("sign") - || title.lowercase().contains("signs") - ) { - return SubtitleType.NON_DIALOGUE - } - if (getSubtitleType(title, listOf("cc", "closed caption"), - SubtitleType.CC - ) == SubtitleType.CC - ) return SubtitleType.CC - if (getSubtitleType(title, listOf("shd", "hh", "Hard-of-Hearing", "Hard of Hearing"), - SubtitleType.SHD - ) == SubtitleType.SHD - ) return SubtitleType.SHD - } - - return SubtitleType.DEFAULT - } - - private fun getSubtitleType(title: String, keys: List, expected: SubtitleType): SubtitleType { - val bracedText = Regex.fromLiteral("[(](?<=\\().*?(?=\\))[)]").find(title) - val brakedText = Regex.fromLiteral("[(](?<=\\().*?(?=\\))[)]").find(title) - - if (bracedText == null || brakedText == null) - return SubtitleType.DEFAULT - - var text = bracedText.value.ifBlank { brakedText.value } - text = Regex.fromLiteral("[\\[\\]()-.,_+]").replace(text, "") - - return if (keys.find { item -> - item.lowercase().contains(text.lowercase()) || text.lowercase().contains(item.lowercase()) - }.isNullOrEmpty()) SubtitleType.DEFAULT else expected - - } -} - diff --git a/CommonCode/src/test/java/no/iktdev/streamit/content/common/NamingTest.kt b/CommonCode/src/test/java/no/iktdev/streamit/content/common/NamingTest.kt deleted file mode 100644 index 5870942e..00000000 --- a/CommonCode/src/test/java/no/iktdev/streamit/content/common/NamingTest.kt +++ /dev/null @@ -1,62 +0,0 @@ -package no.iktdev.streamit.content.common - -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Named -import org.junit.jupiter.api.Test -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource - -class NamingTest { - - @Test - fun checkThatBracketsGetsRemoved() { - val input = "[AAA] Sir fancy - 13 [1080p HEVC][000000]" - val name = Naming(input) - assertThat(name.guessDesiredTitle()).doesNotContain("[") - - } - - @Test - fun checkThatSeasonIsStripped() { - val input = "[AAA] Kafka Topic S2 - 01" - val naming = Naming(input) - val result = naming.guessDesiredTitle() - assertThat(result).isEqualTo("Kafka Topic") - } - -/* - @ParameterizedTest - @MethodSource("serieOnlyTest") - fun ensureOnlySerieAndDecodedCorrectly(testData: TestData) { - val naming = Naming(testData.input).getName() ?: throw NullPointerException("Named is null") - assertThat(naming.type).isEqualTo("serie") - assertThat(naming.season).isEqualTo(testData.expected.season) - assertThat(naming.episode).isEqualTo(testData.expected.episode) - } - - @Test - fun testTest() { - val tmp = TestData(Naming.Name(title = "Demo", season = 1, episode = 1, type = "serie"), "[Kametsu] Ghost in the Shell Arise - 05 - Pyrophoric Cult (BD 1080p Hi10 FLAC) [13FF85A7]") - val naming = Naming(tmp.input).getName() - assertThat(naming).isNotNull() - } - - - fun serieOnlyTest(): List> { - return listOf( - Named.of("Is defined", TestData(Naming.Name(title = "Demo", season = 1, episode = 1, type = "serie"), "Demo - S01E01")), - Named.of("Is decoded", TestData(Naming.Name("Demo!", "serie", season = 1, episode = 1), "[TMP] Demo! - 03")), - Named.of("Is only Episode", TestData(Naming.Name("Demo", "serie", 1, 1), "Demo E1")) - ) - }*/ - -/* - data class TestData( - val expected: Naming.Name, - val input: String - )*/ -} \ No newline at end of file diff --git a/Convert/.gitignore b/Convert/.gitignore deleted file mode 100644 index b63da455..00000000 --- a/Convert/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -.gradle -build/ -!gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/Convert/Dockerfile b/Convert/Dockerfile deleted file mode 100644 index 973b9031..00000000 --- a/Convert/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM bskjon/azuljava:17 -EXPOSE 8080 - -COPY ./build/libs/converter.jar /usr/share/app/app.jar \ No newline at end of file diff --git a/Convert/gradle/wrapper/gradle-wrapper.jar b/Convert/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 249e5832..00000000 Binary files a/Convert/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/Convert/gradle/wrapper/gradle-wrapper.properties b/Convert/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 249c54e8..00000000 --- a/Convert/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,6 +0,0 @@ -#Sun Jul 23 01:48:17 CEST 2023 -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/Convert/gradlew b/Convert/gradlew deleted file mode 100644 index 1b6c7873..00000000 --- a/Convert/gradlew +++ /dev/null @@ -1,234 +0,0 @@ -#!/bin/sh - -# -# Copyright © 2015-2021 the original authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -# -# Gradle start up script for POSIX generated by Gradle. -# -# Important for running: -# -# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is -# noncompliant, but you have some other compliant shell such as ksh or -# bash, then to run this script, type that shell name before the whole -# command line, like: -# -# ksh Gradle -# -# Busybox and similar reduced shells will NOT work, because this script -# requires all of these POSIX shell features: -# * functions; -# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», -# «${var#prefix}», «${var%suffix}», and «$( cmd )»; -# * compound commands having a testable exit status, especially «case»; -# * various built-in commands including «command», «set», and «ulimit». -# -# Important for patching: -# -# (2) This script targets any POSIX shell, so it avoids extensions provided -# by Bash, Ksh, etc; in particular arrays are avoided. -# -# The "traditional" practice of packing multiple parameters into a -# space-separated string is a well documented source of bugs and security -# problems, so this is (mostly) avoided, by progressively accumulating -# options in "$@", and eventually passing that to Java. -# -# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, -# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; -# see the in-line comments for details. -# -# There are tweaks for specific operating systems such as AIX, CygWin, -# Darwin, MinGW, and NonStop. -# -# (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt -# within the Gradle project. -# -# You can find Gradle at https://github.com/gradle/gradle/. -# -############################################################################## - -# Attempt to set APP_HOME - -# Resolve links: $0 may be a link -app_path=$0 - -# Need this for daisy-chained symlinks. -while - APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path - [ -h "$app_path" ] -do - ls=$( ls -ld "$app_path" ) - link=${ls#*' -> '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" -APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ - "$@" - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/Convert/gradlew.bat b/Convert/gradlew.bat deleted file mode 100644 index 107acd32..00000000 --- a/Convert/gradlew.bat +++ /dev/null @@ -1,89 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/Convert/settings.gradle.kts b/Convert/settings.gradle.kts deleted file mode 100644 index b3c13572..00000000 --- a/Convert/settings.gradle.kts +++ /dev/null @@ -1,4 +0,0 @@ -rootProject.name = "Convert" - -include(":CommonCode") -project(":CommonCode").projectDir = File("../CommonCode") \ No newline at end of file diff --git a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertApplication.kt b/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertApplication.kt deleted file mode 100644 index 4486842d..00000000 --- a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertApplication.kt +++ /dev/null @@ -1,19 +0,0 @@ -package no.iktdev.streamit.content.convert - -import mu.KotlinLogging -import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.boot.runApplication -import org.springframework.context.ApplicationContext - -@SpringBootApplication -class ConvertApplication - -private var context: ApplicationContext? = null -@Suppress("unused") -fun getContext(): ApplicationContext? { - return context -} -fun main(args: Array) { - context = runApplication(*args) -} -private val logger = KotlinLogging.logger {} \ No newline at end of file diff --git a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertEnv.kt b/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertEnv.kt deleted file mode 100644 index 9d3ce1b6..00000000 --- a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertEnv.kt +++ /dev/null @@ -1,7 +0,0 @@ -package no.iktdev.streamit.content.convert - -class ConvertEnv { - companion object { - val allowOverwrite = System.getenv("ALLOW_OVERWRITE").toBoolean() ?: false - } -} \ No newline at end of file diff --git a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertRunner.kt b/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertRunner.kt deleted file mode 100644 index 00005c49..00000000 --- a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/ConvertRunner.kt +++ /dev/null @@ -1,88 +0,0 @@ -package no.iktdev.streamit.content.convert - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.delay -import kotlinx.coroutines.withContext -import mu.KotlinLogging -import no.iktdev.library.subtitle.Syncro -import no.iktdev.library.subtitle.classes.DialogType -import no.iktdev.library.subtitle.export.Export -import no.iktdev.library.subtitle.reader.BaseReader -import no.iktdev.library.subtitle.reader.Reader -import no.iktdev.streamit.content.common.dto.reader.SubtitleInfo -import no.iktdev.streamit.content.common.dto.reader.work.ConvertWork -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.content.common.streams.SubtitleType -import java.io.File - -private val logger = KotlinLogging.logger {} - - -class ConvertRunner(val referenceId: String, val listener: IConvertListener) { - - private fun getReade(inputFile: File): BaseReader? { - return Reader(inputFile).getSubtitleReader() - } - private val maxDelay = 1000 * 5 - private var currentDelayed = 0 - suspend fun readAndConvert (subtitleInfo: SubtitleInfo) { - val inFile = File(subtitleInfo.inputFile) - while (!inFile.canRead()) { - if (currentDelayed > maxDelay) { - logger.error { "Could not out wait lock on file!" } - withContext(Dispatchers.Default) { - listener.onError(referenceId, subtitleInfo, "Cant read file!") - } - return - } - logger.error { "$referenceId ${subtitleInfo.inputFile}: Cant read file!" } - delay(500) - currentDelayed += 500 - } - val reader = getReade(inFile) - val dialogs = reader?.read() - if (dialogs.isNullOrEmpty()) { - logger.error { "$referenceId ${subtitleInfo.inputFile}: Dialogs read from file is null or empty!" } - withContext(Dispatchers.Default) { - listener.onError(referenceId, subtitleInfo, "Dialogs read from file is null or empty!") - } - return - } - - withContext(Dispatchers.Default) { - listener.onStarted(referenceId) - } - - val filtered = dialogs.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) } - - val syncedDialogs = Syncro().sync(filtered) - - try { - val converted = Export(inFile, syncedDialogs, ConvertEnv.allowOverwrite).write() - val item = ConvertWork( - inFile = inFile.absolutePath, - collection = subtitleInfo.collection, - language = subtitleInfo.language, - outFiles = converted.map { it.absolutePath } - ) - - withContext(Dispatchers.Default) { - listener.onEnded(referenceId, subtitleInfo, work = item) - } - } catch (e: Exception) { - e.printStackTrace() - withContext(Dispatchers.Default) { - listener.onError(referenceId, subtitleInfo, "See log") - } - } - - } - - -} - -interface IConvertListener { - fun onStarted(referenceId: String) - fun onError(referenceId: String, info: SubtitleInfo, message: String) - fun onEnded(referenceId: String, info: SubtitleInfo, work: ConvertWork) -} \ No newline at end of file diff --git a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/kafka/SubtitleConsumer.kt b/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/kafka/SubtitleConsumer.kt deleted file mode 100644 index f914adb1..00000000 --- a/Convert/src/main/kotlin/no/iktdev/streamit/content/convert/kafka/SubtitleConsumer.kt +++ /dev/null @@ -1,69 +0,0 @@ -package no.iktdev.streamit.content.convert.kafka - -import kotlinx.coroutines.launch -import mu.KotlinLogging -import no.iktdev.exfl.coroutines.Coroutines -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.common.dto.reader.SubtitleInfo -import no.iktdev.streamit.content.common.dto.reader.work.ConvertWork -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.content.convert.ConvertRunner -import no.iktdev.streamit.content.convert.IConvertListener -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.SimpleMessageListener -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.springframework.stereotype.Service -import java.io.File - -private val logger = KotlinLogging.logger {} - -@Service -class SubtitleConsumer: DefaultKafkaReader("convertHandlerSubtitle"), IConvertListener { - - private final val listener = object : SimpleMessageListener( - topic = CommonConfig.kafkaTopic, - consumer = defaultConsumer, - accepts = listOf(KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_ENDED.event) - ) { - override fun onMessageReceived(data: ConsumerRecord) { - val referenceId = data.value().referenceId - val workResult = data.value().dataAs(ExtractWork::class.java) - - if (workResult?.produceConvertEvent == true) { - logger.info { "Using ${data.value().referenceId} ${workResult.outFile} as it is a convert candidate" } - val convertWork = SubtitleInfo( - inputFile = workResult.outFile, - collection = workResult.collection, - language = workResult.language, - ) - produceMessage(KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_STARTED, Message(referenceId = referenceId, Status(statusType = StatusType.PENDING)), convertWork) - Coroutines.io().launch { - ConvertRunner(referenceId, this@SubtitleConsumer).readAndConvert(convertWork) - } - } else { - logger.info { "Skipping ${data.value().referenceId} ${workResult?.outFile} as it is not a convert candidate" } - } - } - } - - init { - listener.listen() - } - - override fun onStarted(referenceId: String) { - produceMessage(KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_STARTED, Message(referenceId = referenceId, Status(statusType = StatusType.SUCCESS)), null) - } - - override fun onError(referenceId: String, info: SubtitleInfo, message: String) { - produceMessage(KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_ENDED, Message(referenceId = referenceId, Status(statusType = StatusType.ERROR, message = message)), null) - } - - override fun onEnded(referenceId: String, info: SubtitleInfo, work: ConvertWork) { - produceMessage(KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_ENDED, Message(referenceId = referenceId, Status(statusType = StatusType.SUCCESS)), work) - } - -} \ No newline at end of file diff --git a/Convert/src/main/resources/application.properties b/Convert/src/main/resources/application.properties deleted file mode 100644 index b67553fc..00000000 --- a/Convert/src/main/resources/application.properties +++ /dev/null @@ -1,3 +0,0 @@ -spring.output.ansi.enabled=always -logging.level.org.apache.kafka=INFO -#logging.level.root=DEBUG diff --git a/Encode/.gitignore b/Encode/.gitignore deleted file mode 100644 index b63da455..00000000 --- a/Encode/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -.gradle -build/ -!gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/Encode/Dockerfile b/Encode/Dockerfile deleted file mode 100644 index cd8a8172..00000000 --- a/Encode/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM bskjon/debian-azuljava17-ffmpeg:latest -EXPOSE 8080 - -COPY ./build/libs/encoder.jar /usr/share/app/app.jar \ No newline at end of file diff --git a/Encode/build.gradle.kts b/Encode/build.gradle.kts deleted file mode 100644 index 712ddcd3..00000000 --- a/Encode/build.gradle.kts +++ /dev/null @@ -1,66 +0,0 @@ -import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.util.archivesName - -plugins { - kotlin("jvm") version "1.8.21" - id("org.springframework.boot") version "2.5.5" - id("io.spring.dependency-management") version "1.0.11.RELEASE" - kotlin("plugin.spring") version "1.5.31" -} - -group = "no.iktdev.streamit.content" -version = "1.0-SNAPSHOT" - -repositories { - mavenCentral() - maven("https://jitpack.io") - maven { - url = uri("https://reposilite.iktdev.no/releases") - } - maven { - url = uri("https://reposilite.iktdev.no/snapshots") - } -} -dependencies { - implementation(project(":CommonCode")) - - implementation("no.iktdev.streamit.library:streamit-library-kafka:0.0.2-alpha84") - implementation("no.iktdev:exfl:0.0.13-SNAPSHOT") - - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") - - - implementation("com.github.pgreze:kotlin-process:1.3.1") - implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") - - implementation("com.google.code.gson:gson:2.8.9") - - implementation("org.springframework.boot:spring-boot-starter-web") - implementation("org.springframework.boot:spring-boot-starter:2.7.0") - implementation("org.springframework.kafka:spring-kafka:2.8.5") - implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") - - - - testImplementation("junit:junit:4.13.2") - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") - testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1") - testImplementation("org.assertj:assertj-core:3.4.1") - testImplementation("org.mockito:mockito-core:3.+") - -} - -tasks.test { - useJUnitPlatform() -} - -tasks.bootJar { - archiveFileName.set("encoder.jar") - launchScript() -} - -tasks.jar { - archivesName.set("encoder.jar") - archiveBaseName.set("encoder") -} -archivesName.set("encoder.jar") \ No newline at end of file diff --git a/Encode/gradle/wrapper/gradle-wrapper.jar b/Encode/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 249e5832..00000000 Binary files a/Encode/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/Encode/gradle/wrapper/gradle-wrapper.properties b/Encode/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index ab96900a..00000000 --- a/Encode/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,6 +0,0 @@ -#Tue Jul 11 02:14:45 CEST 2023 -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/Encode/gradlew b/Encode/gradlew deleted file mode 100644 index 1b6c7873..00000000 --- a/Encode/gradlew +++ /dev/null @@ -1,234 +0,0 @@ -#!/bin/sh - -# -# Copyright © 2015-2021 the original authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -# -# Gradle start up script for POSIX generated by Gradle. -# -# Important for running: -# -# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is -# noncompliant, but you have some other compliant shell such as ksh or -# bash, then to run this script, type that shell name before the whole -# command line, like: -# -# ksh Gradle -# -# Busybox and similar reduced shells will NOT work, because this script -# requires all of these POSIX shell features: -# * functions; -# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», -# «${var#prefix}», «${var%suffix}», and «$( cmd )»; -# * compound commands having a testable exit status, especially «case»; -# * various built-in commands including «command», «set», and «ulimit». -# -# Important for patching: -# -# (2) This script targets any POSIX shell, so it avoids extensions provided -# by Bash, Ksh, etc; in particular arrays are avoided. -# -# The "traditional" practice of packing multiple parameters into a -# space-separated string is a well documented source of bugs and security -# problems, so this is (mostly) avoided, by progressively accumulating -# options in "$@", and eventually passing that to Java. -# -# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, -# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; -# see the in-line comments for details. -# -# There are tweaks for specific operating systems such as AIX, CygWin, -# Darwin, MinGW, and NonStop. -# -# (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt -# within the Gradle project. -# -# You can find Gradle at https://github.com/gradle/gradle/. -# -############################################################################## - -# Attempt to set APP_HOME - -# Resolve links: $0 may be a link -app_path=$0 - -# Need this for daisy-chained symlinks. -while - APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path - [ -h "$app_path" ] -do - ls=$( ls -ld "$app_path" ) - link=${ls#*' -> '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" -APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ - "$@" - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/Encode/gradlew.bat b/Encode/gradlew.bat deleted file mode 100644 index 107acd32..00000000 --- a/Encode/gradlew.bat +++ /dev/null @@ -1,89 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/Encode/settings.gradle.kts b/Encode/settings.gradle.kts deleted file mode 100644 index 783ff466..00000000 --- a/Encode/settings.gradle.kts +++ /dev/null @@ -1,4 +0,0 @@ -rootProject.name = "Encode" - -include(":CommonCode") -project(":CommonCode").projectDir = File("../CommonCode") \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/Configuration.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/Configuration.kt deleted file mode 100644 index add5e85e..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/Configuration.kt +++ /dev/null @@ -1,35 +0,0 @@ -package no.iktdev.streamit.content.encode - -import org.springframework.beans.factory.annotation.Value -import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory -import org.springframework.boot.web.server.WebServerFactoryCustomizer -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.messaging.simp.config.MessageBrokerRegistry -import org.springframework.web.bind.annotation.RestController -import org.springframework.web.method.HandlerTypePredicate -import org.springframework.web.servlet.config.annotation.CorsRegistry -import org.springframework.web.servlet.config.annotation.PathMatchConfigurer -import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer -import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker -import org.springframework.web.socket.config.annotation.StompEndpointRegistry -import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer - -@Configuration -@EnableWebSocketMessageBroker -class WebSocketConfig : WebSocketMessageBrokerConfigurer { - - override fun registerStompEndpoints(registry: StompEndpointRegistry) { - registry.addEndpoint("/ws") - // .setAllowedOrigins("*") - .withSockJS() - - registry.addEndpoint("/") - } - - override fun configureMessageBroker(registry: MessageBrokerRegistry) { - registry.enableSimpleBroker("/topic") - registry.setApplicationDestinationPrefixes("/app") - } -} \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncodeEnv.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncodeEnv.kt deleted file mode 100644 index ac83adda..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncodeEnv.kt +++ /dev/null @@ -1,9 +0,0 @@ -package no.iktdev.streamit.content.encode - -class EncodeEnv { - companion object { - val ffmpeg: String = System.getenv("SUPPORTING_EXECUTABLE_FFMPEG") ?: "ffmpeg" - val allowOverwrite = System.getenv("ALLOW_OVERWRITE").toBoolean() ?: false - val maxRunners: Int = try {System.getenv("SIMULTANEOUS_ENCODE_RUNNERS").toIntOrNull() ?: 1 } catch (e: Exception) {1} - } -} \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncodeWorkConsumer.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncodeWorkConsumer.kt deleted file mode 100644 index 39741a02..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncodeWorkConsumer.kt +++ /dev/null @@ -1,60 +0,0 @@ -package no.iktdev.streamit.content.encode - -import com.google.gson.Gson -import mu.KotlinLogging -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.common.deserializers.DeserializerRegistry -import no.iktdev.streamit.content.common.deserializers.EncodeWorkDeserializer -import no.iktdev.streamit.content.encode.runner.RunnerCoordinator -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.consumers.DefaultConsumer -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.SimpleMessageListener -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import no.iktdev.streamit.library.kafka.listener.deserializer.deserializeIfSuccessful -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.springframework.stereotype.Service - -private val logger = KotlinLogging.logger {} - -@Service -class EncodeWorkConsumer(private val runnerCoordinator: RunnerCoordinator) : DefaultKafkaReader("encodeWork") { - - lateinit var encodeInstructionsListener: EncodeInformationListener - - init { - encodeInstructionsListener = EncodeInformationListener( - topic = CommonConfig.kafkaTopic, - defaultConsumer, - accepts = listOf(KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO.event), - runnerCoordinator - ) - encodeInstructionsListener.listen() - } - - override fun loadDeserializers(): Map> { - return DeserializerRegistry.getEventToDeserializer( - KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO - ) - } - - - class EncodeInformationListener( - topic: String, - consumer: DefaultConsumer, - accepts: List, - val runnerCoordinator: RunnerCoordinator - ) : SimpleMessageListener( - topic, consumer, - accepts - ) { - override fun onMessageReceived(data: ConsumerRecord) { - logger.info { "\nreferenceId: ${data.value().referenceId} \nEvent: ${data.key()} \nData:\n${Gson().toJson(data.value())}" } - val message = data.value().apply { - this.data = EncodeWorkDeserializer().deserializeIfSuccessful(data.value()) - } - runnerCoordinator.addEncodeMessageToQueue(message) - } - } -} \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncoderApplication.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncoderApplication.kt deleted file mode 100644 index 4177968d..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/EncoderApplication.kt +++ /dev/null @@ -1,35 +0,0 @@ -package no.iktdev.streamit.content.encode - -import no.iktdev.exfl.observable.ObservableMap -import no.iktdev.exfl.observable.observableMapOf -import no.iktdev.streamit.content.common.dto.WorkOrderItem -import no.iktdev.streamit.content.encode.progress.Progress -import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.boot.runApplication -import org.springframework.context.ApplicationContext - -@SpringBootApplication -class EncoderApplication - -private var context: ApplicationContext? = null -val progressMap = observableMapOf() - -@Suppress("unused") -fun getContext(): ApplicationContext? { - return context -} -fun main(args: Array) { - context = runApplication(*args) -} - -val encoderItems = ObservableMap() -val extractItems = ObservableMap() - -/*val progress = ObservableMap().also { - it.addListener(object: ObservableMap.Listener { - override fun onPut(key: String, value: EncodeInformation) { - super.onPut(key, value) - logger.info { "$key with progress: $value." } - } - }) -}*/ \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/ExtractWorkConsumer.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/ExtractWorkConsumer.kt deleted file mode 100644 index 361d2fa6..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/ExtractWorkConsumer.kt +++ /dev/null @@ -1,59 +0,0 @@ -package no.iktdev.streamit.content.encode - -import com.google.gson.Gson -import mu.KotlinLogging -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.common.deserializers.DeserializerRegistry -import no.iktdev.streamit.content.common.deserializers.ExtractWorkDeserializer -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.content.encode.runner.RunnerCoordinator -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.consumers.DefaultConsumer -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.SimpleMessageListener -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import no.iktdev.streamit.library.kafka.listener.deserializer.deserializeIfSuccessful -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.springframework.stereotype.Service -private val logger = KotlinLogging.logger {} - -@Service -class ExtractWorkConsumer(private val runnerCoordinator: RunnerCoordinator) : DefaultKafkaReader("extractWork") { - lateinit var encodeInstructionsListener: ExtractWorkListener - - init { - encodeInstructionsListener = ExtractWorkListener( - topic = CommonConfig.kafkaTopic, - defaultConsumer, - accepts = listOf(KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE.event), - runnerCoordinator - ) - encodeInstructionsListener.listen() - } - - override fun loadDeserializers(): Map> { - return DeserializerRegistry.getEventToDeserializer( - KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE - ) - } - - - class ExtractWorkListener( - topic: String, - consumer: DefaultConsumer, - accepts: List, - val runnerCoordinator: RunnerCoordinator - ) : SimpleMessageListener( - topic, consumer, - accepts - ) { - override fun onMessageReceived(data: ConsumerRecord) { - logger.info { "\nreferenceId: ${data.value().referenceId} \nEvent: ${data.key()} \nData:\n${Gson().toJson(data.value())}" } - val message = data.value().apply { - this.data = ExtractWorkDeserializer().deserializeIfSuccessful(data.value()) - } - runnerCoordinator.addExtractMessageToQueue(message) - } - } -} \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/controllers/ProgressController.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/controllers/ProgressController.kt deleted file mode 100644 index 29f5edee..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/controllers/ProgressController.kt +++ /dev/null @@ -1,16 +0,0 @@ -package no.iktdev.streamit.content.encode.controllers - -import com.google.gson.Gson -import no.iktdev.streamit.content.encode.progressMap -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.RestController -import javax.servlet.http.HttpServletResponse - -@RestController -class ProgressController { - @GetMapping("/progress") - fun getValue(response: HttpServletResponse): String { - response.setHeader("Refresh", "5") - return Gson().toJson(progressMap.values) - } -} diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/DecodedProgressData.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/DecodedProgressData.kt deleted file mode 100644 index 5705b35a..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/DecodedProgressData.kt +++ /dev/null @@ -1,18 +0,0 @@ -package no.iktdev.streamit.content.encode.progress - -data class DecodedProgressData( - val frame: Int?, - val fps: Double?, - val stream_0_0_q: Double?, - val bitrate: String?, - val total_size: Int?, - val out_time_us: Long?, - val out_time_ms: Long?, - val out_time: String?, - val dup_frames: Int?, - val drop_frames: Int?, - val speed: Double?, - val progress: String? -) - -data class ECT(val day: Int = 0, val hour: Int = 0, val minute: Int = 0, val second: Int = 0) diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/Progress.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/Progress.kt deleted file mode 100644 index bebf3271..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/Progress.kt +++ /dev/null @@ -1,12 +0,0 @@ -package no.iktdev.streamit.content.encode.progress - -data class Progress( - val workId: String, - val outFileName: String, - val progress: Int = -1, - val time: String, - val duration: String, - val speed: String, - val estimatedCompletionSeconds: Long = -1, - val estimatedCompletion: String = "Unknown", -) \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/ProgressDecoder.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/ProgressDecoder.kt deleted file mode 100644 index 9349c8fd..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/progress/ProgressDecoder.kt +++ /dev/null @@ -1,141 +0,0 @@ -package no.iktdev.streamit.content.encode.progress - -import no.iktdev.streamit.content.common.dto.reader.work.WorkBase -import java.io.File -import java.lang.StringBuilder -import java.time.LocalTime -import java.time.format.DateTimeFormatter -import java.util.concurrent.TimeUnit -import kotlin.math.floor - -class ProgressDecoder(val workBase: WorkBase) { - val expectedKeys = listOf( - "frame=", - "fps=", - "stream_0_0_q=", - "bitrate=", - "total_size=", - "out_time_us=", - "out_time_ms=", - "out_time=", - "dup_frames=", - "drop_frames=", - "speed=", - "progress=" - ) - var duration: Int? = null - set(value) { - if (field == null || field == 0) - field = value - } - var durationTime: String = "NA" - fun parseVideoProgress(lines: List): DecodedProgressData? { - var frame: Int? = null - var progress: String? = null - val metadataMap = mutableMapOf() - - for (line in lines) { - val keyValuePairs = Regex("=\\s*").replace(line, "=").split(" ").filter { it.isNotBlank() } - for (keyValuePair in keyValuePairs) { - val (key, value) = keyValuePair.split("=") - metadataMap[key] = value - } - - if (frame == null) { - frame = metadataMap["frame"]?.toIntOrNull() - } - - progress = metadataMap["progress"] - } - - return if (progress != null) { - // When "progress" is found, build and return the VideoMetadata object - DecodedProgressData( - frame, metadataMap["fps"]?.toDoubleOrNull(), metadataMap["stream_0_0_q"]?.toDoubleOrNull(), - metadataMap["bitrate"], metadataMap["total_size"]?.toIntOrNull(), metadataMap["out_time_us"]?.toLongOrNull(), - metadataMap["out_time_ms"]?.toLongOrNull(), metadataMap["out_time"], metadataMap["dup_frames"]?.toIntOrNull(), - metadataMap["drop_frames"]?.toIntOrNull(), metadataMap["speed"]?.replace("x", "", ignoreCase = true)?.toDoubleOrNull(), progress - ) - } else { - null // If "progress" is not found, return null - } - } - - - fun isDuration(value: String): Boolean { - return value.contains("Duration", ignoreCase = true) - } - fun setDuration(value: String) { - val results = Regex("Duration:\\s*([^,]+),").find(value)?.groupValues?.firstOrNull() - durationTime = Regex("[0-9]+:[0-9]+:[0-9]+.[0-9]+").find(results.toString())?.value ?: "NA" - duration = timeSpanToSeconds(results) - } - - private fun timeSpanToSeconds(time: String?): Int? - { - time ?: return null - val timeString = Regex("[0-9]+:[0-9]+:[0-9]+.[0-9]+").find(time) ?: return null - val strippedMS = Regex("[0-9]+:[0-9]+:[0-9]+").find(timeString.value) ?: return null - val outTime = LocalTime.parse(strippedMS.value, DateTimeFormatter.ofPattern("HH:mm:ss")) - return outTime.toSecondOfDay() - } - - - fun getProgress(decoded: DecodedProgressData): Progress { - if (duration == null) - return Progress(workId = workBase.workId, outFileName = File(workBase.outFile).name, duration = durationTime, time = "NA", speed = "NA") - val progressTime = timeSpanToSeconds(decoded.out_time) ?: 0 - val progress = floor((progressTime.toDouble() / duration!!.toDouble()) *100).toInt() - - val ect = getEstimatedTimeRemaining(decoded) - - return Progress( - workId = workBase.workId, outFileName = File(workBase.outFile).name, - progress = progress, - estimatedCompletionSeconds = ect, - estimatedCompletion = getETA(ect), - duration = durationTime, - time = decoded.out_time ?: "NA", - speed = decoded.speed?.toString() ?: "NA" - ) - } - - fun getEstimatedTimeRemaining(decoded: DecodedProgressData): Long { - val position = timeSpanToSeconds(decoded.out_time) ?: 0 - return if(duration == null || decoded.speed == null) -1 else - Math.round(Math.round(duration!!.toDouble() - position.toDouble()) / decoded.speed) - } - - fun getECT(time: Long): ECT { - var seconds = time - val day = TimeUnit.SECONDS.toDays(seconds) - seconds -= java.util.concurrent.TimeUnit.DAYS.toSeconds(day) - - val hour = TimeUnit.SECONDS.toHours(seconds) - seconds -= java.util.concurrent.TimeUnit.HOURS.toSeconds(hour) - - val minute = TimeUnit.SECONDS.toMinutes(seconds) - seconds -= java.util.concurrent.TimeUnit.MINUTES.toSeconds(minute) - - return ECT(day.toInt(), hour.toInt(), minute.toInt(), seconds.toInt()) - } - private fun getETA(time: Long): String { - val etc = getECT(time) ?: return "Unknown" - val str = StringBuilder() - if (etc.day > 0) { - str.append("${etc.day}d").append(" ") - } - if (etc.hour > 0) { - str.append("${etc.hour}h").append(" ") - } - if (etc.day == 0 && etc.minute > 0) { - str.append("${etc.minute}m").append(" ") - } - if (etc.hour == 0 && etc.second > 0) { - str.append("${etc.second}s").append(" ") - } - return str.toString().trim() - } - - -} \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/EncodeDaemon.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/EncodeDaemon.kt deleted file mode 100644 index 9458f257..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/EncodeDaemon.kt +++ /dev/null @@ -1,108 +0,0 @@ -package no.iktdev.streamit.content.encode.runner - -import mu.KotlinLogging -import no.iktdev.streamit.content.encode.EncodeEnv -import no.iktdev.exfl.observable.ObservableList -import no.iktdev.exfl.observable.observableListOf -import no.iktdev.exfl.using -import no.iktdev.streamit.content.common.deamon.Daemon -import no.iktdev.streamit.content.common.deamon.IDaemon -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.content.encode.progress.DecodedProgressData -import no.iktdev.streamit.content.encode.progress.Progress -import no.iktdev.streamit.content.encode.progress.ProgressDecoder -import java.io.BufferedWriter -import java.io.File -import java.io.FileWriter - -private val logger = KotlinLogging.logger {} - -class EncodeDaemon(val referenceId: String, val work: EncodeWork, val daemonInterface: IEncodeListener, val outFile: File = File("src").using("logs", "${work.workId}-${work.collection}.log")): IDaemon { - var outputCache = observableListOf() - private val decoder = ProgressDecoder(work) - fun produceProgress(items: List): Progress? { - try { - val decodedProgress = decoder.parseVideoProgress(items) - if (decodedProgress != null) { - val progress = decoder.getProgress(decodedProgress) - outputCache.clear() - return progress - } - } catch (e: IndexOutOfBoundsException) { - // Do nothing - } catch (e: Exception) { - //logger.error { e.message } - e.printStackTrace() - } - return null - } - - init { - outputCache.addListener(object : ObservableList.Listener { - override fun onAdded(item: String) { - val progress = produceProgress(outputCache) - progress?.let { - daemonInterface.onProgress(referenceId, work, progress) - } - } - }) - outFile.parentFile.mkdirs() - } - - suspend fun runUsingWorkItem(): Int { - val outFile = File(work.outFile) - if (!outFile.parentFile.exists()) { - outFile.parentFile.mkdirs() - } - val adjustedArgs = (if (EncodeEnv.allowOverwrite) listOf("-y") else listOf("-nostdin")) + listOf( - "-hide_banner", "-i", File(work.inFile).absolutePath, *work.arguments.toTypedArray(), outFile.absolutePath, - "-progress", "pipe:1" - ) - logger.info { "$referenceId @ ${work.workId} ${adjustedArgs.joinToString(" ")}" } - return Daemon(EncodeEnv.ffmpeg, this).run(adjustedArgs) - } - - override fun onStarted() { - super.onStarted() - daemonInterface.onStarted(referenceId, work) - } - - override fun onEnded() { - super.onEnded() - daemonInterface.onEnded(referenceId, work) - } - - override fun onError(code: Int) { - daemonInterface.onError(referenceId, work, code) - } - - override fun onOutputChanged(line: String) { - super.onOutputChanged(line) - if (decoder.isDuration(line)) - decoder.setDuration(line) - if (decoder.expectedKeys.any { line.startsWith(it) }) { - outputCache.add(line) - } - writeToLog(line) - } - fun writeToLog(line: String) { - val fileWriter = FileWriter(outFile, true) // true indikerer at vi ønsker å appende til filen - val bufferedWriter = BufferedWriter(fileWriter) - - // Skriv logglinjen til filen - bufferedWriter.write(line) - bufferedWriter.newLine() // Legg til en ny linje etter logglinjen - - // Lukk BufferedWriter og FileWriter for å frigjøre ressurser - bufferedWriter.close() - fileWriter.close() - } - -} - -interface IEncodeListener { - fun onStarted(referenceId: String, work: EncodeWork) - fun onError(referenceId: String, work: EncodeWork, code: Int) - fun onProgress(referenceId: String, work: EncodeWork, progress: Progress) - fun onEnded(referenceId: String, work: EncodeWork) -} \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/ExtractDaemon.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/ExtractDaemon.kt deleted file mode 100644 index edb06b99..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/ExtractDaemon.kt +++ /dev/null @@ -1,54 +0,0 @@ -package no.iktdev.streamit.content.encode.runner - -import mu.KotlinLogging -import no.iktdev.streamit.content.encode.EncodeEnv -import no.iktdev.exfl.observable.observableListOf -import no.iktdev.streamit.content.common.deamon.Daemon -import no.iktdev.streamit.content.common.deamon.IDaemon -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.content.encode.progress.DecodedProgressData -import java.io.File -private val logger = KotlinLogging.logger {} - -class ExtractDaemon(val referenceId: String, val work: ExtractWork, val daemonInterface: IExtractListener): IDaemon { - var outputCache = observableListOf() - - - suspend fun runUsingWorkItem(): Int { - val outFile = File(work.outFile) - if (!outFile.parentFile.exists()) { - outFile.parentFile.mkdirs() - } - val adjustedArgs = (if (EncodeEnv.allowOverwrite) listOf("-y") else emptyList()) + listOf( - "-i", File(work.inFile).absolutePath, *work.arguments.toTypedArray(), outFile.absolutePath - ) - logger.info { "$referenceId @ ${work.workId} ${adjustedArgs.joinToString(" ")}" } - return Daemon(EncodeEnv.ffmpeg, this).run(adjustedArgs) - } - - override fun onStarted() { - super.onStarted() - daemonInterface.onStarted(referenceId, work) - } - - override fun onEnded() { - super.onEnded() - daemonInterface.onEnded(referenceId, work) - } - - override fun onError(code: Int) { - daemonInterface.onError(referenceId, work, code) - } - override fun onOutputChanged(line: String) { - super.onOutputChanged(line) - outputCache.add(line) - } - -} - -interface IExtractListener { - fun onStarted(referenceId: String, work: ExtractWork) - fun onError(referenceId: String, work: ExtractWork, code: Int) - fun onProgress(referenceId: String, work: ExtractWork, progress: DecodedProgressData) {} - fun onEnded(referenceId: String, work: ExtractWork) -} \ No newline at end of file diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/RunnerCoordinator.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/RunnerCoordinator.kt deleted file mode 100644 index a7f3885f..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/runner/RunnerCoordinator.kt +++ /dev/null @@ -1,324 +0,0 @@ -package no.iktdev.streamit.content.encode.runner - -import com.google.gson.Gson -import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel -import no.iktdev.streamit.content.encode.EncodeEnv -import mu.KotlinLogging -import no.iktdev.exfl.coroutines.Coroutines -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.dto.State -import no.iktdev.streamit.content.common.dto.WorkOrderItem -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.content.encode.encoderItems -import no.iktdev.streamit.content.encode.extractItems -import no.iktdev.streamit.content.encode.progress.Progress -import no.iktdev.streamit.content.encode.progressMap -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.producer.DefaultProducer -import org.springframework.stereotype.Service -import java.util.concurrent.atomic.AtomicInteger - -private val logger = KotlinLogging.logger {} - -data class ExecutionBlock( - val workId: String, - val type: String, - val work: suspend () -> Int -) - -@Service -class RunnerCoordinator( - private var maxConcurrentJobs: Int = 1, -) { - private val logger = KotlinLogging.logger {} - - val producer = DefaultProducer(CommonConfig.kafkaTopic) - final val defaultScope = Coroutines.default() - - private val jobsInProgress = AtomicInteger(0) - private var inProgressJobs = mutableListOf() - val queue = Channel(Channel.UNLIMITED) - - - init { - maxConcurrentJobs = EncodeEnv.maxRunners - repeat(EncodeEnv.maxRunners) { - launchWorker() - } - } - - fun launchWorker() = defaultScope.launch { - while (true) { - logger.info("Worker is waiting for a work item...") - val workItem = queue.receive() // Coroutine will wait here until a work item is available - logger.info("Worker received a work item.") - if (jobsInProgress.get() < maxConcurrentJobs) { - jobsInProgress.incrementAndGet() - val job = processWorkItem(workItem) - inProgressJobs.add(job) - job.invokeOnCompletion { - logger.info { "OnCompletion invoked!\n\nWorkId: ${workItem.workId}-${workItem.type} \n\tCurrent active worksers: ${jobsInProgress.get()}" } - val workers = jobsInProgress.decrementAndGet() - logger.info { "Worker Released: $workers" } - logger.info { "Available: ${workers}/${maxConcurrentJobs}" } - inProgressJobs.remove(job) - } - } - logger.info { "Available workers: ${jobsInProgress.get()}/$maxConcurrentJobs" } - - } - } - - private suspend fun processWorkItem(workItem: ExecutionBlock): Job { - logger.info { "Processing work: ${workItem.type}" } - workItem.work() - return Job().apply { complete() } - } - - - fun addEncodeMessageToQueue(message: Message) { - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_QUEUED.event, - message.withNewStatus(Status(StatusType.PENDING)) - ) - try { - if (message.data != null && message.data is EncodeWork) { - val work = message.data as EncodeWork - encoderItems.put( - message.referenceId, WorkOrderItem( - id = message.referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.QUEUED - ) - ) - - val workBlock = suspend { - val data: EncodeWork = work - val encodeDaemon = EncodeDaemon(message.referenceId, data, encodeListener) - logger.info { "\nreferenceId: ${message.referenceId} \nStarting encoding. \nWorkId: ${data.workId}" } - encodeDaemon.runUsingWorkItem() - } - val result = queue.trySend(ExecutionBlock(work.workId, "encode", workBlock)) - val statusType = when (result.isClosed) { - true -> StatusType.IGNORED // Køen er lukket, jobben ble ignorert - false -> { - if (result.isSuccess) { - StatusType.SUCCESS // Jobben ble sendt til køen - } else { - StatusType.ERROR // Feil ved sending av jobben - } - } - } - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_QUEUED.event, - message.withNewStatus(Status(statusType)) - ) - } else { - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_QUEUED.event, - message.withNewStatus(Status(StatusType.ERROR, "Data is not an instance of EncodeWork or null")) - ) - } - } catch (e: Exception) { - e.printStackTrace() - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_QUEUED.event, - message.withNewStatus(Status(StatusType.ERROR, e.message)) - ) - } - } - - fun addExtractMessageToQueue(message: Message) { - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_QUEUED.event, - message.withNewStatus(Status(StatusType.PENDING)) - ) - try { - if (message.data != null && message.data is ExtractWork) { - val work = message.data as ExtractWork - extractItems.put( - message.referenceId, WorkOrderItem( - id = message.referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.QUEUED - ) - ) - val workBlock = suspend { - val data: ExtractWork = work - val extractDaemon = ExtractDaemon(message.referenceId, data, extractListener) - logger.info { "\nreferenceId: ${message.referenceId} \nStarting extracting. \nWorkId: ${data.workId}" } - extractDaemon.runUsingWorkItem() - } - val result = queue.trySend(ExecutionBlock(work.workId, "extract", workBlock)) - val statusType = when (result.isClosed) { - true -> StatusType.IGNORED // Køen er lukket, jobben ble ignorert - false -> { - if (result.isSuccess) { - StatusType.SUCCESS // Jobben ble sendt til køen - } else { - StatusType.ERROR // Feil ved sending av jobben - } - } - } - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_QUEUED.event, - message.withNewStatus(Status(statusType)) - ) - } else { - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_QUEUED.event, - message.withNewStatus(Status(StatusType.ERROR, "Data is not an instance of ExtractWork")) - ) - } - } catch (e: Exception) { - e.printStackTrace() - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_QUEUED.event, - message.withNewStatus(Status(StatusType.ERROR, e.message)) - ) - } - } - - - val encodeListener = object : IEncodeListener { - override fun onStarted(referenceId: String, work: EncodeWork) { - logger.info { "\nreferenceId: $referenceId \nWorkId ${work.workId} \nEncode: Started\n${work.outFile}" } - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_STARTED.event, - Message(referenceId, Status(statusType = StatusType.SUCCESS), work) - ) - encoderItems.put( - referenceId, WorkOrderItem( - id = referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.STARTED - ) - ) - } - - override fun onError(referenceId: String, work: EncodeWork, code: Int) { - logger.error { "\nreferenceId: $referenceId \nWorkId ${work.workId} \nEncode: Failed\n${work.outFile} \nError: $code" } - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_ENDED.event, - Message(referenceId, Status(StatusType.ERROR, message = code.toString()), work) - ) - encoderItems.put( - referenceId, WorkOrderItem( - id = referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.FAILURE - ) - ) - } - - override fun onProgress(referenceId: String, work: EncodeWork, progress: Progress) { - logger.debug { - "Work progress for $referenceId with WorkId ${work.workId} @ ${work.outFile}: Progress: ${ - Gson().toJson( - progress - ) - }" - } - progressMap.put(work.workId, progress) - encoderItems.put( - referenceId, WorkOrderItem( - id = referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.UPDATED, - progress = progress.progress, - remainingTime = progress.estimatedCompletionSeconds - ) - ) - } - - override fun onEnded(referenceId: String, work: EncodeWork) { - logger.info { "\nreferenceId: $referenceId \nWorkId ${work.workId} \nEncode: Ended\n${work.outFile}" } - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_VIDEO_FILE_ENDED.event, - Message(referenceId, Status(statusType = StatusType.SUCCESS), work) - ) - encoderItems.put( - referenceId, WorkOrderItem( - id = referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.ENDED, - progress = 100, - remainingTime = null - ) - ) - } - } - - val extractListener = object : IExtractListener { - override fun onStarted(referenceId: String, work: ExtractWork) { - logger.info { "\nreferenceId: $referenceId \nWorkId ${work.workId} \nExtract: Started\n${work.outFile}" } - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_STARTED.event, - Message(referenceId, Status(statusType = StatusType.SUCCESS), work) - ) - extractItems.put( - referenceId, WorkOrderItem( - id = referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.STARTED - ) - ) - } - - override fun onError(referenceId: String, work: ExtractWork, code: Int) { - logger.error { "\nreferenceId: $referenceId \nWorkId ${work.workId} \nExtract: Failed\n${work.outFile} \nError: $code" } - - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_ENDED.event, - Message(referenceId, Status(StatusType.ERROR, code.toString()), work) - ) - extractItems.put( - referenceId, WorkOrderItem( - id = referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.FAILURE - ) - ) - } - - override fun onEnded(referenceId: String, work: ExtractWork) { - logger.info { "\nreferenceId: $referenceId \nWorkId ${work.workId} \nExtract: Ended\n${work.outFile}" } - producer.sendMessage( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_ENDED.event, - Message(referenceId, Status(statusType = StatusType.SUCCESS), work) - ) - extractItems.put( - referenceId, WorkOrderItem( - id = referenceId, - inputFile = work.inFile, - outputFile = work.outFile, - collection = work.collection, - state = State.ENDED - ) - ) - } - - } - -} diff --git a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/topics/EncoderTopic.kt b/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/topics/EncoderTopic.kt deleted file mode 100644 index e4d8b5c1..00000000 --- a/Encode/src/main/kotlin/no/iktdev/streamit/content/encode/topics/EncoderTopic.kt +++ /dev/null @@ -1,65 +0,0 @@ -package no.iktdev.streamit.content.encode.topics - -import no.iktdev.exfl.observable.ObservableMap -import no.iktdev.streamit.content.common.dto.WorkOrderItem -import no.iktdev.streamit.content.encode.encoderItems -import no.iktdev.streamit.content.encode.extractItems -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.messaging.handler.annotation.MessageMapping -import org.springframework.messaging.simp.SimpMessagingTemplate -import org.springframework.stereotype.Controller - -@Controller -class EncoderTopic( - @Autowired val template: SimpMessagingTemplate?, -) { - - init { - encoderItems.addListener(object : ObservableMap.Listener { - override fun onMapUpdated(map: Map) { - super.onMapUpdated(map) - pushEncoderQueue() - } - - override fun onPut(key: String, value: WorkOrderItem) { - super.onPut(key, value) - pushEncoderWorkOrder(value) - } - }) - extractItems.addListener(object : ObservableMap.Listener { - override fun onMapUpdated(map: Map) { - super.onMapUpdated(map) - pushExtractorQueue() - } - - override fun onPut(key: String, value: WorkOrderItem) { - super.onPut(key, value) - pushExtractorWorkOrder(value) - } - }) - } - - fun pushEncoderWorkOrder(item: WorkOrderItem) { - template?.convertAndSend("/topic/encoder/workorder", item) - } - - fun pushExtractorWorkOrder(item: WorkOrderItem) { - template?.convertAndSend("/topic/extractor/workorder", item) - } - - @MessageMapping("/encoder/queue") - fun pushEncoderQueue() { - template?.convertAndSend("/topic/encoder/queue", encoderItems.values) - } - - @MessageMapping("/extractor/queue") - fun pushExtractorQueue() { - template?.convertAndSend("/topic/extractor/queue", extractItems.values) - - } - - - - - -} \ No newline at end of file diff --git a/Encode/src/main/resources/application.properties b/Encode/src/main/resources/application.properties deleted file mode 100644 index f0fe28f8..00000000 --- a/Encode/src/main/resources/application.properties +++ /dev/null @@ -1,3 +0,0 @@ -spring.output.ansi.enabled=always -logging.level.org.apache.kafka=WARN -#logging.level.root=DEBUG diff --git a/Encode/src/test/kotlin/no/iktdev/streamit/content/encode/Resources.kt b/Encode/src/test/kotlin/no/iktdev/streamit/content/encode/Resources.kt deleted file mode 100644 index 22bb2314..00000000 --- a/Encode/src/test/kotlin/no/iktdev/streamit/content/encode/Resources.kt +++ /dev/null @@ -1,29 +0,0 @@ -package no.iktdev.streamit.content.encode - -import org.apache.kafka.clients.consumer.ConsumerRecord - -open class Resources { - - fun getText(path: String): String? { - return this.javaClass.classLoader.getResource(path)?.readText() - } - - open class Streams(): Resources() { - fun all(): List { - return listOf( - getSample(0), - getSample(1), - getSample(2), - getSample(3), - getSample(4), - getSample(5), - getSample(6), - ) - } - - fun getSample(number: Int): String { - return getText("streams/sample$number.json")!! - } - } - -} \ No newline at end of file diff --git a/Encode/src/test/kotlin/no/iktdev/streamit/content/encode/progress/DecodedProgressDataDecoderTest.kt b/Encode/src/test/kotlin/no/iktdev/streamit/content/encode/progress/DecodedProgressDataDecoderTest.kt deleted file mode 100644 index e3ee40ce..00000000 --- a/Encode/src/test/kotlin/no/iktdev/streamit/content/encode/progress/DecodedProgressDataDecoderTest.kt +++ /dev/null @@ -1,176 +0,0 @@ -package no.iktdev.streamit.content.encode.progress - -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.content.encode.Resources -import no.iktdev.streamit.content.encode.runner.EncodeDaemon -import no.iktdev.streamit.content.encode.runner.IEncodeListener -import org.assertj.core.api.Assertions.assertThat -import org.junit.BeforeClass -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertDoesNotThrow -import org.mockito.ArgumentMatchers.anyBoolean -import org.mockito.ArgumentMatchers.anyString -import org.mockito.Mockito.* -import java.io.BufferedWriter -import java.io.File -import java.io.FileWriter -import java.util.UUID - -class DecodedProgressDataDecoderTest { - - @Test - fun test() { - val progress = ProgressDecoder(EncodeWork( - workId = UUID.randomUUID().toString(), - collection = "Demo", - inFile = "Demo.mkv", - outFile = "FancyDemo.mp4", - arguments = emptyList() - )) - val lines = text.split("\n") - val cache: MutableList = mutableListOf() - lines.forEach { - cache.add(it) - assertDoesNotThrow { - val progressItem = progress.parseVideoProgress(cache) - progressItem?.progress - } - } - assertThat(lines).isNotEmpty() - } - - - - @Test - fun testCanRead() { - val res = Resources() - val data = res.getText("Output1.txt") ?: "" - assertThat(data).isNotEmpty() - val lines = data.split("\n").map { it.trim() } - assertThat(lines).isNotEmpty() - - val encodeWork = EncodeWork( - workId = UUID.randomUUID().toString(), - collection = "Demo", - inFile = "Demo.mkv", - outFile = "FancyDemo.mp4", - arguments = emptyList() - ) - val decoder = ProgressDecoder(encodeWork) - lines.forEach { decoder.setDuration(it) } - assertThat(decoder.duration).isNotNull() - val produced = mutableListOf() - - val tempFile = File.createTempFile("test", ".log") - - val encoder = EncodeDaemon(UUID.randomUUID().toString(), encodeWork, object : IEncodeListener { - override fun onStarted(referenceId: String, work: EncodeWork) { - } - override fun onError(referenceId: String, work: EncodeWork, code: Int) { - } - override fun onProgress(referenceId: String, work: EncodeWork, progress: Progress) { - produced.add(progress) - } - override fun onEnded(referenceId: String, work: EncodeWork) { - } - - }, tempFile) - - - lines.forEach { - encoder.onOutputChanged(it) - } - assertThat(produced).isNotEmpty() - - - } - - - @Test - fun testThatProgressIsCalculated() { - val encodeWork = EncodeWork( - workId = UUID.randomUUID().toString(), - collection = "Demo", - inFile = "Demo.mkv", - outFile = "FancyDemo.mp4", - arguments = emptyList() - ) - val decoder = ProgressDecoder(encodeWork) - decoder.setDuration("Duration: 01:48:54.82,") - assertThat(decoder.duration).isNotNull() - val decodedProgressData = DecodedProgressData( - frame = null, - fps = null, - stream_0_0_q = null, - bitrate = null, - total_size = null, - out_time_ms = null, - out_time_us = null, - out_time = "01:48:54.82", - dup_frames = null, - drop_frames = null, - speed = 1.0, - progress = "Continue" - ) - val progress = decoder.getProgress(decodedProgressData) - assertThat(progress.progress).isGreaterThanOrEqualTo(99) - } - - @Test - fun testThatProgressIsNotNone() { - val encodeWork = EncodeWork( - workId = UUID.randomUUID().toString(), - collection = "Demo", - inFile = "Demo.mkv", - outFile = "FancyDemo.mp4", - arguments = emptyList() - ) - val decoder = ProgressDecoder(encodeWork) - decoder.setDuration("Duration: 01:48:54.82,") - assertThat(decoder.duration).isNotNull() - val decodedProgressData = DecodedProgressData( - frame = null, - fps = null, - stream_0_0_q = null, - bitrate = null, - total_size = null, - out_time_ms = null, - out_time_us = null, - out_time = "01:00:50.174667", - dup_frames = null, - drop_frames = null, - speed = 1.0, - progress = "Continue" - ) - val progress = decoder.getProgress(decodedProgressData) - assertThat(progress.progress).isGreaterThanOrEqualTo(1) - } - - val text = """ - frame=16811 fps= 88 q=40.0 size= 9984kB time=00:x01:10.79 bitrate=1155.3kbits/s speed=3.71x - fps=88.03 - stream_0_0_q=40.0 - bitrate=1155.3kbits/s - total_size=10223752 - out_time_us=70798005 - out_time_ms=70798005 - out_time=00:01:10.798005 - dup_frames=0 - drop_frames=0 - speed=3.71x - progress=continue - frame= 1710 fps= 84 q=-1.0 Lsize= 12124kB time=00:01:11.91 bitrate=1381.2kbits/s speed=3.53x - frame=1710 - fps=84.01 - stream_0_0_q=-1.0 - bitrate=1381.2kbits/s - total_size=12415473 - out_time_us=71910998 - out_time_ms=71910998 - out_time=00:01:11.910998 - dup_frames=0 - drop_frames=0 - speed=3.53x - progress=end - """.trimIndent() -} \ No newline at end of file diff --git a/Encode/src/test/resources/Output1.txt b/Encode/src/test/resources/Output1.txt deleted file mode 100644 index 6b00ec27..00000000 --- a/Encode/src/test/resources/Output1.txt +++ /dev/null @@ -1,389 +0,0 @@ -Guessed Channel Layout for Input Stream #0.1 : 5.1 -Input #0, matroska,webm, from '/src/input/DemoFile.mkv': - Metadata: - CREATION_TIME : 2019-06-15T08:06:07Z - ENCODER : Lavf57.7.2 - Duration: 01:48:54.82, start: 0.000000, bitrate: 2709 kb/s - Chapter #0:0: start 0.000000, end 328.537000 - Metadata: - title : 00:00:00.000 - Chapter #0:1: start 328.537000, end 419.044000 - Metadata: - title : 00:05:28.537 - Chapter #0:2: start 419.044000, end 916.874000 - Metadata: - title : 00:06:59.044 - Chapter #0:3: start 916.874000, end 1309.433000 - Metadata: - title : 00:15:16.749 - Chapter #0:4: start 1309.433000, end 1399.023000 - Metadata: - title : 00:21:49.391 - Chapter #0:5: start 1399.023000, end 1508.924000 - Metadata: - title : 00:23:19.023 - Chapter #0:6: start 1508.924000, end 1767.099000 - Metadata: - title : 00:25:08.924 - Chapter #0:7: start 1767.099000, end 1975.474000 - Metadata: - title : 00:29:27.099 - Chapter #0:8: start 1975.474000, end 2301.466000 - Metadata: - title : 00:32:55.473 - Chapter #0:9: start 2301.466000, end 2498.246000 - Metadata: - title : 00:38:21.466 - Chapter #0:10: start 2498.246000, end 2622.036000 - Metadata: - title : 00:41:38.246 - Chapter #0:11: start 2622.036000, end 2925.172000 - Metadata: - title : 00:43:42.036 - Chapter #0:12: start 2925.172000, end 3183.472000 - Metadata: - title : 00:48:45.172 - Chapter #0:13: start 3183.472000, end 3467.172000 - Metadata: - title : 00:53:03.472 - Chapter #0:14: start 3467.172000, end 3684.472000 - Metadata: - title : 00:57:47.172 - Chapter #0:15: start 3684.472000, end 3885.840000 - Metadata: - title : 01:01:24.472 - Chapter #0:16: start 3885.840000, end 4063.059000 - Metadata: - title : 01:04:45.840 - Chapter #0:17: start 4063.059000, end 4275.605000 - Metadata: - title : 01:07:43.059 - Chapter #0:18: start 4275.605000, end 4434.263000 - Metadata: - title : 01:11:15.605 - Chapter #0:19: start 4434.263000, end 4709.205000 - Metadata: - title : 01:13:54.263 - Chapter #0:20: start 4709.205000, end 4900.020000 - Metadata: - title : 01:18:29.204 - Chapter #0:21: start 4900.020000, end 5081.201000 - Metadata: - title : 01:21:40.020 - Chapter #0:22: start 5081.201000, end 5211.123000 - Metadata: - title : 01:24:41.201 - Chapter #0:23: start 5211.123000, end 5359.938000 - Metadata: - title : 01:26:51.123 - Chapter #0:24: start 5359.938000, end 5833.786000 - Metadata: - title : 01:29:19.938 - Chapter #0:25: start 5833.786000, end 5953.865000 - Metadata: - title : 01:37:13.786 - Chapter #0:26: start 5953.865000, end 6229.432000 - Metadata: - title : 01:39:13.865 - Chapter #0:27: start 6229.432000, end 6534.779000 - Metadata: - title : 01:43:49.181 - Stream #0:0: Video: h264 (High), yuv420p(tv, bt709, progressive), 1920x1080 [SAR 1:1 DAR 16:9], 23.98 fps, 23.98 tbr, 1k tbn, 47.95 tbc (default) - Stream #0:1(eng): Audio: ac3, 48000 Hz, 5.1, fltp (default) - Metadata: - title : Surround - Stream #0:2(jpn): Audio: ac3, 48000 Hz, 5.1(side), fltp, 640 kb/s - Metadata: - title : Surround - Stream #0:3(eng): Subtitle: ass (default) (forced) - Stream #0:4(eng): Subtitle: ass -Stream mapping: - Stream #0:0 -> #0:0 (h264 (native) -> hevc (libx265)) - Stream #0:2 -> #0:1 (ac3 (native) -> eac3 (native)) -x265 [info]: HEVC encoder version 3.4 -x265 [info]: build info [Linux][GCC 9.3.0][64 bit] 8bit+10bit+12bit -x265 [info]: using cpu capabilities: MMX2 SSE2Fast LZCNT SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2 -x265 [info]: Main profile, Level-4 (Main tier) -x265 [info]: Thread pool created using 12 threads -x265 [info]: Slices : 1 -x265 [info]: frame threads / pool features : 3 / wpp(17 rows) -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -set_mempolicy: Operation not permitted -x265 [info]: Coding QT: max CU size, min CU size : 64 / 8 -x265 [info]: Residual QT: max TU size, max depth : 32 / 1 inter / 1 intra -x265 [info]: ME / range / subpel / merge : hex / 57 / 2 / 3 -x265 [info]: Keyframe min / max / scenecut / bias : 23 / 250 / 40 / 5.00 -x265 [info]: Lookahead / bframes / badapt : 20 / 4 / 2 -x265 [info]: b-pyramid / weightp / weightb : 1 / 1 / 0 -x265 [info]: References / ref-limit cu / depth : 3 / off / on -x265 [info]: AQ: mode / str / qg-size / cu-tree : 2 / 1.0 / 32 / 1 -x265 [info]: Rate Control / qCompress : CRF-16.0 / 0.60 -x265 [info]: tools: rd=3 psy-rd=2.00 early-skip rskip mode=1 signhide tmvp -x265 [info]: tools: b-intra strong-intra-smoothing lslices=6 deblock sao -Output #0, mp4, to '/src/output/Demo/Demo.mp4': - Metadata: - encoder : Lavf58.45.100 - Chapter #0:0: start 0.000000, end 328.537000 - Metadata: - title : 00:00:00.000 - Chapter #0:1: start 328.537000, end 419.044000 - Metadata: - title : 00:05:28.537 - Chapter #0:2: start 419.044000, end 916.874000 - Metadata: - title : 00:06:59.044 - Chapter #0:3: start 916.874000, end 1309.433000 - Metadata: - title : 00:15:16.749 - Chapter #0:4: start 1309.433000, end 1399.023000 - Metadata: - title : 00:21:49.391 - Chapter #0:5: start 1399.023000, end 1508.924000 - Metadata: - title : 00:23:19.023 - Chapter #0:6: start 1508.924000, end 1767.099000 - Metadata: - title : 00:25:08.924 - Chapter #0:7: start 1767.099000, end 1975.474000 - Metadata: - title : 00:29:27.099 - Chapter #0:8: start 1975.474000, end 2301.466000 - Metadata: - title : 00:32:55.473 - Chapter #0:9: start 2301.466000, end 2498.246000 - Metadata: - title : 00:38:21.466 - Chapter #0:10: start 2498.246000, end 2622.036000 - Metadata: - title : 00:41:38.246 - Chapter #0:11: start 2622.036000, end 2925.172000 - Metadata: - title : 00:43:42.036 - Chapter #0:12: start 2925.172000, end 3183.472000 - Metadata: - title : 00:48:45.172 - Chapter #0:13: start 3183.472000, end 3467.172000 - Metadata: - title : 00:53:03.472 - Chapter #0:14: start 3467.172000, end 3684.472000 - Metadata: - title : 00:57:47.172 - Chapter #0:15: start 3684.472000, end 3885.840000 - Metadata: - title : 01:01:24.472 - Chapter #0:16: start 3885.840000, end 4063.059000 - Metadata: - title : 01:04:45.840 - Chapter #0:17: start 4063.059000, end 4275.605000 - Metadata: - title : 01:07:43.059 - Chapter #0:18: start 4275.605000, end 4434.263000 - Metadata: - title : 01:11:15.605 - Chapter #0:19: start 4434.263000, end 4709.205000 - Metadata: - title : 01:13:54.263 - Chapter #0:20: start 4709.205000, end 4900.020000 - Metadata: - title : 01:18:29.204 - Chapter #0:21: start 4900.020000, end 5081.201000 - Metadata: - title : 01:21:40.020 - Chapter #0:22: start 5081.201000, end 5211.123000 - Metadata: - title : 01:24:41.201 - Chapter #0:23: start 5211.123000, end 5359.938000 - Metadata: - title : 01:26:51.123 - Chapter #0:24: start 5359.938000, end 5833.786000 - Metadata: - title : 01:29:19.938 - Chapter #0:25: start 5833.786000, end 5953.865000 - Metadata: - title : 01:37:13.786 - Chapter #0:26: start 5953.865000, end 6229.432000 - Metadata: - title : 01:39:13.865 - Chapter #0:27: start 6229.432000, end 6534.779000 - Metadata: - title : 01:43:49.181 - Stream #0:0: Video: hevc (libx265) (hev1 / 0x31766568), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], q=-1--1, 23.98 fps, 24k tbn, 23.98 tbc (default) - Metadata: - encoder : Lavc58.91.100 libx265 - Side data: - cpb: bitrate max/min/avg: 0/0/0 buffer size: 0 vbv_delay: N/A - Stream #0:1(jpn): Audio: eac3 (ec-3 / 0x332D6365), 48000 Hz, 5.1(side), fltp, 448 kb/s - Metadata: - title : Surround - encoder : Lavc58.91.100 eac3 -frame= 49 fps=0.0 q=24.0 size= 1kB time=00:00:02.52 bitrate= 2.4kbits/s speed=4.85x -frame=49 -fps=0.00 -stream_0_0_q=24.0 -bitrate= 2.4kbits/s -total_size=772 -out_time_us=2526667 -out_time_ms=2526667 -out_time=00:00:02.526667 -dup_frames=0 -drop_frames=0 -speed=4.85x -progress=continue -frame= 87 fps= 84 q=16.7 size= 1kB time=00:00:04.09 bitrate= 1.5kbits/s speed=3.96x -frame=87 -fps=84.21 -stream_0_0_q=16.7 -bitrate= 1.5kbits/s -total_size=772 -out_time_us=4094667 -out_time_ms=4094667 -out_time=00:00:04.094667 -dup_frames=0 -drop_frames=0 -speed=3.96x -progress=continue -frame= 115 fps= 75 q=22.4 size= 257kB time=00:00:05.27 bitrate= 398.5kbits/s speed=3.44x -frame=115 -fps=74.95 -stream_0_0_q=22.4 -bitrate= 398.5kbits/s -total_size=262916 -out_time_us=5278667 -out_time_ms=5278667 -out_time=00:00:05.278667 -dup_frames=0 -drop_frames=0 -speed=3.44x -progress=continue -frame= 146 fps= 72 q=22.6 size= 257kB time=00:00:06.55 bitrate= 320.7kbits/s speed=3.22x -frame=146 -fps=71.64 -stream_0_0_q=22.6 -bitrate= 320.7kbits/s -total_size=262916 -out_time_us=6558667 -out_time_ms=6558667 -out_time=00:00:06.558667 -dup_frames=0 -drop_frames=0 -speed=3.22x -progress=continue -frame= 175 fps= 69 q=20.5 size= 513kB time=00:00:07.77 bitrate= 540.3kbits/s speed=3.06x -frame=175 -fps=68.82 -stream_0_0_q=20.5 -bitrate= 540.3kbits/s -total_size=525060 -out_time_us=7774667 -out_time_ms=7774667 -out_time=00:00:07.774667 -dup_frames=0 -drop_frames=0 -speed=3.06x -progress=continue -frame= 204 fps= 67 q=21.1 size= 769kB time=00:00:08.99 bitrate= 700.5kbits/s speed=2.94x -frame=204 -fps=66.66 -stream_0_0_q=21.1 -bitrate= 700.5kbits/s -total_size=787204 -out_time_us=8990667 -out_time_ms=8990667 -out_time=00:00:08.990667 -dup_frames=0 -drop_frames=0 -speed=2.94x -progress=continue -frame= 231 fps= 65 q=20.5 size= 1025kB time=00:00:10.11 bitrate= 830.3kbits/s speed=2.83x -frame=231 -fps=64.66 -stream_0_0_q=20.5 -bitrate= 830.3kbits/s -total_size=1049348 -out_time_us=10110667 -out_time_ms=10110667 -out_time=00:00:10.110667 -dup_frames=0 -drop_frames=0 -speed=2.83x -progress=continue -frame= 268 fps= 65 q=20.7 size= 1025kB time=00:00:11.64 bitrate= 720.8kbits/s speed=2.84x -frame=268 -fps=65.29 -stream_0_0_q=20.7 -bitrate= 720.8kbits/s -total_size=1049348 -out_time_us=11646667 -out_time_ms=11646667 -out_time=00:00:11.646667 -dup_frames=0 -drop_frames=0 -speed=2.84x -progress=continue -frame= 312 fps= 68 q=21.0 size= 1281kB time=00:00:13.47 bitrate= 778.9kbits/s speed=2.92x -frame=312 -fps=67.67 -stream_0_0_q=21.0 -bitrate= 778.9kbits/s -total_size=1311492 -out_time_us=13470667 -out_time_ms=13470667 -out_time=00:00:13.470667 -dup_frames=0 -drop_frames=0 -speed=2.92x -progress=continue -frame= 353 fps= 69 q=19.9 size= 1281kB time=00:00:15.19 bitrate= 690.3kbits/s speed=2.97x -frame=353 -fps=68.97 -stream_0_0_q=19.9 -bitrate= 690.3kbits/s -total_size=1311492 -out_time_us=15198667 -out_time_ms=15198667 -out_time=00:00:15.198667 -dup_frames=0 -drop_frames=0 -speed=2.97x -progress=continue -frame= 372 fps= 66 q=17.9 size= 1537kB time=00:00:15.99 bitrate= 786.9kbits/s speed=2.84x -frame=372 -fps=66.01 -stream_0_0_q=17.9 -bitrate= 786.9kbits/s -total_size=1573636 -out_time_us=15998667 -out_time_ms=15998667 -out_time=00:00:15.998667 -dup_frames=0 -drop_frames=0 -speed=2.84x -progress=continue \ No newline at end of file diff --git a/Reader/.gitignore b/Reader/.gitignore deleted file mode 100644 index b63da455..00000000 --- a/Reader/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -.gradle -build/ -!gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/Reader/Dockerfile b/Reader/Dockerfile deleted file mode 100644 index f244074f..00000000 --- a/Reader/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM bskjon/debian-azuljava17-ffmpeg:latest -EXPOSE 8080 - -COPY ./build/libs/reader.jar /usr/share/app/app.jar \ No newline at end of file diff --git a/Reader/build.gradle.kts b/Reader/build.gradle.kts deleted file mode 100644 index f33f789f..00000000 --- a/Reader/build.gradle.kts +++ /dev/null @@ -1,79 +0,0 @@ -import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.util.archivesName - -plugins { - kotlin("jvm") version "1.8.21" - id("org.springframework.boot") version "2.5.5" - id("io.spring.dependency-management") version "1.0.11.RELEASE" - kotlin("plugin.spring") version "1.5.31" -} - -archivesName.set("reader.jar") -group = "no.iktdev.streamit.content" -version = "1.0-SNAPSHOT" - -repositories { - mavenCentral() - maven("https://jitpack.io") - maven { - url = uri("https://reposilite.iktdev.no/releases") - } - maven { - url = uri("https://reposilite.iktdev.no/snapshots") - } -} - -val exposedVersion = "0.38.2" -dependencies { - implementation("no.iktdev.streamit.library:streamit-library-kafka:0.0.2-alpha84") - implementation("no.iktdev:exfl:0.0.13-SNAPSHOT") - - implementation("no.iktdev.streamit.library:streamit-library-db:0.0.6-alpha14") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") - - - implementation("org.jetbrains.exposed:exposed-core:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") - implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion") - implementation ("mysql:mysql-connector-java:8.0.29") - - implementation("com.github.pgreze:kotlin-process:1.3.1") - implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") - implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") - - implementation("com.google.code.gson:gson:2.8.9") - implementation("org.json:json:20210307") - - implementation("org.springframework.boot:spring-boot-starter-web") - implementation("org.springframework.boot:spring-boot-starter:2.7.0") - implementation("org.springframework.kafka:spring-kafka:2.8.5") - implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") - - - implementation(project(":CommonCode")) - - testImplementation("junit:junit:4.13.2") - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") - testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1") - testImplementation("org.assertj:assertj-core:3.4.1") - testImplementation("org.mockito:mockito-core:3.+") - - - -} - - -tasks.test { - useJUnitPlatform() -} - -tasks.bootJar { - archiveFileName.set("reader.jar") - launchScript() -} - -tasks.jar { - archivesName.set("reader.jar") - archiveBaseName.set("reader") -} \ No newline at end of file diff --git a/Reader/gradle/wrapper/gradle-wrapper.jar b/Reader/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 249e5832..00000000 Binary files a/Reader/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/Reader/gradle/wrapper/gradle-wrapper.properties b/Reader/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 4db8978e..00000000 --- a/Reader/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,6 +0,0 @@ -#Tue Jul 11 02:16:45 CEST 2023 -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/Reader/gradlew b/Reader/gradlew deleted file mode 100644 index 1b6c7873..00000000 --- a/Reader/gradlew +++ /dev/null @@ -1,234 +0,0 @@ -#!/bin/sh - -# -# Copyright © 2015-2021 the original authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -# -# Gradle start up script for POSIX generated by Gradle. -# -# Important for running: -# -# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is -# noncompliant, but you have some other compliant shell such as ksh or -# bash, then to run this script, type that shell name before the whole -# command line, like: -# -# ksh Gradle -# -# Busybox and similar reduced shells will NOT work, because this script -# requires all of these POSIX shell features: -# * functions; -# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», -# «${var#prefix}», «${var%suffix}», and «$( cmd )»; -# * compound commands having a testable exit status, especially «case»; -# * various built-in commands including «command», «set», and «ulimit». -# -# Important for patching: -# -# (2) This script targets any POSIX shell, so it avoids extensions provided -# by Bash, Ksh, etc; in particular arrays are avoided. -# -# The "traditional" practice of packing multiple parameters into a -# space-separated string is a well documented source of bugs and security -# problems, so this is (mostly) avoided, by progressively accumulating -# options in "$@", and eventually passing that to Java. -# -# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, -# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; -# see the in-line comments for details. -# -# There are tweaks for specific operating systems such as AIX, CygWin, -# Darwin, MinGW, and NonStop. -# -# (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt -# within the Gradle project. -# -# You can find Gradle at https://github.com/gradle/gradle/. -# -############################################################################## - -# Attempt to set APP_HOME - -# Resolve links: $0 may be a link -app_path=$0 - -# Need this for daisy-chained symlinks. -while - APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path - [ -h "$app_path" ] -do - ls=$( ls -ld "$app_path" ) - link=${ls#*' -> '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" -APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ - "$@" - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/Reader/gradlew.bat b/Reader/gradlew.bat deleted file mode 100644 index 107acd32..00000000 --- a/Reader/gradlew.bat +++ /dev/null @@ -1,89 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/Reader/settings.gradle.kts b/Reader/settings.gradle.kts deleted file mode 100644 index ffe04905..00000000 --- a/Reader/settings.gradle.kts +++ /dev/null @@ -1,4 +0,0 @@ -rootProject.name = "Reader" - -include(":CommonCode") -project(":CommonCode").projectDir = File("../CommonCode") \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/ReaderApplication.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/ReaderApplication.kt deleted file mode 100644 index 98de0060..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/ReaderApplication.kt +++ /dev/null @@ -1,66 +0,0 @@ -package no.iktdev.streamit.content.reader - -import kotlinx.coroutines.launch -import mu.KotlinLogging -import no.iktdev.exfl.coroutines.Coroutines -import no.iktdev.exfl.observable.Observables -import no.iktdev.streamit.content.reader.analyzer.encoding.helpers.PreferenceReader -import no.iktdev.streamit.library.db.datasource.MySqlDataSource -import no.iktdev.streamit.library.db.tables.* -import no.iktdev.streamit.library.db.tables.helper.cast_errors -import no.iktdev.streamit.library.db.tables.helper.data_audio -import no.iktdev.streamit.library.db.tables.helper.data_video -import org.jetbrains.exposed.sql.SchemaUtils -import org.jetbrains.exposed.sql.transactions.transaction -import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.boot.runApplication -import org.springframework.context.ApplicationContext - -private val logger = KotlinLogging.logger {} - -@SpringBootApplication -class ReaderApplication - -val preference = PreferenceReader().getPreference() -private var context: ApplicationContext? = null - -@Suppress("unused") -fun getContext(): ApplicationContext? { - return context -} -fun main(args: Array) { - Coroutines.addListener(object : Observables.ObservableValue.ValueListener { - override fun onUpdated(value: Throwable) { - logger.error { "Received error: ${value.message}" } - value.cause?.printStackTrace() - } - }) - - - val ds = MySqlDataSource.fromDatabaseEnv().createDatabase() - System.out.println(ds) - - Coroutines.default().launch { - val tables = arrayOf( - catalog, - genre, - movie, - serie, - subtitle, - summary, - users, - progress, - data_audio, - data_video, - cast_errors - ) - transaction { - SchemaUtils.createMissingTablesAndColumns(*tables) - logger.info {"Database transaction completed"} - } - } - - context = runApplication(*args) - -} - diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/ReaderEnv.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/ReaderEnv.kt deleted file mode 100644 index adaf0ff7..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/ReaderEnv.kt +++ /dev/null @@ -1,11 +0,0 @@ -package no.iktdev.streamit.content.reader - -import java.io.File - -class ReaderEnv { - companion object { - val metadataTimeOut: Long = System.getenv("TIMEOUT_READER_WAIT_FOR_METADATA")?.toLongOrNull() ?: 300000 - val ffprobe: String = System.getenv("SUPPORTING_EXECUTABLE_FFPROBE") ?: "ffprobe" - val encodePreference: File = File("/data/config/preference.json") - } -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/ContentDeterminate.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/ContentDeterminate.kt deleted file mode 100644 index 3c0e3cee..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/ContentDeterminate.kt +++ /dev/null @@ -1,122 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.contentDeterminator - -import mu.KotlinLogging -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.common.deserializers.FileResultDeserializer -import no.iktdev.streamit.content.common.deserializers.MetadataResultDeserializer -import no.iktdev.streamit.content.common.dto.ContentOutName -import no.iktdev.streamit.content.common.dto.Metadata -import no.iktdev.streamit.content.common.dto.reader.EpisodeInfo -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.content.common.dto.reader.MovieInfo -import no.iktdev.streamit.content.reader.ReaderEnv -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import no.iktdev.streamit.library.kafka.listener.sequential.ISequentialMessageEvent -import no.iktdev.streamit.library.kafka.listener.sequential.SequentialMessageListener -import org.springframework.stereotype.Service - -private val logger = KotlinLogging.logger {} - -@Service -class ContentDeterminate: DefaultKafkaReader("contentDeterminate"), ISequentialMessageEvent { - - final val mainListener = object : SequentialMessageListener( - topic = CommonConfig.kafkaTopic, - consumer = defaultConsumer, - accept = KafkaEvents.EVENT_READER_RECEIVED_FILE.event, - subAccepts = listOf(KafkaEvents.EVENT_METADATA_OBTAINED.event), - deserializers = loadDeserializers(), - listener = this, - validity = ReaderEnv.metadataTimeOut - ) {} - - init { - mainListener.listen() - } - - - - override fun getRequiredMessages(): List { - return mainListener.subAccepts + listOf(mainListener.accept) - } - - override fun onAllMessagesProcessed(referenceId: String, result: Map) { - logger.info { "All messages are received" } - - val initMessage = result[KafkaEvents.EVENT_READER_RECEIVED_FILE.event] - if (initMessage == null || initMessage.status.statusType != StatusType.SUCCESS) { - produceErrorMessage( - KafkaEvents.EVENT_READER_DETERMINED_FILENAME, - Message(referenceId = referenceId, status = Status(statusType = StatusType.ERROR)), - "Initiator message not found!" - ) - return - } - val fileResult = initMessage.data as FileResult? - if (fileResult == null) { - produceErrorMessage( - KafkaEvents.EVENT_READER_DETERMINED_FILENAME, - initMessage, - "FileResult is either null or not deserializable!" - ) - return - } - - val metadataMessage = result[KafkaEvents.EVENT_METADATA_OBTAINED.event] - val metadata = - if (metadataMessage?.status?.statusType == StatusType.SUCCESS) metadataMessage.data as Metadata? else null - - - // Due to the fact that the sources might say serie, but it is not a serie input we will give serie a try then default to movie - - - val videoInfo = when (metadata?.type) { - "serie" -> { - FileNameDeterminate( - fileResult.title, - fileResult.sanitizedName, - FileNameDeterminate.ContentType.SERIE - ).getDeterminedVideoInfo() - } - - "movie" -> { - FileNameDeterminate( - fileResult.title, - fileResult.sanitizedName, - FileNameDeterminate.ContentType.MOVIE - ).getDeterminedVideoInfo() - } - - else -> null - } ?: FileNameDeterminate(fileResult.title, fileResult.sanitizedName).getDeterminedVideoInfo() - - if (videoInfo == null) { - produceErrorMessage(KafkaEvents.EVENT_READER_DETERMINED_FILENAME, initMessage, "VideoInfo is null.") - return - } - - - - if (videoInfo is EpisodeInfo) { - produceSuccessMessage(KafkaEvents.EVENT_READER_DETERMINED_SERIE, referenceId, videoInfo) - } else if (videoInfo is MovieInfo) { - produceSuccessMessage(KafkaEvents.EVENT_READER_DETERMINED_MOVIE, referenceId, videoInfo) - } - - val out = ContentOutName(videoInfo.fullName) - produceSuccessMessage(KafkaEvents.EVENT_READER_DETERMINED_FILENAME, referenceId, out) - } - - final override fun loadDeserializers(): Map> { - return mutableMapOf( - KafkaEvents.EVENT_READER_RECEIVED_FILE.event to FileResultDeserializer(), - KafkaEvents.EVENT_METADATA_OBTAINED.event to MetadataResultDeserializer() - ) - } - -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/EncodedStreams.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/EncodedStreams.kt deleted file mode 100644 index 6ec0cea1..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/EncodedStreams.kt +++ /dev/null @@ -1,150 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.encoding - -import mu.KotlinLogging -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.common.deserializers.ContentOutNameDeserializer -import no.iktdev.streamit.content.common.deserializers.DeserializerRegistry -import no.iktdev.streamit.content.common.deserializers.FileResultDeserializer -import no.iktdev.streamit.content.common.deserializers.MediaStreamsDeserializer -import no.iktdev.streamit.content.common.dto.ContentOutName -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.content.common.streams.MediaStreams -import no.iktdev.streamit.content.reader.analyzer.encoding.helpers.EncodeArgumentSelector -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.collector.CollectorMessageListener -import no.iktdev.streamit.library.kafka.listener.collector.ICollectedMessagesEvent -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import no.iktdev.streamit.library.kafka.listener.deserializer.deserializeIfSuccessful -import no.iktdev.streamit.library.kafka.listener.sequential.ISequentialMessageEvent -import no.iktdev.streamit.library.kafka.listener.sequential.SequentialMessageListener -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.springframework.stereotype.Service -import java.io.File - -private val logger = KotlinLogging.logger {} - -@Service -class EncodedStreams : DefaultKafkaReader("streamSelector"), ISequentialMessageEvent { - - val listener = object : SequentialMessageListener( - topic = CommonConfig.kafkaTopic, - consumer = defaultConsumer, - accept = KafkaEvents.EVENT_READER_RECEIVED_FILE.event, - subAccepts = listOf( - KafkaEvents.EVENT_READER_DETERMINED_FILENAME.event, - KafkaEvents.EVENT_READER_RECEIVED_STREAMS.event, - ), - listener = this, - deserializers = this.loadDeserializers() - ) {} - - init { - listener.listen() - } - - fun createEncodeWork(referenceId: String, collection: String?, inFile: String?, streams: MediaStreams?, outFileName: String?) { - if (inFile.isNullOrBlank()) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO, referenceId, "No input file received"); return - } - if (streams == null) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO, referenceId, "No input streams received"); return - } - if (outFileName.isNullOrBlank()) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO, referenceId, "No output file name received!"); return - } - if (collection.isNullOrBlank()) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO, referenceId, "No collection provided for file!"); return - } - - val encodeInformation = - EncodeArgumentSelector(collection = collection, inputFile = inFile, streams = streams, outFileName = outFileName) - - val videoInstructions = encodeInformation.getVideoAndAudioArguments() - if (videoInstructions == null) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO, referenceId, "Failed to generate Video Arguments Bundle") - return - } - produceSuccessMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_VIDEO, referenceId, videoInstructions) - - } - - fun createExtractWork(referenceId: String, collection: String?, inFile: String?, streams: MediaStreams?, outFileName: String?) { - if (inFile.isNullOrBlank()) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE, referenceId, "No input file received"); return - } - if (streams == null) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE, referenceId, "No input streams received"); return - } - if (outFileName.isNullOrBlank()) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE, referenceId, "No output file name received!"); return - } - if (collection.isNullOrBlank()) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE, referenceId, "No collection provided for file!"); return - } - - val argsSelector = EncodeArgumentSelector(collection = collection, inputFile = inFile, streams = streams, outFileName = outFileName) - val items = argsSelector.getSubtitleArguments() - if (argsSelector == null || items.isEmpty()) { - produceErrorMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE, referenceId, "Failed to generate Subtitle Arguments Bundle") - return - } - - argsSelector.getSubtitleArguments().forEach { - produceMessage(KafkaEvents.EVENT_READER_ENCODE_GENERATED_SUBTITLE, Message(referenceId, Status(StatusType.SUCCESS)), it) - - } - - } - - - final override fun loadDeserializers(): Map> { - return DeserializerRegistry.getEventToDeserializer( - KafkaEvents.EVENT_READER_RECEIVED_FILE, - KafkaEvents.EVENT_READER_RECEIVED_STREAMS, - KafkaEvents.EVENT_READER_DETERMINED_FILENAME - ) - } - - override fun getRequiredMessages(): List { - return listener.subAccepts + listOf(listener.accept) - } - - override fun onAllMessagesProcessed(referenceId: String, result: Map) { - logger.info { "Collection received" } - if (result.keys.isEmpty()) { - logger.error { "\nConsumer $subId collected: is null or empty!" } - } else { - logger.info { "\nConsumer $subId collected:\n ${result.keys.joinToString("\n\t")}" } - } - - val outFileNameWithoutExtension: String? = if (getFileName(result) != null) { - getFileName(result)?.baseName - } else { - logger.info { "Getting filename from ${KafkaEvents.EVENT_READER_DETERMINED_FILENAME.event} resulted in null. Falling back to sanitized name" } - getFileResult(result)?.sanitizedName - } - - val fileResult = getFileResult(result) - createEncodeWork(referenceId, fileResult?.title, fileResult?.file, getStreams(result), outFileNameWithoutExtension) - createExtractWork(referenceId, fileResult?.title, fileResult?.file, getStreams(result), outFileNameWithoutExtension) - } - - fun getFileResult(result: Map): FileResult? { - val record = result[KafkaEvents.EVENT_READER_RECEIVED_FILE.event] ?: return null - return FileResultDeserializer().deserializeIfSuccessful(record) - } - - fun getFileName(result: Map): ContentOutName? { - val record = result[KafkaEvents.EVENT_READER_DETERMINED_FILENAME.event] ?: return null - return ContentOutNameDeserializer().deserializeIfSuccessful(record) - } - - fun getStreams(result: Map): MediaStreams? { - val record = result[KafkaEvents.EVENT_READER_RECEIVED_STREAMS.event] ?: return null - return MediaStreamsDeserializer().deserializeIfSuccessful(record) - } -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/ResultCollection.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/ResultCollection.kt deleted file mode 100644 index 2ad56484..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/ResultCollection.kt +++ /dev/null @@ -1,21 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.encoding - -import no.iktdev.streamit.content.common.deserializers.ContentOutNameDeserializer -import no.iktdev.streamit.content.common.deserializers.FileResultDeserializer -import no.iktdev.streamit.content.common.deserializers.MediaStreamsDeserializer -import no.iktdev.streamit.content.common.dto.ContentOutName -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.content.common.streams.MediaStreams -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.collector.DefaultEventCollection -import no.iktdev.streamit.library.kafka.listener.deserializer.deserializeIfSuccessful -import org.apache.kafka.clients.consumer.ConsumerRecord - -class ResultCollection: DefaultEventCollection() { - - fun getFirstOrNull(events: KafkaEvents): ConsumerRecord? { - return getRecords().firstOrNull { it.key() == events.event } - } - -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/AudioEncodeArguments.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/AudioEncodeArguments.kt deleted file mode 100644 index bf38c331..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/AudioEncodeArguments.kt +++ /dev/null @@ -1,24 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.encoding.dto - -import no.iktdev.streamit.content.common.streams.AudioStream -import no.iktdev.streamit.content.reader.preference - -class AudioEncodeArguments(val audio: AudioStream, val index: Int) { - - fun isAudioCodecEqual() = audio.codec_name.lowercase() == preference.audio.codec.lowercase() - - fun shouldUseEAC3(): Boolean { - return (preference.audio.defaultToEAC3OnSurroundDetected && audio.channels > 2 && audio.codec_name.lowercase() != "eac3") - } - - fun getAudioArguments(): MutableList { - val result = mutableListOf() - if (shouldUseEAC3()) { - result.addAll(listOf("-c:a", "eac3")) - } else if (!isAudioCodecEqual()) { - result.addAll(listOf("-c:a", preference.audio.codec)) - } else result.addAll(listOf("-acodec", "copy")) - result.addAll(listOf("-map", "0:a:${index}")) - return result - } -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/SubtitleEncodeArguments.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/SubtitleEncodeArguments.kt deleted file mode 100644 index 86c5b3d6..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/SubtitleEncodeArguments.kt +++ /dev/null @@ -1,14 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.encoding.dto - -import no.iktdev.streamit.content.common.streams.SubtitleStream - -class SubtitleEncodeArguments(val subtitle: SubtitleStream, val index: Int) { - - fun getSubtitleArguments(): List { - val result = mutableListOf() - result.addAll(listOf("-c:s", "copy")) - result.addAll(listOf("-map", "0:s:$index")) - return result - } - -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/VideoEncodeArguments.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/VideoEncodeArguments.kt deleted file mode 100644 index a216eb9a..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/dto/VideoEncodeArguments.kt +++ /dev/null @@ -1,36 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.encoding.dto - -import no.iktdev.streamit.content.common.streams.VideoStream -import no.iktdev.streamit.content.reader.preference - -class VideoEncodeArguments(val video: VideoStream, val index: Int) { - - fun isVideoCodecEqual() = getCodec(video.codec_name) == getCodec(preference.video.codec.lowercase()) - - - fun getVideoArguments(): List { - val result = mutableListOf() - if (isVideoCodecEqual()) result.addAll(listOf( - "-vcodec", "copy" - )) else { - result.addAll(listOf("-c:v", getCodec(preference.video.codec.lowercase()))) - result.addAll(listOf("-crf", preference.video.threshold.toString())) - } - if (preference.video.pixelFormatPassthrough.none { it == video.pix_fmt }) { - result.addAll(listOf("-pix_fmt", preference.video.pixelFormat)) - } - result.addAll(listOf("-map", "0:v:${index}")) - return result - } - - - protected fun getCodec(name: String): String { - return when(name) { - "hevc", "hevec", "h265", "h.265", "libx265" - -> "libx265" - "h.264", "h264", "libx264" - -> "libx264" - else -> name - } - } -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/helpers/EncodeArgumentSelector.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/helpers/EncodeArgumentSelector.kt deleted file mode 100644 index d215dd59..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/helpers/EncodeArgumentSelector.kt +++ /dev/null @@ -1,88 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.encoding.helpers - -import no.iktdev.exfl.using -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.content.common.streams.* -import no.iktdev.streamit.content.reader.analyzer.encoding.dto.AudioEncodeArguments -import no.iktdev.streamit.content.reader.analyzer.encoding.dto.SubtitleEncodeArguments -import no.iktdev.streamit.content.reader.analyzer.encoding.dto.VideoEncodeArguments -import no.iktdev.streamit.content.reader.preference - -class EncodeArgumentSelector(val collection: String, val inputFile: String, val streams: MediaStreams, val outFileName: String) { - var defaultSelectedVideo: VideoStream? = defaultSelectedVideo() - var defaultSelectedAudio: AudioStream? = defaultSelectedAudio() - - private fun obtainAudioStreams() = streams.streams.filterIsInstance() - private fun obtainVideoStreams() = streams.streams.filterIsInstance() - - - private fun defaultSelectedVideo(): VideoStream? { - return obtainVideoStreams().filter { (it.duration_ts ?: 0) > 0 }.maxByOrNull { it.duration_ts!! } ?: obtainVideoStreams().minByOrNull { it.index } - } - - private fun defaultSelectedAudio(): AudioStream? { - return obtainAudioStreams().filter { (it.duration_ts ?: 0) > 0 }.maxByOrNull { it.duration_ts!! } ?: obtainAudioStreams().minByOrNull { it.index } - } - - /** - * @return VideoStream based on preference or defaultSelectedVideo - */ - /*private fun getSelectedVideoBasedOnPreference(): VideoStream { - val - }*/ - - /** - * @return AudioStrem based on preference or defaultSelectedAudio - */ - private fun getSelectedAudioBasedOnPreference(): AudioStream? { - val languageFiltered = obtainAudioStreams().filter { it.tags.language == preference.audio.language } - val channeledAndCodec = languageFiltered.find { it.channels >= (preference.audio.channels ?: 2) && it.codec_name == preference.audio.codec.lowercase() } - return channeledAndCodec ?: return languageFiltered.minByOrNull { it.index } ?: defaultSelectedAudio - } - - - fun getVideoAndAudioArguments(): EncodeWork? { - val selectedVideo = defaultSelectedVideo - val selectedAudio = getSelectedAudioBasedOnPreference() ?: defaultSelectedAudio - return if (selectedVideo == null || selectedAudio == null) return null - else { - val outFileName = "$outFileName.mp4" - val outFile = CommonConfig.outgoingContent.using(collection, outFileName) - val audioIndex = obtainAudioStreams().indexOf(selectedAudio) - val videoIndex = obtainVideoStreams().indexOf(selectedVideo) - EncodeWork( - collection = collection, - inFile = inputFile, - arguments = VideoEncodeArguments(selectedVideo, videoIndex).getVideoArguments() + - AudioEncodeArguments(selectedAudio, audioIndex).getAudioArguments(), - outFile = outFile.absolutePath - ) - } - } - - fun getSubtitleArguments(): List { - val availableSubtitleStreams = streams.streams.filterIsInstance() - val subtitleStreams = SubtitleStreamSelector(availableSubtitleStreams) - - val conversionCandidates = subtitleStreams.getCandidateForConversion() - - return subtitleStreams.getDesiredStreams().map { - val args = SubtitleEncodeArguments(it, availableSubtitleStreams.indexOf(it)) - val language = it.tags.language ?: "eng" - val outFileName = "$outFileName.${subtitleStreams.getFormatToCodec(it.codec_name)}" - val outFile = CommonConfig.outgoingContent.using(collection, "sub", language, outFileName) - - ExtractWork( - collection = collection, - language = language, - inFile = inputFile, - outFile = outFile.absolutePath, - arguments = args.getSubtitleArguments(), - produceConvertEvent = conversionCandidates.contains(it) - ) - } - - } -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/helpers/EncodingPreference.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/helpers/EncodingPreference.kt deleted file mode 100644 index 4c9d8e8d..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/encoding/helpers/EncodingPreference.kt +++ /dev/null @@ -1,85 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.encoding.helpers - -import com.google.gson.Gson -import no.iktdev.streamit.content.reader.ReaderEnv -import org.slf4j.LoggerFactory - -data class EncodingPreference( - val video: VideoPreference, - val audio: AudioPreference -) - -data class VideoPreference( - val codec: String = "h264", - val pixelFormat: String = "yuv420p", - val pixelFormatPassthrough: List = listOf("yuv420p", "yuv420p10le"), - val threshold: Int = 16 -) - -data class AudioPreference( - val codec: String = "aac", - val sample_rate: Int? = null, - val channels: Int? = null, - val language: String = "eng", //ISO3 format - val preserveChannels: Boolean = true, - val defaultToEAC3OnSurroundDetected: Boolean = true, - val forceStereo: Boolean = false -) - - -class PreferenceReader { - fun getPreference(): EncodingPreference { - val defaultPreference = EncodingPreference(video = VideoPreference(), audio = AudioPreference()) - val preferenceText = readPreference() ?: return defaultPreference - val configured = deserialize(preferenceText) - - printConfiguration("Audio", "Codec", configured?.audio?.codec, defaultPreference.audio.codec) - printConfiguration("Audio", "Language", configured?.audio?.language, defaultPreference.audio.language) - printConfiguration("Audio", "Channels", configured?.audio?.channels.toString(), defaultPreference.audio.channels.toString()) - printConfiguration("Audio", "Sample rate", configured?.audio?.sample_rate.toString(), defaultPreference.audio.sample_rate.toString()) - printConfiguration("Audio", "Override to EAC3 for surround", configured?.audio?.defaultToEAC3OnSurroundDetected.toString(), defaultPreference.audio.defaultToEAC3OnSurroundDetected.toString()) - - - printConfiguration("Video", "Codec", configured?.video?.codec, defaultPreference.video.codec) - printConfiguration("Video", "Pixel format", configured?.video?.pixelFormat, defaultPreference.video.pixelFormat) - printConfiguration("Video", "Threshold", configured?.video?.threshold.toString(), defaultPreference.video.threshold.toString()) - - - return configured ?: defaultPreference - } - - fun printConfiguration(sourceType: String, key: String, value: String?, default: String?) { - val usedValue = if (!value.isNullOrEmpty()) value else if (!default.isNullOrEmpty()) "$default (default)" else "no changes will be made" - LoggerFactory.getLogger(javaClass.simpleName).info("$sourceType: $key => $usedValue") - - } - - - fun readPreference(): String? { - val prefFile = ReaderEnv.encodePreference - if (!prefFile.exists()) { - LoggerFactory.getLogger(javaClass.simpleName).info("Preference file: ${prefFile.absolutePath} does not exists...") - LoggerFactory.getLogger(javaClass.simpleName).info("Using default configuration") - return null - } - else { - LoggerFactory.getLogger(javaClass.simpleName).info("Preference file: ${prefFile.absolutePath} found") - } - - try { - val instr = prefFile.inputStream() - return instr.bufferedReader().use { it.readText() } - } - catch (e: Exception) { - LoggerFactory.getLogger(javaClass.simpleName).error("Failed to read preference file: ${prefFile.absolutePath}.. Will use default configuration") - } - return null - } - - fun deserialize(value: String?): EncodingPreference? { - value ?: return null - return Gson().fromJson(value, EncodingPreference::class.java) ?: null - } - - -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/ResultCollection.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/ResultCollection.kt deleted file mode 100644 index 41b35e4e..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/ResultCollection.kt +++ /dev/null @@ -1,72 +0,0 @@ -package no.iktdev.streamit.content.reader.collector - -import no.iktdev.streamit.content.common.deserializers.* -import no.iktdev.streamit.content.common.dto.ContentOutName -import no.iktdev.streamit.content.common.dto.Metadata -import no.iktdev.streamit.content.common.dto.reader.EpisodeInfo -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.content.common.dto.reader.MovieInfo -import no.iktdev.streamit.content.common.dto.reader.work.EncodeWork -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.listener.collector.DefaultEventCollection -import no.iktdev.streamit.library.kafka.listener.deserializer.deserializeIfSuccessful -import org.apache.kafka.clients.consumer.ConsumerRecord - -class ResultCollection: DefaultEventCollection() { - - fun getFirstOrNull(events: KafkaEvents): ConsumerRecord? { - return getRecords().firstOrNull { it.key() == events.event } - } - - fun getReferenceId(): String? { - return getRecords().firstOrNull()?.value()?.referenceId - } - - /** - * @see KafkaEvents.EVENT_READER_RECEIVED_FILE - * @see FileResult for data structure - */ - fun getFileResult(): FileResult? { - val record = getRecords().firstOrNull { it.key() == KafkaEvents.EVENT_READER_RECEIVED_FILE.event } ?: return null - return FileResultDeserializer().deserializeIfSuccessful(record.value()) - } - - /** - * @see KafkaEvents.EVENT_READER_DETERMINED_FILENAME - * @see ContentOutName for data structure - */ - fun getFileName(): ContentOutName? { - val record = getFirstOrNull(KafkaEvents.EVENT_READER_DETERMINED_FILENAME) ?: return null - return ContentOutNameDeserializer().deserializeIfSuccessful(record.value()) - } - - /** - * @see KafkaEvents.EVENT_METADATA_OBTAINED and - * @see Metadata for datastructure - */ - fun getMetadata(): Metadata? { - return firstOrNull(KafkaEvents.EVENT_METADATA_OBTAINED)?.let { - MetadataResultDeserializer().deserializeIfSuccessful(it.value()) - } - } - - fun getMovieInfo(): MovieInfo? { - return firstOrNull(KafkaEvents.EVENT_READER_DETERMINED_MOVIE)?.let { - MovieInfoDeserializer().deserializeIfSuccessful(it.value()) - } - } - - fun getSerieInfo(): EpisodeInfo? { - return firstOrNull(KafkaEvents.EVENT_READER_DETERMINED_SERIE)?.let { - EpisodeInfoDeserializer().deserializeIfSuccessful(it.value()) - } - } - - fun getEncodeWork(): EncodeWork? { - return firstOrNull(KafkaEvents.EVENT_ENCODER_VIDEO_FILE_ENDED)?.let { - EncodeWorkDeserializer().deserializeIfSuccessful(it.value()) - } - } - -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/SubtitleConsumer.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/SubtitleConsumer.kt deleted file mode 100644 index 6c699fbb..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/SubtitleConsumer.kt +++ /dev/null @@ -1,120 +0,0 @@ -package no.iktdev.streamit.content.reader.collector - -import mu.KotlinLogging -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.common.deserializers.DeserializerRegistry -import no.iktdev.streamit.content.common.dto.reader.work.ConvertWork -import no.iktdev.streamit.content.common.dto.reader.work.ExtractWork -import no.iktdev.streamit.library.db.query.SubtitleQuery -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.SimpleMessageListener -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.jetbrains.exposed.sql.transactions.transaction -import org.springframework.stereotype.Service -import java.io.File - -private val logger = KotlinLogging.logger {} - -@Service -class SubtitleConsumer : DefaultKafkaReader("collectorConsumerExtractedSubtitle") { - - private val listener = object: SimpleMessageListener( - topic = CommonConfig.kafkaTopic, - consumer = defaultConsumer, - accepts = listOf( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_ENDED.event, - KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_ENDED.event - ) - ) { - override fun onMessageReceived(data: ConsumerRecord) { - val referenceId = data.value().referenceId - if (data.key() == KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_ENDED.event) { - val work = data.value().dataAs(ExtractWork::class.java) - if (work == null) { - logger.info { "Event: ${data.key()} value is null" } - } else { - storeExtractWork(referenceId, work) - } - } else if (data.key() == KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_ENDED.event) { - val work = data.value().dataAs(ConvertWork::class.java) - if (work == null) { - logger.info { "Event: ${data.key()} value is null" } - } else { - storeConvertWork(referenceId, work) - } - } else { - if (data.value().isSuccessful()) { - logger.warn { "Event: ${data.key()} is not captured" } - } else { - logger.info { "Event: ${data.key()} is not ${StatusType.SUCCESS.name}" } - } - } - } - } - - init { - listener.listen() - } - - fun produceMessage(referenceId: String, outFile: String, statusType: StatusType, result: Any?) { - if (statusType == StatusType.SUCCESS) { - produceSuccessMessage(KafkaEvents.EVENT_COLLECTOR_STORED_SUBTITLE, referenceId) - logger.info { "Stored ${File(outFile).absolutePath} subtitle" } - } else { - produceErrorMessage(KafkaEvents.EVENT_COLLECTOR_STORED_SUBTITLE, Message(referenceId, Status(statusType), result), "See log") - logger.error { "Failed to store ${File(outFile).absolutePath} subtitle" } - } - } - - fun storeExtractWork(referenceId: String, work: ExtractWork) { - val of = File(work.outFile) - val status = transaction { - SubtitleQuery( - associatedWithVideo = of.nameWithoutExtension, - language = work.language, - collection = work.collection, - format = of.extension.uppercase(), - file = File(work.outFile).name - ) - .insertAndGetStatus() - } - produceMessage(referenceId, work.outFile, if (status) StatusType.SUCCESS else StatusType.ERROR, "Store Extracted: $status") - } - - fun storeConvertWork(referenceId: String, work: ConvertWork) { - - val status = transaction { - work.outFiles.map { - val of = File(it) - transaction { - SubtitleQuery( - associatedWithVideo = of.nameWithoutExtension, - language = work.language, - collection = work.collection, - format = of.extension.uppercase(), - file = of.name - ) - .insertAndGetStatus() - } to it - } - } - val failed = status.filter { !it.first }.map { it.second } - val success = status.filter { it.first }.map { it.second } - - produceSuccessMessage(KafkaEvents.EVENT_COLLECTOR_STORED_SUBTITLE, referenceId, success) - produceErrorMessage(KafkaEvents.EVENT_COLLECTOR_STORED_SUBTITLE, Message(referenceId, Status(StatusType.ERROR), failed), "See log") - } - - - override fun loadDeserializers(): Map> { - return DeserializerRegistry.getEventToDeserializer( - KafkaEvents.EVENT_ENCODER_SUBTITLE_FILE_ENDED, - KafkaEvents.EVENT_CONVERTER_SUBTITLE_FILE_ENDED - ) - } -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/VideoConsumer.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/VideoConsumer.kt deleted file mode 100644 index e1c56381..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/collector/VideoConsumer.kt +++ /dev/null @@ -1,183 +0,0 @@ -package no.iktdev.streamit.content.reader.collector - -import kotlinx.coroutines.runBlocking -import mu.KotlinLogging -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.common.Downloader -import no.iktdev.streamit.content.common.deserializers.DeserializerRegistry -import no.iktdev.streamit.content.common.dto.Metadata -import no.iktdev.streamit.content.common.dto.reader.EpisodeInfo -import no.iktdev.streamit.library.db.query.* -import no.iktdev.streamit.library.db.tables.catalog -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.listener.collector.CollectorMessageListener -import no.iktdev.streamit.library.kafka.listener.collector.ICollectedMessagesEvent -import no.iktdev.streamit.library.kafka.listener.deserializer.IMessageDataDeserialization -import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq -import org.jetbrains.exposed.sql.andWhere -import org.jetbrains.exposed.sql.insert -import org.jetbrains.exposed.sql.select -import org.jetbrains.exposed.sql.transactions.transaction -import org.jetbrains.exposed.sql.update -import org.springframework.stereotype.Service -import java.io.File -import kotlin.math.log - -private val logger = KotlinLogging.logger {} - -@Service -class VideoConsumer: DefaultKafkaReader("collectorConsumerEncodedVideo"), ICollectedMessagesEvent { - - val listener = CollectorMessageListener( - topic = CommonConfig.kafkaTopic, - consumer = defaultConsumer, - initiatorEvent = KafkaEvents.EVENT_READER_RECEIVED_FILE, - completionEvent = KafkaEvents.EVENT_ENCODER_VIDEO_FILE_ENDED, - acceptsFilter = listOf( - KafkaEvents.EVENT_METADATA_OBTAINED, - KafkaEvents.EVENT_READER_DETERMINED_SERIE, - KafkaEvents.EVENT_READER_DETERMINED_MOVIE, - ), - listener = this, - eventCollectionClass = ResultCollection::class.java - ) - - - init { - listener.listen() - } - - - override fun loadDeserializers(): Map> { - return DeserializerRegistry.getEventToDeserializer(*listener.acceptsFilter.toTypedArray(), listener.initiatorEvent, listener.completionEvent) - } - - override fun onCollectionCompleted(collection: ResultCollection?) { - val metadata = collection?.getMetadata() - val fileData = collection?.getFileResult() - val encodeWork = collection?.getEncodeWork() - val serieData = collection?.getSerieInfo() - val movieData = collection?.getMovieInfo() - logger.info { "Obtained collection: \n\t${collection?.getRecords()?.map { it.key() }?.joinToString("\n\t")}" } - - if (fileData == null || encodeWork == null || collection.getReferenceId() == null) { - logger.error { "Required data is null, as it has either status as non successful or simply missing" } - return - } - val videoFileNameWithExtension = File(encodeWork.outFile).name - val outDir = File(encodeWork.outFile).parentFile - - val iid = transaction { - if (serieData != null) { - val serieInsertStatus = getSerieQueryInstance(serieData, videoFileNameWithExtension)?.insertAndGetStatus() - if (serieInsertStatus == false) { - logger.warn { "Failed to insert episode $videoFileNameWithExtension" } - } - } - if (serieData == null || metadata?.type == "movie") { - val iid = MovieQuery(videoFileNameWithExtension).insertAndGetId() - if (iid == null) { - logger.warn { "Failed to insert movie and get id for it $videoFileNameWithExtension" } - } - iid - } else null - } - - val coverUrl = metadata?.cover - val currentCover = getExistingCover(outDir) - val coverFile = if (currentCover == null || !currentCover.exists()) { - if (coverUrl != null) { - logger.info { "Downloading Cover: $coverUrl" } - runBlocking { - try { - val _file = Downloader(coverUrl, outDir, fileData.title).download() - if (_file == null || !_file.exists()) { - logger.info { "Failed to download the file" } - } - _file - } catch (e: Exception) { - // No cover - e.printStackTrace() - null - } - } - } else { - logger.info { "No cover url received" } - null - } - } else currentCover - - - - - // Serie må alltid fullføres før catalog. dette i tilfelle catalog allerede eksisterer og den thrower slik at transaskjonen blir versertert! - - val status = try { - transaction { - val genres = metadata?.let { insertAndGetGenres(it) } - - val cq = CatalogQuery( - title = fileData.title, - cover = coverFile?.name, - type = if (serieData == null) "movie" else "serie", - collection = fileData.title, - iid = iid, - genres = genres - ) - val catalogType = if (serieData == null) "movie" else "serie" - cq.insertAndGetStatus() - - if (coverFile != null) { - val qres = catalog.select { catalog.title eq fileData.title }.andWhere { catalog.type eq catalogType}.firstOrNull() ?: null - if (qres != null && qres[catalog.cover].isNullOrBlank()) { - catalog.update({ catalog.id eq qres[catalog.id] }) { - it[catalog.cover] = coverFile.name - } - } - } - - val cqId = cq.getId() ?: throw RuntimeException("No Catalog id found!") - metadata?.let { - val summary = it.summary - if (summary != null) { - val success = SummaryQuery(cid = cqId, language = "eng", description = summary).insertAndGetStatus() - } - } - } - } catch (e: Exception) { - e.printStackTrace() - } - - produceSuccessMessage(KafkaEvents.EVENT_COLLECTOR_STORED_VIDEO, collection.getReferenceId() ?: "M.I.A", status) - logger.info { "Stored ${encodeWork.outFile} video" } - } - - /** - * Needs to be wrapped in transaction - */ - fun insertAndGetGenres(meta: Metadata): String? { - val gq = GenreQuery(*meta.genres.toTypedArray()) - gq.insertAndGetIds() - return gq.getIds().joinToString(",") - } - - fun getSerieQueryInstance(data: EpisodeInfo?, baseName: String?): SerieQuery? { - if (data == null || baseName == null) return null - return SerieQuery(data.episodeTitle, data.episode, data.season, data.title, baseName) - } - - val validCoverFormat = listOf( - "png", - "jpg", - "jpeg", - "webp", - "bmp", - "tiff" - ) - fun getExistingCover(contentDir: File): File? { - val possibleCovers = contentDir.walkTopDown().filter { it.isFile && validCoverFormat.contains(it.extension)} - return possibleCovers.firstOrNull() - } - -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/dto/CompletedItem.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/dto/CompletedItem.kt deleted file mode 100644 index 5f66ebc2..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/dto/CompletedItem.kt +++ /dev/null @@ -1,14 +0,0 @@ -package no.iktdev.streamit.content.reader.dto - -data class CompletedItem( - val name: String, - val fullName: String, - val time: String, - val operations: List -) - -enum class CompletedTypes { - ENCODE, - EXTRACT, - CONVERT -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcher.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcher.kt deleted file mode 100644 index cbd07cb1..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcher.kt +++ /dev/null @@ -1,127 +0,0 @@ -package no.iktdev.streamit.content.reader.fileWatcher - -import com.google.gson.Gson -import dev.vishna.watchservice.KWatchEvent -import dev.vishna.watchservice.asWatchChannel -import kotlinx.coroutines.channels.consumeEach -import kotlinx.coroutines.launch -import mu.KotlinLogging -import no.iktdev.exfl.coroutines.Coroutines -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.Naming -import no.iktdev.streamit.content.common.dto.reader.FileResult - -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.consumers.DefaultConsumer -import no.iktdev.streamit.library.kafka.listener.SimpleMessageListener -import no.iktdev.streamit.library.kafka.producer.DefaultProducer -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.springframework.stereotype.Service -import java.io.File - -private val logger = KotlinLogging.logger {} -@Service -class FileWatcher: FileWatcherEvents { - val messageProducer = DefaultProducer(CommonConfig.kafkaTopic) - val defaultConsumer = DefaultConsumer(subId = "fileWatcher") - - val queue = FileWatcherQueue() - - - val watcherChannel = CommonConfig.incomingContent.asWatchChannel() - init { - Coroutines.io().launch { - watcherChannel.consumeEach { - when (it.kind) { - KWatchEvent.Kind.Deleted -> { - queue.removeFromQueue(it.file, this@FileWatcher::onFileRemoved) - } - - KWatchEvent.Kind.Created, KWatchEvent.Kind.Initialized -> { - if (validVideoFiles().contains(it.file.extension)) { - queue.addToQueue(it.file, this@FileWatcher::onFilePending, this@FileWatcher::onFileAvailable) - } else if (it.file.isFile) { - logger.warn { "${it.file.name} is not a valid file type" } - } else if (it.file.isDirectory) { - val valid = it.file.walkTopDown().filter { f -> f.isFile && f.extension in validVideoFiles() } - logger.warn { "Ignoring directory: ${it.file.name}" } - } - } - - else -> { - logger.info { "Ignoring event kind: ${it.kind.name} for file ${it.file.name}" } - } - } - } - } - - object : SimpleMessageListener(CommonConfig.kafkaTopic, defaultConsumer, listOf(KafkaEvents.REQUEST_FILE_READ.event)) { - override fun onMessageReceived(data: ConsumerRecord) { - if (data.value().status.statusType == StatusType.SUCCESS) { - if (data.value().data is String) { - val file = File(CommonConfig.incomingContent, data.value().data as String) - Coroutines.io().launch { - watcherChannel?.send(KWatchEvent( - file = file, - kind = KWatchEvent.Kind.Initialized, - tag = null - )) - } - } - } - } - } - } - - fun validVideoFiles(): List = listOf( - "mkv", - "avi", - "mp4", - "wmv", - "webm", - "mov" - ) - - - override fun onFileAvailable(file: PendingFile) { - logger.debug { "onFileAvailable har mottatt pendingFile ${file.file.name}" } - val naming = Naming(file.file.nameWithoutExtension) - val message = Message( - referenceId = file.id, - status = Status(StatusType.SUCCESS), - data = FileResult(file = file.file.absolutePath, title = naming.guessDesiredTitle(), sanitizedName = naming.guessDesiredFileName()) - ) - logger.debug { "Producing message: ${Gson().toJson(message)}" } - messageProducer.sendMessage(KafkaEvents.EVENT_READER_RECEIVED_FILE.event, message) - } - - override fun onFilePending(file: PendingFile) { - val message = Message( - status = Status(StatusType.PENDING), - data = FileResult(file = file.file.absolutePath) - ) - messageProducer.sendMessage(KafkaEvents.EVENT_READER_RECEIVED_FILE.event , message) - } - - override fun onFileFailed(file: PendingFile) { - val message = Message( - status = Status(StatusType.ERROR), - data = file.file.absolutePath - ) - messageProducer.sendMessage(KafkaEvents.EVENT_READER_RECEIVED_FILE.event , message) - } - - override fun onFileRemoved(file: PendingFile) { - val message = Message( - status = Status(StatusType.IGNORED), - data = file.file.absolutePath - ) - messageProducer.sendMessage(KafkaEvents.EVENT_READER_RECEIVED_FILE.event , message) - } - - - -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcherEvents.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcherEvents.kt deleted file mode 100644 index e65cae2c..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcherEvents.kt +++ /dev/null @@ -1,20 +0,0 @@ -package no.iktdev.streamit.content.reader.fileWatcher - -import java.io.File - -interface FileWatcherEvents { - fun onFileAvailable(file: PendingFile) - - /** - * If the file is being copied or incomplete, or in case a process currently owns the file, pending should be issued - */ - fun onFilePending(file: PendingFile) - - /** - * If the file is either removed or is not a valid file - */ - fun onFileFailed(file: PendingFile) - - - fun onFileRemoved(file: PendingFile) -} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/streams/StreamsReader.kt b/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/streams/StreamsReader.kt deleted file mode 100644 index e9f8b8c1..00000000 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/streams/StreamsReader.kt +++ /dev/null @@ -1,78 +0,0 @@ -package no.iktdev.streamit.content.reader.streams - -import kotlinx.coroutines.runBlocking -import mu.KotlinLogging -import no.iktdev.streamit.content.common.CommonConfig -import no.iktdev.streamit.content.common.deamon.Daemon -import no.iktdev.streamit.content.common.deamon.IDaemon -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.content.reader.ReaderEnv -import no.iktdev.streamit.content.reader.fileWatcher.FileWatcher -import no.iktdev.streamit.library.kafka.KafkaEvents -import no.iktdev.streamit.library.kafka.KafkaEvents.EVENT_READER_RECEIVED_FILE -import no.iktdev.streamit.library.kafka.consumers.DefaultConsumer -import no.iktdev.streamit.library.kafka.dto.Message -import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType -import no.iktdev.streamit.library.kafka.listener.SimpleMessageListener -import no.iktdev.streamit.library.kafka.producer.DefaultProducer -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.springframework.stereotype.Service - -private val logger = KotlinLogging.logger {} - -@Service -class StreamsReader { - - val messageProducer = DefaultProducer(CommonConfig.kafkaTopic) - val defaultConsumer = DefaultConsumer(subId = "streamReader") - - - init { - object: SimpleMessageListener(topic = CommonConfig.kafkaTopic, consumer = defaultConsumer, accepts = listOf(EVENT_READER_RECEIVED_FILE.event)) { - override fun onMessageReceived(data: ConsumerRecord) { - logger.info { "RECORD: ${data.key()}" } - if (data.value().status.statusType != StatusType.SUCCESS) { - logger.info { "Ignoring event: ${data.key()} as status is not Success!" } - return - } - val dataValue = data.value().dataAs(FileResult::class.java) - - if (dataValue == null) { - logger.info { "Ignoring event: ${data.key()} as values is not of expected type!" } - return - } - logger.info { "Preparing Probe for ${dataValue.file}" } - val output = mutableListOf() - val d = Daemon(executable = ReaderEnv.ffprobe, daemonInterface = object: - IDaemon { - override fun onOutputChanged(line: String) { - output.add(line) - } - - override fun onStarted() { - logger.info { "Probe started for ${dataValue.file}" } - } - - override fun onError(code: Int) { - logger.error { "An error occurred for ${dataValue.file}" } - } - - override fun onEnded() { - logger.info { "Probe ended for ${dataValue.file}" } - } - - }) - val resultCode = runBlocking { - val args = listOf("-v", "quiet", "-print_format", "json", "-show_streams", dataValue.file) - d.run(args) - } - - val message = Message(referenceId = data.value().referenceId, status = Status( statusType = if (resultCode == 0) StatusType.SUCCESS else StatusType.ERROR), data = output.joinToString("\n")) - messageProducer.sendMessage(KafkaEvents.EVENT_READER_RECEIVED_STREAMS.event, message) - } - - }.listen() - } - -} \ No newline at end of file diff --git a/Reader/src/main/resources/application.properties b/Reader/src/main/resources/application.properties deleted file mode 100644 index b67553fc..00000000 --- a/Reader/src/main/resources/application.properties +++ /dev/null @@ -1,3 +0,0 @@ -spring.output.ansi.enabled=always -logging.level.org.apache.kafka=INFO -#logging.level.root=DEBUG diff --git a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/Resources.kt b/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/Resources.kt deleted file mode 100644 index 9449301e..00000000 --- a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/Resources.kt +++ /dev/null @@ -1,33 +0,0 @@ -package no.iktdev.streamit.content.reader - -import org.apache.kafka.clients.consumer.ConsumerRecord - -open class Resources { - - fun getText(path: String): String? { - return this.javaClass.classLoader.getResource(path)?.readText() - } - - open class Streams(): Resources() { - fun all(): List { - return listOf( - getSample(0), - getSample(1), - getSample(2), - getSample(3), - getSample(4), - getSample(5), - getSample(6), - ) - } - - fun getSample(number: Int): String { - return getText("streams/sample$number.json")!! - } - } - - fun getConsumerRecord(event: String, data: T): ConsumerRecord { - return ConsumerRecord("testTopic", 0, 0L, event, data) - } - -} \ No newline at end of file diff --git a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/analyzer/EncodedDeserializersTest.kt b/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/analyzer/EncodedDeserializersTest.kt deleted file mode 100644 index 18b41a09..00000000 --- a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/analyzer/EncodedDeserializersTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer - -import no.iktdev.streamit.content.common.deserializers.MediaStreamsDeserializer -import no.iktdev.streamit.content.common.streams.MediaStreams -import no.iktdev.streamit.library.kafka.consumers.DefaultConsumer -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Assertions.* -import org.junit.jupiter.api.Test - -class EncodedDeserializersTest { - - val consumer = DefaultConsumer.GsonDeserializer() - - @Test - fun testDeserializationOfMediaStreams() { - val message = consumer.deserialize("demo", messageMediaStream.toByteArray()) - val result = MediaStreamsDeserializer().deserialize(message) - assertInstanceOf(MediaStreams::class.java, result) - assertThat(result?.streams).isNotNull() - assertThat(result?.streams).isNotEmpty() - } - - - - val messageMediaStream = """ - { - "referenceId": "18c1af44-7a5f-4896-a34c-9a527ef618aa", - "actionType": "ALL", - "status": { - "statusType": "SUCCESS" - }, - "data": "{\n \"streams\": [\n {\n \"index\": 0,\n \"codec_name\": \"hevc\",\n \"codec_long_name\": \"H.265 / HEVC (High Efficiency Video Coding)\",\n \"profile\": \"Main 10\",\n \"codec_type\": \"video\",\n \"codec_time_base\": \"1001/24000\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"width\": 1920,\n \"height\": 1080,\n \"coded_width\": 1920,\n \"coded_height\": 1080,\n \"closed_captions\": 0,\n \"has_b_frames\": 2,\n \"sample_aspect_ratio\": \"1:1\",\n \"display_aspect_ratio\": \"16:9\",\n \"pix_fmt\": \"yuv420p10le\",\n \"level\": 150,\n \"color_range\": \"tv\",\n \"color_space\": \"bt709\",\n \"color_transfer\": \"bt709\",\n \"color_primaries\": \"bt709\",\n \"chroma_location\": \"left\",\n \"field_order\": \"progressive\",\n \"refs\": 1,\n \"r_frame_rate\": \"24000/1001\",\n \"avg_frame_rate\": \"24000/1001\",\n \"time_base\": \"1/1000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"disposition\": {\n \"default\": 1,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"ENCODER\": \"Lavc60.3.100 libx265\",\n \"BPS\": \"1712472\",\n \"DURATION\": \"00:24:00.063708333\",\n \"NUMBER_OF_FRAMES\": \"34527\",\n \"NUMBER_OF_BYTES\": \"308258548\",\n \"_STATISTICS_WRITING_APP\": \"mkvpropedit v76.0 ('Celebration') 64-bit\",\n \"_STATISTICS_WRITING_DATE_UTC\": \"2023-06-28 18:08:19\",\n \"_STATISTICS_TAGS\": \"BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES\"\n }\n },\n {\n \"index\": 1,\n \"codec_name\": \"aac\",\n \"codec_long_name\": \"AAC (Advanced Audio Coding)\",\n \"profile\": \"LC\",\n \"codec_type\": \"audio\",\n \"codec_time_base\": \"1/44100\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"sample_fmt\": \"fltp\",\n \"sample_rate\": \"44100\",\n \"channels\": 2,\n \"channel_layout\": \"stereo\",\n \"bits_per_sample\": 0,\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/1000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"disposition\": {\n \"default\": 1,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"language\": \"jpn\",\n \"BPS\": \"128002\",\n \"DURATION\": \"00:24:00.101000000\",\n \"NUMBER_OF_FRAMES\": \"62021\",\n \"NUMBER_OF_BYTES\": \"23041997\",\n \"_STATISTICS_WRITING_APP\": \"mkvpropedit v76.0 ('Celebration') 64-bit\",\n \"_STATISTICS_WRITING_DATE_UTC\": \"2023-06-28 18:08:19\",\n \"_STATISTICS_TAGS\": \"BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES\"\n }\n },\n {\n \"index\": 2,\n \"codec_name\": \"ass\",\n \"codec_long_name\": \"ASS (Advanced SSA) subtitle\",\n \"codec_type\": \"subtitle\",\n \"codec_time_base\": \"0/1\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/1000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 1440125,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 1,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"language\": \"eng\",\n \"title\": \"English subs\",\n \"BPS\": \"91\",\n \"DURATION\": \"00:23:27.220000000\",\n \"NUMBER_OF_FRAMES\": \"267\",\n \"NUMBER_OF_BYTES\": \"16015\",\n \"_STATISTICS_WRITING_APP\": \"mkvpropedit v76.0 ('Celebration') 64-bit\",\n \"_STATISTICS_WRITING_DATE_UTC\": \"2023-06-28 18:08:19\",\n \"_STATISTICS_TAGS\": \"BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES\"\n }\n },\n {\n \"index\": 3,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"Roboto-Medium.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 4,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"Roboto-MediumItalic.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 5,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"arial.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 6,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"arialbd.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 7,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"comic.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 8,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"comicbd.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 9,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"times.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 10,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"timesbd.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 11,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"trebuc.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 12,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"trebucbd.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 13,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"verdana.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 14,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"verdanab.ttf\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 15,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"CONSOLA.TTF\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n },\n {\n \"index\": 16,\n \"codec_name\": \"ttf\",\n \"codec_long_name\": \"TrueType font\",\n \"codec_type\": \"attachment\",\n \"codec_tag_string\": \"[0][0][0][0]\",\n \"codec_tag\": \"0x0000\",\n \"r_frame_rate\": \"0/0\",\n \"avg_frame_rate\": \"0/0\",\n \"time_base\": \"1/90000\",\n \"start_pts\": 0,\n \"start_time\": \"0.000000\",\n \"duration_ts\": 129611250,\n \"duration\": \"1440.125000\",\n \"disposition\": {\n \"default\": 0,\n \"dub\": 0,\n \"original\": 0,\n \"comment\": 0,\n \"lyrics\": 0,\n \"karaoke\": 0,\n \"forced\": 0,\n \"hearing_impaired\": 0,\n \"visual_impaired\": 0,\n \"clean_effects\": 0,\n \"attached_pic\": 0,\n \"timed_thumbnails\": 0\n },\n \"tags\": {\n \"filename\": \"CONSOLAB.TTF\",\n \"mimetype\": \"application/x-truetype-font\"\n }\n }\n ]\n}" - } - """.trimIndent() -} \ No newline at end of file diff --git a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/FileNameDeterminateTest.kt b/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/FileNameDeterminateTest.kt deleted file mode 100644 index 9d9c1d52..00000000 --- a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/FileNameDeterminateTest.kt +++ /dev/null @@ -1,251 +0,0 @@ -package no.iktdev.streamit.content.reader.analyzer.contentDeterminator - -import no.iktdev.streamit.content.common.dto.reader.EpisodeInfo -import no.iktdev.streamit.content.common.dto.reader.MovieInfo -import no.iktdev.streamit.content.common.dto.reader.VideoInfo -import org.assertj.core.api.AssertionsForInterfaceTypes.assertThat -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Named -import org.junit.jupiter.api.Test -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource - -data class DataHolder( - val title: String, - val sanitizedName: String, - val ctype: FileNameDeterminate.ContentType = FileNameDeterminate.ContentType.UNDEFINED -) - -class FileNameDeterminateTest { - - data class TestData( - val expected: VideoInfo, - val input: DataHolder - ) - - @ParameterizedTest - @MethodSource("serieTestCases") - fun testDetermineFileNameForSerie(namedTestData: TestData) { - val fileNameDeterminate = - FileNameDeterminate( - namedTestData.input.title, - namedTestData.input.sanitizedName, - FileNameDeterminate.ContentType.SERIE - ) - val result = fileNameDeterminate.getDeterminedVideoInfo() - assertThat(result).isNotNull() - assertThat(result?.fullName).isEqualTo(namedTestData.expected.fullName) - } - - @ParameterizedTest(name = "{0}") - @MethodSource("movieTestCases") - fun testDetermineFileNameForMovie(namedTestData: TestData) { - val fileNameDeterminate = - FileNameDeterminate( - namedTestData.input.title, - namedTestData.input.sanitizedName, - FileNameDeterminate.ContentType.MOVIE - ) - val result = fileNameDeterminate.getDeterminedVideoInfo() - assertThat(result).isNotNull() - assertThat(result?.fullName).isEqualTo(namedTestData.expected.fullName) - } - - @ParameterizedTest() - @MethodSource("undefinedTestCases") - fun testDetermineFileNameForUndefined(namedTestData: TestData) { - val fileNameDeterminate = - FileNameDeterminate( - namedTestData.input.title, - namedTestData.input.sanitizedName, - FileNameDeterminate.ContentType.UNDEFINED - ) - val result = fileNameDeterminate.getDeterminedVideoInfo() - assertThat(result).isNotNull() - assertThat(result?.fullName).isEqualTo(namedTestData.expected.fullName) - } - - @Test - fun test() { - val fileNameDeterminate = FileNameDeterminate( - "Game of Thrones", "Game of Thrones - 01", FileNameDeterminate.ContentType.UNDEFINED - ) - assertThat(fileNameDeterminate.getDeterminedVideoInfo()?.fullName).isEqualTo("Game of Thrones - S01E01") - - - val td = TestData( - expected = EpisodeInfo(title = "Game of Thrones", fullName = "Game of Thrones - S01E01", episode = 1, season = 1, episodeTitle = ""), - input = DataHolder("Game of Thrones", "Game of Thrones - 01") - ) - - val fileNameDeterminate2 = FileNameDeterminate( - td.input.title, td.input.sanitizedName, FileNameDeterminate.ContentType.UNDEFINED - ) - assertThat(fileNameDeterminate2.getDeterminedVideoInfo()?.fullName).isEqualTo(td.expected.fullName) - - } - - @Test - fun testWildStuff() { - val fileNameDeterminate = FileNameDeterminate( - "The Potato man", "The.Potato.man.2023.1080p.L950XL.x265-WIN10", FileNameDeterminate.ContentType.UNDEFINED - ) - assertThat(fileNameDeterminate.getDeterminedVideoInfo()?.fullName).isEqualTo("The Potato man") - } - - companion object { - @JvmStatic - fun serieTestCases(): List> { - return listOf( - Named.of("Is defined", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13", episode = 13, season = 1, episodeTitle = "" ), - DataHolder("Iseleve","Iseleve - 13") - )), - Named.of("Is defined", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13 - potetmos", episode = 13, season = 1, episodeTitle = "potetmos" ), - input = DataHolder("Iseleve","Iseleve - 13 potetmos") - )), - Named.of("Season and Episode in S01E01 format", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13", episode = 13, season = 1, episodeTitle = "" ), - input = DataHolder("Iseleve","Iseleve - S1E13") - )), - - - Named.of("Season and Episode with episode title", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13 - potetmos", episode = 13, season = 1, episodeTitle = "potetmos" ), - input = DataHolder("Iseleve","Iseleve - S1E13 potetmos") - )), - Named.of("Season and Episode with space separator", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13", episode = 13, season = 1, episodeTitle = "" ), - input = DataHolder("Iseleve","Iseleve - S1 13") - )), - Named.of("Season and Episode with space separator and episode title", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13 - potetos", episode = 13, season = 1, episodeTitle = "" ), - input = DataHolder("Iseleve","Iseleve - S1 13 potetos") - )), - Named.of("Lowercase season and episode", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13", episode = 13, season = 1, episodeTitle = "" ), - input = DataHolder("Iseleve","Iseleve - s1e13") - )), - Named.of("Episode title with Season and Episode in text", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13", episode = 13, season = 1, episodeTitle = "" ), - input = DataHolder("Iseleve","Iseleve - Season 1 Episode 13") - )), - Named.of("Episode title with Season and Episode in text and episode title", TestData( - expected = EpisodeInfo(title = "Iseleve", fullName = "Iseleve - S01E13 - Potetmos", episode = 13, season = 1, episodeTitle = "Potetmos" ), - input = DataHolder("Iseleve","Iseleve - Season 1 Episode 13 Potetmos") - )), - ) - } - - @JvmStatic - fun movieTestCases(): List> { - return listOf( - Named.of( - "Movie with year", TestData( - MovieInfo("Some Movie", "Some Movie"), - DataHolder("Some Movie (2012)", "Some Movie (2012)", FileNameDeterminate.ContentType.MOVIE) - ) - ), - Named.of( - "Movie without year", TestData( - MovieInfo("Another Movie", "Another Movie"), - DataHolder("Another Movie", "Another Movie", FileNameDeterminate.ContentType.MOVIE) - ) - - ), - Named.of( - "Movie with year and additional info", TestData( - expected = MovieInfo("Awesome Movie", "Awesome Movie - Part 1"), - DataHolder("Awesome Movie (2012) - Part 1", "Awesome Movie (2012) - Part 1") - ) - - ), - Named.of("Movie with title as '2012'", TestData( - expected = MovieInfo("2012", "2012"), - DataHolder("2012", "2012") - )), - Named.of("Movie with title as '2020'", TestData( - expected = MovieInfo("2020", "2020"), - DataHolder("2020 (2012)", "2020 (2012)") - )), - Named.of("Movie with title as '2049'", TestData( - expected = MovieInfo("2049", "2049"), - DataHolder("2049 (2017)", "2049 (2017)") - )), - Named.of("Movie with title as '3000'", TestData( - expected = MovieInfo("3000", "3000"), - DataHolder("3000 (2000)", "3000 (2000)") - )), - Named.of("Avengers - Endgame", TestData( - expected = MovieInfo("Avengers", "Avengers - Endgame"), - - DataHolder("Avengers - Endgame", "Avengers - Endgame") - )), - Named.of( - "Ghost in the Shell (S.A.C) - Solid State Society", TestData( - expected = MovieInfo("Ghost in the Shell", "Ghost in the Shell (S A C) - Solid State Society"), - DataHolder( - "Ghost in the Shell - Solid State Society", - "Ghost in the Shell (S.A.C) - Solid State Society" - ) - ) - ), - ) - } - - @JvmStatic - fun undefinedTestCases(): List> { - return listOf( - Named.of("Undefined - Movie", TestData( - expected = MovieInfo("Avengers", "Avengers - Endgame"), - input = DataHolder("Avengers", "Avengers - Endgame") - )), - Named.of("Undefined - Series", TestData( - expected = MovieInfo("Stranger Things", "Stranger Things"), - input = DataHolder("Stranger Things", "Stranger Things") - )), - Named.of("Undefined - Movie with Year", TestData( - expected = MovieInfo("Inception", "Inception"), - input = DataHolder("Inception", "Inception (2010)") - )), - Named.of("Undefined - Series with Year", TestData( - expected = MovieInfo("Friends", "Friends"), - input = DataHolder("Friends", "Friends (1994)") - )), - Named.of("Undefined - Movie with Genre", TestData( - expected = MovieInfo("The Dark Knight", "The Dark Knight"), - input = DataHolder("The Dark Knight", "The Dark Knight") - )), - Named.of("Undefined - Series with Genre", TestData( - expected = MovieInfo("Breaking Bad", "Breaking Bad"), - input = DataHolder("Breaking Bad", "Breaking Bad") - )), - Named.of( - "Undefined - Movie with Keywords", - TestData( - expected = MovieInfo("The Lord of the Rings", "The Lord of the Rings"), - input = DataHolder("The Lord of the Rings", "The Lord of the Rings (Movie)") - ) - ), - Named.of("Undefined - Series with Keywords", TestData( - expected = EpisodeInfo("Game of Thrones", fullName = "Game of Thrones 01", episode = 1, season = 1, episodeTitle = ""), - input = DataHolder("Game of Thrones", "Game of Thrones 01") - )), - Named.of("Undefined - Series with Keywords", TestData( - expected = MovieInfo("Game of Thrones", fullName = "Game of Thrones 01"), - input = DataHolder("Game of Thrones", "Game of Thrones 01") - )), - Named.of( - "Undefined - Series with number", - TestData( - expected = EpisodeInfo(title = "Game of Thrones", fullName = "Game of Thrones - S01E01", episode = 1, season = 1, episodeTitle = ""), - input = DataHolder("Game of Thrones", "Game of Thrones - 01") - ) - ), - ) - } - } - - -} diff --git a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/streams/StreamsReaderTest.kt b/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/streams/StreamsReaderTest.kt deleted file mode 100644 index 7e32146f..00000000 --- a/Reader/src/test/kotlin/no/iktdev/streamit/content/reader/streams/StreamsReaderTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -package no.iktdev.streamit.content.reader.streams - -import com.google.gson.Gson -import no.iktdev.streamit.content.common.dto.reader.FileResult -import no.iktdev.streamit.content.reader.fileWatcher.FileWatcher -import no.iktdev.streamit.library.kafka.dto.Message -import org.assertj.core.api.AssertionsForInterfaceTypes.assertThat -import org.junit.jupiter.api.Assertions.* -import org.junit.jupiter.api.Test - -class StreamsReaderTest { - - @Test - fun testDecode() { - val data = """ - { - "referenceId": "7b332099-c663-4158-84d0-9972770316bb", - "status": { - "statusType": "SUCCESS" - }, - "data": { - "file": "/src/input/[AAA] Iseleve - 13 [1080p HEVC][00000].mkv", - "title": "Iseleve", - "desiredNewName": "Iseleve - 13 " - } - } - """.trimIndent() - assertDoesNotThrow { - val message = Gson().fromJson(data, Message::class.java) - val result = message.dataAs(FileResult::class.java) - assertThat(result?.title).isEqualTo("Iseleve") - } - } - -} \ No newline at end of file diff --git a/Reader/src/test/resources/streams/sample1.json b/Reader/src/test/resources/streams/sample1.json deleted file mode 100644 index 3b347322..00000000 --- a/Reader/src/test/resources/streams/sample1.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "streams": [ - { - "index": 0, - "codec_name": "hevc", - "codec_long_name": "H.265 / HEVC (High Efficiency Video Coding)", - "profile": "Main 10", - "codec_type": "video", - "codec_time_base": "1/25", - "codec_tag_string": "hev1", - "codec_tag": "0x31766568", - "width": 1920, - "height": 960, - "coded_width": 1920, - "coded_height": 960, - "has_b_frames": 2, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "2:1", - "pix_fmt": "yuv420p10le", - "level": 120, - "color_range": "tv", - "refs": 1, - "r_frame_rate": "25/1", - "avg_frame_rate": "25/1", - "time_base": "1/25000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 70902000, - "duration": "2836.080000", - "bit_rate": "1999184", - "nb_frames": "70902", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "creation_time": "2022-01-04T07:01:48.000000Z", - "language": "und", - "handler_name": "VideoHandler" - } - }, - { - "index": 1, - "codec_name": "aac", - "codec_long_name": "AAC (Advanced Audio Coding)", - "profile": "LC", - "codec_type": "audio", - "codec_time_base": "1/48000", - "codec_tag_string": "mp4a", - "codec_tag": "0x6134706d", - "sample_fmt": "fltp", - "sample_rate": "48000", - "channels": 6, - "channel_layout": "5.1", - "bits_per_sample": 0, - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/48000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 136131024, - "duration": "2836.063000", - "bit_rate": "224000", - "max_bit_rate": "224000", - "nb_frames": "132943", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "creation_time": "2022-01-04T07:01:48.000000Z", - "language": "nor", - "handler_name": "SoundHandler" - } - } - ] -} \ No newline at end of file diff --git a/Reader/src/test/resources/streams/sample2.json b/Reader/src/test/resources/streams/sample2.json deleted file mode 100644 index 98dfecf3..00000000 --- a/Reader/src/test/resources/streams/sample2.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "streams": [ - { - "index": 0, - "codec_name": "hevc", - "codec_long_name": "H.265 / HEVC (High Efficiency Video Coding)", - "profile": "Main 10", - "codec_type": "video", - "codec_time_base": "1/24", - "codec_tag_string": "hev1", - "codec_tag": "0x31766568", - "width": 1920, - "height": 960, - "coded_width": 1920, - "coded_height": 960, - "has_b_frames": 2, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "2:1", - "pix_fmt": "yuv420p10le", - "level": 120, - "color_range": "tv", - "refs": 1, - "r_frame_rate": "24/1", - "avg_frame_rate": "24/1", - "time_base": "1/24000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 58857000, - "duration": "2452.375000", - "bit_rate": "1999262", - "nb_frames": "58857", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "creation_time": "2021-12-03T08:59:16.000000Z", - "language": "und", - "handler_name": "VideoHandler" - } - }, - { - "index": 1, - "codec_name": "aac", - "codec_long_name": "AAC (Advanced Audio Coding)", - "profile": "LC", - "codec_type": "audio", - "codec_time_base": "1/48000", - "codec_tag_string": "mp4a", - "codec_tag": "0x6134706d", - "sample_fmt": "fltp", - "sample_rate": "48000", - "channels": 6, - "channel_layout": "5.1", - "bits_per_sample": 0, - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/48000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 117714384, - "duration": "2452.383000", - "bit_rate": "224003", - "max_bit_rate": "224003", - "nb_frames": "114958", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "creation_time": "2021-12-03T08:59:16.000000Z", - "language": "eng", - "handler_name": "SoundHandler" - } - } - ], - "format": { - "filename": "Alex.Rider.S02E01.1080p.WEBRip.x265-RARBG.mp4", - "nb_streams": 2, - "nb_programs": 0, - "format_name": "mov,mp4,m4a,3gp,3g2,mj2", - "format_long_name": "QuickTime / MOV", - "start_time": "0.000000", - "duration": "2452.426000", - "size": "683226674", - "bit_rate": "2228737", - "probe_score": 100, - "tags": { - "major_brand": "isom", - "minor_version": "512", - "compatible_brands": "isomiso2mp41", - "creation_time": "2021-12-03T08:59:16.000000Z", - "title": "Alex.Rider.S02E01.1080p.WEBRip.x265-RARBG", - "encoder": "Lavf58.20.100", - "comment": "Alex.Rider.S02E01.1080p.WEBRip.x265-RARBG" - } - } -} \ No newline at end of file diff --git a/Reader/src/test/resources/streams/sample3.json b/Reader/src/test/resources/streams/sample3.json deleted file mode 100644 index fdded673..00000000 --- a/Reader/src/test/resources/streams/sample3.json +++ /dev/null @@ -1,550 +0,0 @@ -{ - "streams": [ - { - "index": 0, - "codec_name": "hevc", - "codec_long_name": "H.265 / HEVC (High Efficiency Video Coding)", - "profile": "Main 10", - "codec_type": "video", - "codec_time_base": "1001/24000", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "width": 1920, - "height": 804, - "coded_width": 1920, - "coded_height": 808, - "has_b_frames": 2, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "160:67", - "pix_fmt": "yuv420p10le", - "level": 123, - "color_range": "tv", - "color_space": "bt709", - "color_transfer": "bt709", - "color_primaries": "bt709", - "refs": 1, - "r_frame_rate": "24000/1001", - "avg_frame_rate": "24000/1001", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "title": "Presented By EMBER", - "BPS": "3796879", - "DURATION": "02:01:28.782000000", - "NUMBER_OF_FRAMES": "174756", - "NUMBER_OF_BYTES": "3459328516", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 1, - "codec_name": "ac3", - "codec_long_name": "ATSC A/52A (AC-3)", - "codec_type": "audio", - "codec_time_base": "1/48000", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "sample_fmt": "fltp", - "sample_rate": "48000", - "channels": 6, - "channel_layout": "5.1(side)", - "bits_per_sample": 0, - "dmix_mode": "-1", - "ltrt_cmixlev": "-1.000000", - "ltrt_surmixlev": "-1.000000", - "loro_cmixlev": "-1.000000", - "loro_surmixlev": "-1.000000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "bit_rate": "448000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "BPS": "448000", - "DURATION": "02:01:28.832000000", - "NUMBER_OF_FRAMES": "227776", - "NUMBER_OF_BYTES": "408174592", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 2, - "codec_name": "ac3", - "codec_long_name": "ATSC A/52A (AC-3)", - "codec_type": "audio", - "codec_time_base": "1/48000", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "sample_fmt": "fltp", - "sample_rate": "48000", - "channels": 6, - "channel_layout": "5.1(side)", - "bits_per_sample": 0, - "dmix_mode": "-1", - "ltrt_cmixlev": "-1.000000", - "ltrt_surmixlev": "-1.000000", - "loro_cmixlev": "-1.000000", - "loro_surmixlev": "-1.000000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "bit_rate": "448000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "jpn", - "BPS": "448000", - "DURATION": "02:01:28.832000000", - "NUMBER_OF_FRAMES": "227776", - "NUMBER_OF_BYTES": "408174592", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 3, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 7288832, - "duration": "7288.832000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "title": "Signs & Songs@EMBER", - "BPS": "5", - "DURATION": "01:54:41.630000000", - "NUMBER_OF_FRAMES": "90", - "NUMBER_OF_BYTES": "4696", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 4, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 7288832, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "title": "Dialogue@EMBER", - "BPS": "78", - "DURATION": "01:56:48.150000000", - "NUMBER_OF_FRAMES": "1434", - "NUMBER_OF_BYTES": "69001", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 5, - "codec_name": "hdmv_pgs_subtitle", - "codec_long_name": "HDMV Presentation Graphic Stream subtitles", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 7288832, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "title": "Signs & Songs@USBD", - "BPS": "402", - "DURATION": "01:50:49.111000000", - "NUMBER_OF_FRAMES": "56", - "NUMBER_OF_BYTES": "334551", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 6, - "codec_name": "hdmv_pgs_subtitle", - "codec_long_name": "HDMV Presentation Graphic Stream subtitles", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 7288832, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "title": "Dialogue@USBD", - "BPS": "21019", - "DURATION": "02:00:56.802000000", - "NUMBER_OF_FRAMES": "2829", - "NUMBER_OF_BYTES": "19067149", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 7, - "codec_name": "hdmv_pgs_subtitle", - "codec_long_name": "HDMV Presentation Graphic Stream subtitles", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 7288832, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "title": "CC@USBD", - "BPS": "34179", - "DURATION": "01:58:57.850000000", - "NUMBER_OF_FRAMES": "4338", - "NUMBER_OF_BYTES": "30495881", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-05-24 07:43:44", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 8, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 655994880, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "GandhiSans-BoldItalic.otf", - "mimetype": "font/otf" - } - }, - { - "index": 9, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 655994880, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "HAPPYHELL.TTF", - "mimetype": "font/ttf" - } - }, - { - "index": 10, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 655994880, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "AVERIALIBRE-BOLD.TTF", - "mimetype": "font/ttf" - } - }, - { - "index": 11, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 655994880, - "duration": "7288.832000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "GandhiSans-Bold.otf", - "mimetype": "font/otf" - } - }, - { - "index": 12, - "codec_name": "mjpeg", - "codec_long_name": "Motion JPEG", - "profile": "Baseline", - "codec_type": "video", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "width": 640, - "height": 360, - "coded_width": 640, - "coded_height": 360, - "has_b_frames": 0, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "16:9", - "pix_fmt": "yuvj420p", - "level": -99, - "color_range": "pc", - "color_space": "bt470bg", - "chroma_location": "center", - "refs": 1, - "r_frame_rate": "90000/1", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 655994880, - "duration": "7288.832000", - "bits_per_raw_sample": "8", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 1, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "cover.jpg", - "mimetype": "image/jpeg" - } - } - ], - "format": { - "filename": "[EMBER] Belle - Ryuu to Sobakasu no Hime (2021) (Movie) [BDRip] [804p Dual Audio HEVC 10 bits DD].mkv", - "nb_streams": 13, - "nb_programs": 0, - "format_name": "matroska,webm", - "format_long_name": "Matroska / WebM", - "start_time": "0.000000", - "duration": "7288.832000", - "size": "4333518626", - "bit_rate": "4756338", - "probe_score": 100, - "tags": { - "title": "Belle.1080p.Dual.Audio.BDRip.10.bits.DD.x265-EMBER", - "encoder": "libebml v1.4.2 + libmatroska v1.6.4", - "creation_time": "2022-05-24T07:43:44.000000Z" - } - } -} \ No newline at end of file diff --git a/Reader/src/test/resources/streams/sample4.json b/Reader/src/test/resources/streams/sample4.json deleted file mode 100644 index e93715d7..00000000 --- a/Reader/src/test/resources/streams/sample4.json +++ /dev/null @@ -1,1093 +0,0 @@ -{ - "streams": [ - { - "index": 0, - "codec_name": "h264", - "codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10", - "profile": "High", - "codec_type": "video", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "width": 1920, - "height": 1080, - "coded_width": 1920, - "coded_height": 1080, - "closed_captions": 0, - "has_b_frames": 0, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "16:9", - "pix_fmt": "yuv420p", - "level": 40, - "chroma_location": "left", - "field_order": "progressive", - "refs": 1, - "is_avc": "true", - "nal_length_size": "4", - "r_frame_rate": "24000/1001", - "avg_frame_rate": "24000/1001", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "bits_per_raw_sample": "8", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - } - }, - { - "index": 1, - "codec_name": "aac", - "codec_long_name": "AAC (Advanced Audio Coding)", - "profile": "LC", - "codec_type": "audio", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "sample_fmt": "fltp", - "sample_rate": "44100", - "channels": 2, - "channel_layout": "stereo", - "bits_per_sample": 0, - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "jpn", - "title": "Japanese" - } - }, - { - "index": 2, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "title": "English (United States)" - } - }, - { - "index": 3, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "ger", - "title": "German" - } - }, - { - "index": 4, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "spa", - "title": "Spanish" - } - }, - { - "index": 5, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "spa", - "title": "Spanish (Latin America)" - } - }, - { - "index": 6, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "fre", - "title": "French" - } - }, - { - "index": 7, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "ita", - "title": "Italian" - } - }, - { - "index": 8, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "por", - "title": "Portuguese (Brazil)" - } - }, - { - "index": 9, - "codec_name": "ass", - "codec_long_name": "ASS (Advanced SSA) subtitle", - "codec_type": "subtitle", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1420016, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "rus", - "title": "Russian" - } - }, - { - "index": 10, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "arial.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 11, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "arialbd.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 12, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "arialbi.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 13, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "ariali.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 14, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "arialuni.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 15, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "ariblk.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 16, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "cour.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 17, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "courbd.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 18, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "courbi.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 19, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "couri.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 20, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "NotoSans-Medium.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 21, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "tahoma.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 22, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "times.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 23, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "timesbd.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 24, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "timesbi.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 25, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "timesi.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 26, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "trebuc.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 27, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "trebucbd.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 28, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "trebucbi.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 29, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "trebucit.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 30, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "verdana.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 31, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "verdanab.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 32, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "verdanai.ttf", - "mimetype": "font/ttf" - } - }, - { - "index": 33, - "codec_type": "attachment", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127801440, - "duration": "1420.016000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "verdanaz.ttf", - "mimetype": "font/ttf" - } - } - ] -} \ No newline at end of file diff --git a/Reader/src/test/resources/streams/sample5.json b/Reader/src/test/resources/streams/sample5.json deleted file mode 100644 index 67aa6a96..00000000 --- a/Reader/src/test/resources/streams/sample5.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "streams": [ - { - "index": 0, - "codec_name": "hevc", - "codec_long_name": "H.265 / HEVC (High Efficiency Video Coding)", - "profile": "Main 10", - "codec_type": "video", - "codec_tag_string": "hev1", - "codec_tag": "0x31766568", - "width": 1920, - "height": 960, - "coded_width": 1920, - "coded_height": 960, - "closed_captions": 0, - "has_b_frames": 2, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "2:1", - "pix_fmt": "yuv420p10le", - "level": 120, - "color_range": "tv", - "chroma_location": "left", - "refs": 1, - "r_frame_rate": "24/1", - "avg_frame_rate": "24/1", - "time_base": "1/24000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 84883000, - "duration": "3536.791667", - "bit_rate": "1998078", - "nb_frames": "84883", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "creation_time": "2022-05-19T19:59:17.000000Z", - "language": "und", - "handler_name": "VideoHandler", - "vendor_id": "[0][0][0][0]" - } - }, - { - "index": 1, - "codec_name": "aac", - "codec_long_name": "AAC (Advanced Audio Coding)", - "profile": "LC", - "codec_type": "audio", - "codec_tag_string": "mp4a", - "codec_tag": "0x6134706d", - "sample_fmt": "fltp", - "sample_rate": "48000", - "channels": 6, - "channel_layout": "5.1", - "bits_per_sample": 0, - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/48000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 169766400, - "duration": "3536.800000", - "bit_rate": "224001", - "nb_frames": "165790", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "creation_time": "2022-05-19T19:59:17.000000Z", - "language": "eng", - "handler_name": "SoundHandler", - "vendor_id": "[0][0][0][0]" - } - } - ] -} \ No newline at end of file diff --git a/Reader/src/test/resources/streams/sample6.json b/Reader/src/test/resources/streams/sample6.json deleted file mode 100644 index dae6a8e7..00000000 --- a/Reader/src/test/resources/streams/sample6.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "streams": [ - { - "index": 0, - "codec_name": "hevc", - "codec_long_name": "H.265 / HEVC (High Efficiency Video Coding)", - "profile": "Main 10", - "codec_type": "video", - "codec_time_base": "1001/24000", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "width": 1920, - "height": 1080, - "coded_width": 1920, - "coded_height": 1080, - "has_b_frames": 2, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "16:9", - "pix_fmt": "yuv420p10le", - "level": 120, - "color_range": "tv", - "color_space": "bt709", - "color_transfer": "bt709", - "color_primaries": "bt709", - "refs": 1, - "r_frame_rate": "24000/1001", - "avg_frame_rate": "24000/1001", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "title": "Presented By EMBER", - "BPS": "2438576", - "DURATION": "00:23:42.004000000", - "NUMBER_OF_FRAMES": "34094", - "NUMBER_OF_BYTES": "433458227", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-07-06 20:30:37", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 1, - "codec_name": "eac3", - "codec_long_name": "ATSC A/52B (AC-3, E-AC-3)", - "codec_type": "audio", - "codec_time_base": "1/48000", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "sample_fmt": "fltp", - "sample_rate": "48000", - "channels": 2, - "bits_per_sample": 0, - "dmix_mode": "-1", - "ltrt_cmixlev": "-1.000000", - "ltrt_surmixlev": "-1.000000", - "loro_cmixlev": "-1.000000", - "loro_surmixlev": "-1.000000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "jpn", - "BPS": "128000", - "DURATION": "00:23:42.112000000", - "NUMBER_OF_FRAMES": "44441", - "NUMBER_OF_BYTES": "22753792", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-07-06 20:30:37", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 2, - "codec_name": "subrip", - "codec_long_name": "SubRip subtitle", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 1422112, - "duration": "1422.112000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "BPS": "65", - "DURATION": "00:23:25.487000000", - "NUMBER_OF_FRAMES": "342", - "NUMBER_OF_BYTES": "11595", - "_STATISTICS_WRITING_APP": "mkvmerge v65.0.0 ('Too Much') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2022-07-06 20:30:37", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 3, - "codec_name": "mjpeg", - "codec_long_name": "Motion JPEG", - "profile": "Progressive", - "codec_type": "video", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "width": 400, - "height": 564, - "coded_width": 400, - "coded_height": 564, - "has_b_frames": 0, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "100:141", - "pix_fmt": "yuvj420p", - "level": -99, - "color_range": "pc", - "color_space": "bt470bg", - "chroma_location": "center", - "refs": 1, - "r_frame_rate": "90000/1", - "avg_frame_rate": "0/0", - "time_base": "1/90000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 127990080, - "duration": "1422.112000", - "bits_per_raw_sample": "8", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 1, - "timed_thumbnails": 0 - }, - "tags": { - "filename": "cover.jpg", - "mimetype": "image/jpeg" - } - } - ] -} \ No newline at end of file diff --git a/Reader/src/test/resources/streams/sample7.json b/Reader/src/test/resources/streams/sample7.json deleted file mode 100644 index 14273089..00000000 --- a/Reader/src/test/resources/streams/sample7.json +++ /dev/null @@ -1,205 +0,0 @@ -{ - "streams": [ - { - "index": 0, - "codec_name": "hevc", - "codec_long_name": "H.265 / HEVC (High Efficiency Video Coding)", - "profile": "Main 10", - "codec_type": "video", - "codec_time_base": "1/25", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "width": 1920, - "height": 952, - "coded_width": 1920, - "coded_height": 952, - "has_b_frames": 2, - "sample_aspect_ratio": "1:1", - "display_aspect_ratio": "240:119", - "pix_fmt": "yuv420p10le", - "level": 120, - "color_range": "tv", - "refs": 1, - "r_frame_rate": "25/1", - "avg_frame_rate": "25/1", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "BPS": "3698552", - "BPS-eng": "3698552", - "DURATION": "00:43:59.240000000", - "DURATION-eng": "00:43:59.240000000", - "NUMBER_OF_FRAMES": "65981", - "NUMBER_OF_FRAMES-eng": "65981", - "NUMBER_OF_BYTES": "1220170846", - "NUMBER_OF_BYTES-eng": "1220170846", - "_STATISTICS_WRITING_APP": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_APP-eng": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2019-05-21 18:17:28", - "_STATISTICS_WRITING_DATE_UTC-eng": "2019-05-21 18:17:28", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES", - "_STATISTICS_TAGS-eng": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 1, - "codec_name": "aac", - "codec_long_name": "AAC (Advanced Audio Coding)", - "profile": "LC", - "codec_type": "audio", - "codec_time_base": "1/48000", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "sample_fmt": "fltp", - "sample_rate": "48000", - "channels": 2, - "channel_layout": "stereo", - "bits_per_sample": 0, - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 20, - "start_time": "0.020000", - "disposition": { - "default": 1, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "nor", - "BPS": "152584", - "BPS-eng": "152584", - "DURATION": "00:43:58.250000000", - "DURATION-eng": "00:43:58.250000000", - "NUMBER_OF_FRAMES": "123668", - "NUMBER_OF_FRAMES-eng": "123668", - "NUMBER_OF_BYTES": "50319602", - "NUMBER_OF_BYTES-eng": "50319602", - "_STATISTICS_WRITING_APP": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_APP-eng": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2019-05-21 18:17:28", - "_STATISTICS_WRITING_DATE_UTC-eng": "2019-05-21 18:17:28", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES", - "_STATISTICS_TAGS-eng": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 2, - "codec_name": "subrip", - "codec_long_name": "SubRip subtitle", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 2639240, - "duration": "2639.240000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "eng", - "BPS": "21", - "BPS-eng": "21", - "DURATION": "00:43:00.840000000", - "DURATION-eng": "00:43:00.840000000", - "NUMBER_OF_FRAMES": "197", - "NUMBER_OF_FRAMES-eng": "197", - "NUMBER_OF_BYTES": "6798", - "NUMBER_OF_BYTES-eng": "6798", - "_STATISTICS_WRITING_APP": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_APP-eng": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2019-05-21 18:17:28", - "_STATISTICS_WRITING_DATE_UTC-eng": "2019-05-21 18:17:28", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES", - "_STATISTICS_TAGS-eng": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - }, - { - "index": 3, - "codec_name": "subrip", - "codec_long_name": "SubRip subtitle", - "codec_type": "subtitle", - "codec_time_base": "0/1", - "codec_tag_string": "[0][0][0][0]", - "codec_tag": "0x0000", - "r_frame_rate": "0/0", - "avg_frame_rate": "0/0", - "time_base": "1/1000", - "start_pts": 0, - "start_time": "0.000000", - "duration_ts": 2639240, - "duration": "2639.240000", - "disposition": { - "default": 0, - "dub": 0, - "original": 0, - "comment": 0, - "lyrics": 0, - "karaoke": 0, - "forced": 0, - "hearing_impaired": 0, - "visual_impaired": 0, - "clean_effects": 0, - "attached_pic": 0, - "timed_thumbnails": 0 - }, - "tags": { - "language": "dan", - "BPS": "37", - "BPS-eng": "37", - "DURATION": "00:43:20.306000000", - "DURATION-eng": "00:43:20.306000000", - "NUMBER_OF_FRAMES": "276", - "NUMBER_OF_FRAMES-eng": "276", - "NUMBER_OF_BYTES": "12172", - "NUMBER_OF_BYTES-eng": "12172", - "_STATISTICS_WRITING_APP": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_APP-eng": "mkvmerge v17.0.0 ('Be Ur Friend') 64-bit", - "_STATISTICS_WRITING_DATE_UTC": "2019-05-21 18:17:28", - "_STATISTICS_WRITING_DATE_UTC-eng": "2019-05-21 18:17:28", - "_STATISTICS_TAGS": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES", - "_STATISTICS_TAGS-eng": "BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES" - } - } - ] -} \ No newline at end of file diff --git a/UI/.gitignore b/UI/.gitignore deleted file mode 100644 index dada482c..00000000 --- a/UI/.gitignore +++ /dev/null @@ -1,43 +0,0 @@ -.gradle -build/ -!gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ - - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/UI/build.gradle.kts b/UI/build.gradle.kts deleted file mode 100644 index 1e61e47e..00000000 --- a/UI/build.gradle.kts +++ /dev/null @@ -1,74 +0,0 @@ -import org.springframework.boot.gradle.tasks.bundling.BootJar - -plugins { - id("org.springframework.boot") version "2.7.4" - id("io.spring.dependency-management") version "1.0.14.RELEASE" - kotlin("jvm") version "1.8.21" - - kotlin("plugin.spring") version "1.6.21" -} - -base.archivesBaseName = "ui" - -group = "no.iktdev.streamit.content" -version = "1.0-SNAPSHOT" - -repositories { - mavenCentral() - maven("https://jitpack.io") - maven { - url = uri("https://reposilite.iktdev.no/releases") - } - maven { - url = uri("https://reposilite.iktdev.no/snapshots") - } -} - -dependencies { - implementation("no.iktdev.streamit.library:streamit-library-kafka:0.0.2-alpha85") - - implementation("org.springframework.boot:spring-boot-starter-web:3.0.4") - implementation("org.springframework.kafka:spring-kafka:2.8.5") - - implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2") - implementation("org.jetbrains.kotlin:kotlin-reflect") - implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") - - implementation("com.google.code.gson:gson:2.9.0") - implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") - implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") - implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") - - - implementation("no.iktdev:exfl:0.0.13-SNAPSHOT") - - - - - implementation(project(":CommonCode")) -} - -tasks.test { - useJUnitPlatform() -} - -tasks.withType { - dependsOn(":buildFrontend") -} - - -tasks.register("buildFrontend") { - workingDir = file("web") // Stien til frontend-mappen - commandLine("npm", "install") // Installer frontend-avhengigheter - commandLine("npm", "run", "build") // Bygg frontend - - doLast { - copy { - from(file("web/build")) // Byggresultatet fra React-appen - into(file("src/main/resources/static/")) // Mappen der du vil plassere det i Spring Boot-prosjektet - } - } -} - -// Kjør frontendbygget før backendbygget -//tasks.getByName("bootJar").dependsOn("buildFrontend") diff --git a/UI/gradle/wrapper/gradle-wrapper.jar b/UI/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 249e5832..00000000 Binary files a/UI/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/UI/gradlew b/UI/gradlew deleted file mode 100644 index 1b6c7873..00000000 --- a/UI/gradlew +++ /dev/null @@ -1,234 +0,0 @@ -#!/bin/sh - -# -# Copyright © 2015-2021 the original authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -# -# Gradle start up script for POSIX generated by Gradle. -# -# Important for running: -# -# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is -# noncompliant, but you have some other compliant shell such as ksh or -# bash, then to run this script, type that shell name before the whole -# command line, like: -# -# ksh Gradle -# -# Busybox and similar reduced shells will NOT work, because this script -# requires all of these POSIX shell features: -# * functions; -# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», -# «${var#prefix}», «${var%suffix}», and «$( cmd )»; -# * compound commands having a testable exit status, especially «case»; -# * various built-in commands including «command», «set», and «ulimit». -# -# Important for patching: -# -# (2) This script targets any POSIX shell, so it avoids extensions provided -# by Bash, Ksh, etc; in particular arrays are avoided. -# -# The "traditional" practice of packing multiple parameters into a -# space-separated string is a well documented source of bugs and security -# problems, so this is (mostly) avoided, by progressively accumulating -# options in "$@", and eventually passing that to Java. -# -# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, -# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; -# see the in-line comments for details. -# -# There are tweaks for specific operating systems such as AIX, CygWin, -# Darwin, MinGW, and NonStop. -# -# (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt -# within the Gradle project. -# -# You can find Gradle at https://github.com/gradle/gradle/. -# -############################################################################## - -# Attempt to set APP_HOME - -# Resolve links: $0 may be a link -app_path=$0 - -# Need this for daisy-chained symlinks. -while - APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path - [ -h "$app_path" ] -do - ls=$( ls -ld "$app_path" ) - link=${ls#*' -> '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" -APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ - "$@" - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/UI/gradlew.bat b/UI/gradlew.bat deleted file mode 100644 index 107acd32..00000000 --- a/UI/gradlew.bat +++ /dev/null @@ -1,89 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/UI/settings.gradle.kts b/UI/settings.gradle.kts deleted file mode 100644 index 257a3e2b..00000000 --- a/UI/settings.gradle.kts +++ /dev/null @@ -1,4 +0,0 @@ -rootProject.name = "UI" - -include(":CommonCode") -project(":CommonCode").projectDir = File("../CommonCode") \ No newline at end of file diff --git a/UI/src/main/resources/application.properties b/UI/src/main/resources/application.properties deleted file mode 100644 index b6533ddd..00000000 --- a/UI/src/main/resources/application.properties +++ /dev/null @@ -1,13 +0,0 @@ - -#logging.level.org.springframework=INFO -#logging.level.root=INFO - -spring.output.ansi.enabled=always -logging.level.org.apache.kafka=INFO -logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats = INFO -spring.cloud.stream.kafka.binder.replication-factor=1 -logging.level.org.springframework.messaging.simp=INFO -#spring.kafka.bootstrap-servers=192.168.2.250:19092 - -management.endpoints.web.exposure.include=health - diff --git a/apps/build.gradle.kts b/apps/build.gradle.kts new file mode 100644 index 00000000..6daca501 --- /dev/null +++ b/apps/build.gradle.kts @@ -0,0 +1,24 @@ +plugins { + id("java") + kotlin("jvm") +} + +group = "no.iktdev.mediaprocessing" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") + implementation(kotlin("stdlib-jdk8")) +} + +tasks.test { + useJUnitPlatform() +} +kotlin { + jvmToolchain(17) +} \ No newline at end of file diff --git a/apps/converter/build.gradle.kts b/apps/converter/build.gradle.kts new file mode 100644 index 00000000..c38bf115 --- /dev/null +++ b/apps/converter/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + id("java") +} + +group = "no.iktdev.mediaprocessing" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") +} + +tasks.test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/apps/converter/src/main/java/no/iktdev/mediaprocessing/Main.java b/apps/converter/src/main/java/no/iktdev/mediaprocessing/Main.java new file mode 100644 index 00000000..a8ec1821 --- /dev/null +++ b/apps/converter/src/main/java/no/iktdev/mediaprocessing/Main.java @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing; + +public class Main { + public static void main(String[] args) { + System.out.println("Hello world!"); + } +} \ No newline at end of file diff --git a/apps/coordinator/README.md b/apps/coordinator/README.md new file mode 100644 index 00000000..2fc50209 --- /dev/null +++ b/apps/coordinator/README.md @@ -0,0 +1,29 @@ +# FLOW: +### Inputs: +- File watcher +- UI selected file + +## Flows - Video file: +### Flow: + - File watcher + - Coordinator: + - Creates process start with: + - START + - type: FLOW + - file: AbssolutePath + - ReadVideoFileStreams: + - Reads started event + - Reads result + - Produces message with result + - BaseInfo: + - Extracts info from filename + - Extracts info from file media streams + - Produces title and sanitized + - pyMetadata: + - Picks up event + - Searches with sources using title and sanitized + - Produces result + + ---- + - Extract & Encode + - Starts \ No newline at end of file diff --git a/Convert/build.gradle.kts b/apps/coordinator/build.gradle.kts similarity index 59% rename from Convert/build.gradle.kts rename to apps/coordinator/build.gradle.kts index 07429434..6063215e 100644 --- a/Convert/build.gradle.kts +++ b/apps/coordinator/build.gradle.kts @@ -1,13 +1,11 @@ -import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.util.archivesName - plugins { - kotlin("jvm") version "1.8.21" + id("java") + kotlin("jvm") id("org.springframework.boot") version "2.5.5" id("io.spring.dependency-management") version "1.0.11.RELEASE" - kotlin("plugin.spring") version "1.5.31" } -group = "no.iktdev.streamit.content" +group = "no.iktdev.mediaprocessing" version = "1.0-SNAPSHOT" repositories { @@ -20,43 +18,37 @@ repositories { url = uri("https://reposilite.iktdev.no/snapshots") } } + dependencies { - implementation(project(":CommonCode")) - implementation("no.iktdev.library:subtitle:1.7.5-SNAPSHOT") + implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") + implementation("org.springframework.boot:spring-boot-starter-web") + implementation("org.springframework.boot:spring-boot-starter:2.7.0") + implementation("com.google.code.gson:gson:2.8.9") + implementation("org.json:json:20210307") + implementation(project(mapOf("path" to ":shared"))) - implementation("no.iktdev.streamit.library:streamit-library-kafka:0.0.2-alpha84") implementation("no.iktdev:exfl:0.0.13-SNAPSHOT") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") - - implementation("com.github.pgreze:kotlin-process:1.3.1") - implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") - - implementation("com.google.code.gson:gson:2.8.9") - - implementation("org.springframework.boot:spring-boot-starter-web") - implementation("org.springframework.boot:spring-boot-starter:2.7.0") - implementation("org.springframework.kafka:spring-kafka:2.8.5") implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") + implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") + implementation(project(mapOf("path" to ":shared:kafka"))) + + implementation("org.springframework.kafka:spring-kafka:3.0.1") + implementation(project(mapOf("path" to ":shared:contract"))) + testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") + implementation(kotlin("stdlib-jdk8")) } tasks.test { useJUnitPlatform() } - -tasks.bootJar { - archiveFileName.set("converter.jar") - launchScript() -} - -tasks.jar { - archivesName.set("converter.jar") - archiveBaseName.set("converter") -} -archivesName.set("converter.jar") \ No newline at end of file +kotlin { + jvmToolchain(17) +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Coordinator.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Coordinator.kt new file mode 100644 index 00000000..5b0fabc7 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/Coordinator.kt @@ -0,0 +1,160 @@ +package no.iktdev.mediaprocessing.coordinator + +import com.google.gson.Gson +import kotlinx.coroutines.launch +import mu.KotlinLogging +import no.iktdev.exfl.coroutines.Coroutines +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.contract.ProcessType +import no.iktdev.mediaprocessing.shared.kafka.CoordinatorProducer +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.core.DefaultMessageListener +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.* +import no.iktdev.mediaprocessing.shared.kafka.dto.isSuccess +import no.iktdev.mediaprocessing.shared.persistance.PersistentDataReader +import no.iktdev.mediaprocessing.shared.persistance.PersistentDataStore +import no.iktdev.mediaprocessing.shared.persistance.PersistentMessage +import no.iktdev.mediaprocessing.shared.persistance.events +import no.iktdev.streamit.library.kafka.dto.Status +import org.springframework.stereotype.Service +import java.io.File +import java.util.UUID + +@Service +class Coordinator { + val producer = CoordinatorProducer() + private val log = KotlinLogging.logger {} + + + private val listeners: MutableList = mutableListOf() + fun addListener(listener: TaskCreatorListener) { + listeners.add(listener) + } + + + public fun startProcess(file: File, type: ProcessType) { + val processStartEvent = ProcessStarted( + status = Status.STARTED, + file = file.absolutePath, + type = type + ) + producer.sendMessage(UUID.randomUUID().toString(), KafkaEvents.EVENT_PROCESS_STARTED, processStartEvent) + } + + fun produceEncodeWork(message: PersistentMessage) { + if (message.event != KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED) { + throw RuntimeException("Incorrect event passed ${message.event}") + } + if (message.data !is FfmpegWorkerArgumentsCreated) { + throw RuntimeException("Invalid data passed:\n${Gson().toJson(message)}") + } + val data = message.data as FfmpegWorkerArgumentsCreated + data.entries.forEach { + FfmpegWorkRequestCreated( + inputFile = data.inputFile, + arguments = it.arguments, + outFile = it.outputFile + ).let { createdRequest -> + producer.sendMessage(message.referenceId, + KafkaEvents.EVENT_WORK_ENCODE_CREATED, + createdRequest) + } + } + } + + fun produceExtractWork(message: PersistentMessage) { + if (message.event != KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED) { + throw RuntimeException("Incorrect event passed ${message.event}") + } + if (message.data !is FfmpegWorkerArgumentsCreated) { + throw RuntimeException("Invalid data passed:\n${Gson().toJson(message)}") + } + val data = message.data as FfmpegWorkerArgumentsCreated + data.entries.forEach { + val eventId = UUID.randomUUID().toString() + FfmpegWorkRequestCreated( + inputFile = data.inputFile, + arguments = it.arguments, + outFile = it.outputFile + ).let { createdRequest -> + producer.sendMessage(message.eventId, + KafkaEvents.EVENT_WORK_EXTRACT_CREATED, + eventId, + createdRequest) + } + val outFile = File(it.outputFile) + ConvertWorkerRequest( + requiresEventId = eventId, + inputFile = it.outputFile, + true, + outFileBaseName = outFile.nameWithoutExtension, + outDirectory = outFile.parentFile.absolutePath + ).let { createdRequest -> + producer.sendMessage(message.referenceId, KafkaEvents.EVENT_WORK_CONVERT_CREATED, + createdRequest) + } + } + } + + + val io = Coroutines.io() + private val listener = DefaultMessageListener(SharedConfig.kafkaTopic) { event -> + val success = PersistentDataStore().storeMessage(event.key.event, event.value) + if (!success) { + log.error { "Unable to store message: ${event.key.event} in database!" } + } else + readAllMessagesFor(event.value.referenceId, event.value.eventId) + } + + fun readAllMessagesFor(referenceId: String, eventId: String) { + io.launch { + val messages = PersistentDataReader().getMessagesFor(referenceId) + createTasksBasedOnEventsAndPersistance(referenceId, eventId, messages) + buildModelBasedOnMessagesFor(referenceId, messages) + } + } + + suspend fun buildModelBasedOnMessagesFor(referenceId: String, messages: List) { + if (messages.any { it.data is ProcessCompleted }) { + // TODO: Build and insert into database + } + } + + fun createTasksBasedOnEventsAndPersistance(referenceId: String, eventId: String, messages: List) { + io.launch { + val triggered = messages.find { it.eventId == eventId } ?: return@launch + listeners.forEach { it.onEventReceived(referenceId, triggered, messages) } + if (listOf(KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED, KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED).contains(triggered.event) && triggered.data.isSuccess()) { + val processStarted = messages.find { it.event == KafkaEvents.EVENT_PROCESS_STARTED }?.data as ProcessStarted + if (processStarted.type == ProcessType.FLOW) { + log.info { "Process for $referenceId was started from flow and will be processed" } + if (triggered.event == KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED) { + produceEncodeWork(triggered) + } else if (triggered.event == KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED) { + produceExtractWork(triggered) + } + } else { + log.info { "Process for $referenceId was started manually and will require user input for continuation" } + } + } + } + } + + + + init { + io.launch { listener.listen() } + } +} + + +abstract class TaskCreator: TaskCreatorListener { + val producer = CoordinatorProducer() + open fun isPrerequisitesOk(events: List): Boolean { + return true + } +} + +interface TaskCreatorListener { + fun onEventReceived(referenceId: String, event: PersistentMessage, events: List): Unit +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt new file mode 100644 index 00000000..7c2c95a6 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/CoordinatorApplication.kt @@ -0,0 +1,25 @@ +package no.iktdev.mediaprocessing.coordinator + +import kotlinx.coroutines.launch +import no.iktdev.exfl.coroutines.Coroutines +import no.iktdev.mediaprocessing.shared.datasource.MySqlDataSource +import no.iktdev.mediaprocessing.shared.persistance.events +import no.iktdev.mediaprocessing.shared.socket.SocketImplementation +import org.springframework.boot.autoconfigure.SpringBootApplication + +@SpringBootApplication +class CoordinatorApplication { +} + +fun main(args: Array) { + val dataSource = MySqlDataSource.fromDatabaseEnv(); + dataSource.createDatabase() + dataSource.createTables( + events + ) +} + + +class SocketImplemented: SocketImplementation() { + +} diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/MetadataMapping.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/MetadataMapping.kt new file mode 100644 index 00000000..2a3e83e6 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/MetadataMapping.kt @@ -0,0 +1,35 @@ +package no.iktdev.mediaprocessing.coordinator.mapping + +import no.iktdev.mediaprocessing.shared.dto.MetadataDto +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.BaseInfoPerformed +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.MetadataPerformed +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.pyMetadata +import no.iktdev.mediaprocessing.shared.kafka.dto.isSuccess +import no.iktdev.mediaprocessing.shared.persistance.PersistentMessage +import no.iktdev.streamit.library.kafka.dto.Status + + + +class MetadataMapping(val events: List) { + + + fun map(): MetadataDto { + val baseInfo = events.find { it.data is BaseInfoPerformed }?.data as BaseInfoPerformed? + val meta = events.find { it.data is MetadataPerformed }?.data as MetadataPerformed? + + if (!baseInfo.isSuccess()) { + return + } + + return MetadataDto( + title = meta.data?.title, + type = meta.data.type, + + + ) + + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/ProcessMapping.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/ProcessMapping.kt new file mode 100644 index 00000000..fcaa7b08 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/ProcessMapping.kt @@ -0,0 +1,51 @@ +package no.iktdev.mediaprocessing.coordinator.mapping + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.ProcessStarted +import no.iktdev.mediaprocessing.shared.persistance.PersistentMessage +import no.iktdev.mediaprocessing.shared.contract.reader.MediaProcessedDto + +class ProcessMapping(val events: List) { + + fun map(): MediaProcessedDto? { + val referenceId = events.firstOrNull()?.referenceId ?: return null + val processStarted = getProcessStarted() + return MediaProcessedDto( + referenceId = referenceId, + process = processStarted?.type, + inputFile = processStarted?.file, + metadata = MetadataMapping(events).map(), + outputFiles = null + ) + } + + fun getProcessStarted(): ProcessStarted? { + return events.lastOrNull { it.data is ProcessStarted }?.data as ProcessStarted? + } + + fun waitsForEncode(): Boolean { + val arguments = events.find { it.event == KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED.event } != null + val performed = events.find { it.event == KafkaEvents.EVENT_WORK_ENCODE_PERFORMED.event } != null + val isSkipped = events.find { it.event == KafkaEvents.EVENT_WORK_ENCODE_SKIPPED.event } != null + return !(isSkipped || (arguments && performed)) + } + + fun waitsForExtract(): Boolean { + val arguments = events.find { it.event == KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED.event } != null + val performed = events.find { it.event == KafkaEvents.EVENT_WORK_EXTRACT_PERFORMED.event } != null + val isSkipped = events.find { it.event == KafkaEvents.EVENT_WORK_EXTRACT_SKIPPED.event } != null + return !(isSkipped || (arguments && performed)) + } + + fun waitsForConvert(): Boolean { + val arguments = events.find { it.event == KafkaEvents.EVENT_WORK_CONVERT_CREATED.event } != null + val performed = events.find { it.event == KafkaEvents.EVENT_WORK_CONVERT_PERFORMED.event } != null + val isSkipped = events.find { it.event == KafkaEvents.EVENT_WORK_CONVERT_SKIPPED.event } != null + return !(isSkipped || (arguments && performed)) + } + + fun canCollect(): Boolean { + return waitsForEncode() && waitsForExtract() && waitsForConvert() + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/BaseInfoFromFile.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/BaseInfoFromFile.kt new file mode 100644 index 00000000..9ccb5a65 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/BaseInfoFromFile.kt @@ -0,0 +1,51 @@ +package no.iktdev.mediaprocessing.coordinator.reader + +import kotlinx.coroutines.launch +import no.iktdev.exfl.coroutines.Coroutines +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.kafka.CoordinatorProducer +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.core.DefaultMessageListener +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.BaseInfoPerformed +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.ProcessStarted +import no.iktdev.mediaprocessing.shared.parsing.FileNameParser +import no.iktdev.streamit.library.kafka.dto.Status +import org.springframework.stereotype.Service +import java.io.File + +@Service +class BaseInfoFromFile { + val io = Coroutines.io() + val listener = DefaultMessageListener(SharedConfig.kafkaTopic) { event -> + val message = event.value() + if (message.data is ProcessStarted) { + io.launch { + readFileInfo(message.referenceId, message.data as ProcessStarted) + } + } + } + val producer = CoordinatorProducer() + + init { + io.launch { + listener.listen() + } + } + + suspend fun readFileInfo(referenceId: String, started: ProcessStarted) { + val result = try { + val fileName = File(started.file).nameWithoutExtension + val fileNameParser = FileNameParser(fileName) + BaseInfoPerformed( + Status.COMPLETED, + title = fileNameParser.guessDesiredTitle(), + sanitizedName = fileNameParser.guessDesiredFileName() + ) + } catch (e: Exception) { + e.printStackTrace() + MessageDataWrapper(Status.ERROR, e.message ?: "Unable to obtain proper info from file") + } + producer.sendMessage(referenceId, KafkaEvents.EVENT_MEDIA_READ_BASE_INFO_PERFORMED, result) + } +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/MediaStreamsAnalyze.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/MediaStreamsAnalyze.kt new file mode 100644 index 00000000..97b9ffc6 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/MediaStreamsAnalyze.kt @@ -0,0 +1,22 @@ +package no.iktdev.mediaprocessing.coordinator.reader + +import no.iktdev.exfl.coroutines.Coroutines +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.core.DefaultMessageListener +import no.iktdev.streamit.library.kafka.dto.Status +import org.springframework.stereotype.Service + +@Service +class MediaStreamsAnalyze { + val io = Coroutines.io() + + val listener = DefaultMessageListener(SharedConfig.kafkaTopic) { event -> + if (event.key() == KafkaEvents.EVENT_MEDIA_READ_STREAM_PERFORMED.event) { + if (event.value().data?.status == Status.COMPLETED) { + + } + } + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/ParseVideoFileStreams.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/ParseVideoFileStreams.kt new file mode 100644 index 00000000..740aa62f --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/ParseVideoFileStreams.kt @@ -0,0 +1,83 @@ +package no.iktdev.mediaprocessing.coordinator.reader + +import com.google.gson.Gson +import com.google.gson.JsonObject +import kotlinx.coroutines.launch +import no.iktdev.exfl.coroutines.Coroutines +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.ffmpeg.AudioStream +import no.iktdev.mediaprocessing.shared.ffmpeg.ParsedMediaStreams +import no.iktdev.mediaprocessing.shared.ffmpeg.SubtitleStream +import no.iktdev.mediaprocessing.shared.ffmpeg.VideoStream +import no.iktdev.mediaprocessing.shared.kafka.CoordinatorProducer +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.core.DefaultMessageListener +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.ReaderPerformed +import no.iktdev.streamit.library.kafka.dto.Status +import org.springframework.stereotype.Service + + +@Service +class ParseVideoFileStreams { + val io = Coroutines.io() + val listener = DefaultMessageListener(SharedConfig.kafkaTopic) { event -> + val message = event.value() + if (message.data is ReaderPerformed) { + io.launch { + parseStreams(message.referenceId, message.data as ReaderPerformed) + } + } + } + val producer = CoordinatorProducer() + + init { + io.launch { + listener.listen() + } + } + + suspend fun parseStreams(referenceId: String, data: ReaderPerformed) { + val gson = Gson() + try { + val jsonObject = gson.fromJson(data.output, JsonObject::class.java) + val jStreams = jsonObject.getAsJsonArray("streams") + + val videoStreams = mutableListOf() + val audioStreams = mutableListOf() + val subtitleStreams = mutableListOf() + + jStreams.forEach { streamJson -> + val streamObject = streamJson.asJsonObject + + val codecType = streamObject.get("codec_type").asString + if (streamObject.has("codec_name") && streamObject.get("codec_name").asString == "mjpeg") { + } else { + when (codecType) { + "video" -> videoStreams.add(gson.fromJson(streamObject, VideoStream::class.java)) + "audio" -> audioStreams.add(gson.fromJson(streamObject, AudioStream::class.java)) + "subtitle" -> subtitleStreams.add(gson.fromJson(streamObject, SubtitleStream::class.java)) + } + } + } + + val parsedStreams = ParsedMediaStreams( + videoStream = videoStreams, + audioStream = audioStreams, + subtitleStream = subtitleStreams + ) + producer.sendMessage(referenceId, KafkaEvents.EVENT_MEDIA_PARSE_STREAM_PERFORMED, + MessageDataWrapper(Status.COMPLETED, gson.toJson(parsedStreams) + ) + ) + + } catch (e: Exception) { + e.printStackTrace() + producer.sendMessage(referenceId, KafkaEvents.EVENT_MEDIA_PARSE_STREAM_PERFORMED, + MessageDataWrapper(Status.ERROR, message = e.message) + ) + } + + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/ReadVideoFileStreams.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/ReadVideoFileStreams.kt new file mode 100644 index 00000000..4ffc7d66 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/reader/ReadVideoFileStreams.kt @@ -0,0 +1,61 @@ +package no.iktdev.mediaprocessing.coordinator.reader + +import kotlinx.coroutines.launch +import no.iktdev.exfl.coroutines.Coroutines +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.kafka.CoordinatorProducer +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.core.DefaultMessageListener +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.ProcessStarted +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.ReaderPerformed +import no.iktdev.mediaprocessing.shared.runner.CodeToOutput +import no.iktdev.mediaprocessing.shared.runner.getOutputUsing +import no.iktdev.streamit.library.kafka.dto.Status +import org.springframework.stereotype.Service +import java.io.File + +@Service +class ReadVideoFileStreams { + val io = Coroutines.io() + val listener = DefaultMessageListener(SharedConfig.kafkaTopic) { event -> + val message = event.value() + if (message.data is ProcessStarted) { + io.launch { + fileReadStreams(message.referenceId, message.data as ProcessStarted) + } + } + } + val producer = CoordinatorProducer() + + init { + io.launch { + listener.listen() + } + } + + suspend fun fileReadStreams(referenceId: String, started: ProcessStarted) { + val file = File(started.file) + if (file.exists() && file.isFile) { + val result = readStreams(file) + + producer.sendMessage( + referenceId, KafkaEvents.EVENT_MEDIA_READ_STREAM_PERFORMED, + ReaderPerformed(Status.COMPLETED, file = started.file, output = result.output.joinToString("\n")) + ) + } else { + producer.sendMessage(referenceId, KafkaEvents.EVENT_MEDIA_READ_STREAM_PERFORMED, + MessageDataWrapper(Status.ERROR, "File in data is not a file or does not exist") + ) + } + } + + suspend fun readStreams(file: File): CodeToOutput { + val result = getOutputUsing( + SharedConfig.ffprobe, + "-v", "quiet", "-print_format", "json", "-show_streams", file.absolutePath + ) + return result + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/MetadataAndBaseInfoToFileOutAndCoverTask.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/MetadataAndBaseInfoToFileOutAndCoverTask.kt new file mode 100644 index 00000000..25814a33 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/MetadataAndBaseInfoToFileOutAndCoverTask.kt @@ -0,0 +1,119 @@ +package no.iktdev.mediaprocessing.coordinator.tasks.event + +import mu.KotlinLogging +import no.iktdev.exfl.using +import no.iktdev.mediaprocessing.coordinator.Coordinator +import no.iktdev.mediaprocessing.coordinator.TaskCreatorListener +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.datasource.toEpochSeconds +import no.iktdev.mediaprocessing.shared.kafka.CoordinatorProducer +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.* +import no.iktdev.mediaprocessing.shared.kafka.dto.isSuccess +import no.iktdev.mediaprocessing.shared.parsing.FileNameDeterminate +import no.iktdev.mediaprocessing.shared.persistance.PersistentMessage +import no.iktdev.streamit.library.kafka.dto.Status +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service +import java.time.LocalDateTime + +/** + * + */ +@Service +class MetadataAndBaseInfoToFileOutAndCoverTask(@Autowired coordinator: Coordinator): TaskCreatorListener { + private val log = KotlinLogging.logger {} + init { + coordinator.addListener(this) + } + val producer = CoordinatorProducer() + val waitingProcessesForMeta: MutableMap = mutableMapOf() + + + override fun onEventReceived(referenceId: String, event: PersistentMessage, events: List) { + if (!listOf( + KafkaEvents.EVENT_MEDIA_READ_BASE_INFO_PERFORMED, + KafkaEvents.EVENT_MEDIA_METADATA_SEARCH_PERFORMED) + .contains(event.event)) { + return + } + + val baseInfo = events.findLast { it.data is BaseInfoPerformed }?.data as BaseInfoPerformed? + val meta = events.findLast { it.data is MetadataPerformed }?.data as MetadataPerformed? + + // Only Return here as both baseInfo events are required to continue + if (!baseInfo.isSuccess() || !baseInfo.hasValidData() || events.any { it.event == KafkaEvents.EVENT_MEDIA_READ_OUT_NAME_AND_TYPE }) { + return + } + if (baseInfo.isSuccess() && meta == null) { + if (!waitingProcessesForMeta.containsKey(referenceId)) { + waitingProcessesForMeta[referenceId] + } + return + } + + baseInfo ?: return // Return if baseInfo is null + + val metaContentType: String? = if (meta.isSuccess()) meta?.data?.type else null + val contentType = when (metaContentType) { + "serie", "tv" -> FileNameDeterminate.ContentType.SERIE + "movie" -> FileNameDeterminate.ContentType.MOVIE + else -> FileNameDeterminate.ContentType.UNDEFINED + } + + val fileDeterminate = FileNameDeterminate(baseInfo.title, baseInfo.sanitizedName, contentType) + if (waitingProcessesForMeta.containsKey(referenceId)) { + waitingProcessesForMeta.remove(referenceId) + } + + val outputDirectory = SharedConfig.outgoingContent.using(baseInfo.title) + + val vi = fileDeterminate.getDeterminedVideoInfo() + if (vi != null) { + producer.sendMessage( + referenceId, + KafkaEvents.EVENT_MEDIA_READ_OUT_NAME_AND_TYPE, + data = VideoInfoPerformed(Status.COMPLETED, vi) + ) + } else { + producer.sendMessage( + referenceId, + KafkaEvents.EVENT_MEDIA_READ_OUT_NAME_AND_TYPE, + data = MessageDataWrapper(Status.ERROR, "No VideoInfo found...") + ) + } + + + val coverUrl = meta?.data?.cover + if (coverUrl.isNullOrBlank()) { + log.warn { "No cover available for ${baseInfo.title}" } + } else { + producer.sendMessage( + referenceId, + KafkaEvents.EVENT_MEDIA_DOWNLOAD_COVER_PARAMETER_CREATED, + CoverInfoPerformed( + status = Status.COMPLETED, + url = coverUrl, + outFileBaseName = baseInfo.title, + outDir = outputDirectory.absolutePath + ) + ) + } + + } + + @Scheduled(fixedDelay = (60_000)) + fun sendErrorMessageForMetadata() { + //val timeThresholdInMinutes = 10 * 60_000 + val expired = waitingProcessesForMeta.filter { + LocalDateTime.now().toEpochSeconds() > (it.value.toEpochSeconds() + 10 * 60) + } + expired.forEach { + producer.sendMessage(it.key, KafkaEvents.EVENT_MEDIA_METADATA_SEARCH_PERFORMED, MessageDataWrapper(status = Status.ERROR, "Timed Out by: ${this::javaClass.name}")) + waitingProcessesForMeta.remove(it.key) + } + } + +} \ No newline at end of file diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/OutNameToWorkArgumentCreator.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/OutNameToWorkArgumentCreator.kt new file mode 100644 index 00000000..0ff5408d --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/OutNameToWorkArgumentCreator.kt @@ -0,0 +1,298 @@ +package no.iktdev.mediaprocessing.coordinator.tasks.event + +import com.google.gson.Gson +import mu.KotlinLogging +import no.iktdev.exfl.using +import no.iktdev.mediaprocessing.coordinator.Coordinator +import no.iktdev.mediaprocessing.coordinator.TaskCreator +import no.iktdev.mediaprocessing.shared.Preference +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.contract.ffmpeg.* +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.* +import no.iktdev.mediaprocessing.shared.kafka.dto.isSuccess +import no.iktdev.mediaprocessing.shared.persistance.PersistentMessage +import no.iktdev.streamit.library.kafka.dto.Status +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import java.io.File + +/** + * Is to be called or to run with the result from FileOout + */ +@Service +class OutNameToWorkArgumentCreator(@Autowired coordinator: Coordinator) : TaskCreator() { + private val log = KotlinLogging.logger {} + + init { + coordinator.addListener(this) + } + + override fun isPrerequisitesOk(events: List): Boolean { + val required = listOf( + KafkaEvents.EVENT_PROCESS_STARTED.event, + KafkaEvents.EVENT_MEDIA_READ_BASE_INFO_PERFORMED.event, + KafkaEvents.EVENT_MEDIA_PARSE_STREAM_PERFORMED.event + ) + return events.filter { it.eventId in required }.all { it.data.isSuccess() } + } + + override fun onEventReceived(referenceId: String, event: PersistentMessage, events: List) { + val preference = Preference.getPreference() + if (event.event != KafkaEvents.EVENT_MEDIA_PARSE_STREAM_PERFORMED) + return + + if (!isPrerequisitesOk(events)) { + return + } + val inputFile = events.find { it.data is ProcessStarted }?.data as ProcessStarted + val baseInfo = events.findLast { it.data is BaseInfoPerformed }?.data as BaseInfoPerformed + val readStreamsEvent = events.find { it.data is MediaStreamsParsePerformed }?.data as MediaStreamsParsePerformed + val serializedParsedStreams = + Gson().fromJson(readStreamsEvent.parsedAsJson, ParsedMediaStreams::class.java) + + val outDir = SharedConfig.outgoingContent.using(baseInfo.title) + + getFfmpegVideoArguments( + inputFile = inputFile.file, + outDir = outDir, + preference = preference.encodePreference, + baseInfo = baseInfo, + serializedParsedStreams = serializedParsedStreams + ).let { producer.sendMessage(referenceId, KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED, it) } + + getFfmpegSubtitleArguments( + inputFile = inputFile.file, + outDir = outDir, + baseInfo = baseInfo, + serializedParsedStreams = serializedParsedStreams + ).let { producer.sendMessage(referenceId, KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED, it) } + + + } + + private fun getFfmpegVideoArguments( + inputFile: String, + outDir: File, + preference: EncodingPreference, + baseInfo: BaseInfoPerformed, + serializedParsedStreams: ParsedMediaStreams + ): MessageDataWrapper { + val outVideoFile = outDir.using("${baseInfo.sanitizedName}.mp4").absolutePath + + val vaas = VideoAndAudioSelector(serializedParsedStreams, preference) + + val vArg = vaas.getVideoStream()?.let { VideoArguments(it, serializedParsedStreams, preference.video).getVideoArguments() } + val aArg = vaas.getAudioStream()?.let { AudioArguments(it, serializedParsedStreams, preference.audio).getAudioArguments() } + + val vaArgs = toFfmpegWorkerArguments(vArg, aArg) + return if (vaArgs.isEmpty()) { + MessageDataWrapper(Status.ERROR, message = "Unable to produce arguments") + } else { + FfmpegWorkerArgumentsCreated( + status = Status.COMPLETED, + inputFile = inputFile, + entries = listOf(FfmpegWorkerArgument( + outputFile = outVideoFile, + arguments = vaArgs + )) + ) + } + } + + private fun getFfmpegSubtitleArguments( + inputFile: String, + outDir: File, + baseInfo: BaseInfoPerformed, + serializedParsedStreams: ParsedMediaStreams + ): MessageDataWrapper { + val subRootDir = outDir.using("sub") + val sArg = SubtitleArguments(serializedParsedStreams.subtitleStream).getSubtitleArguments() + + val entries = sArg.mapNotNull { + FfmpegWorkerArgument( + arguments = it.codecParameters + it.optionalParameters + listOf("-map", "0:s:${it.index}"), + outputFile = subRootDir.using(it.language, "${baseInfo.sanitizedName}.${it.format}").absolutePath + ) + } + return FfmpegWorkerArgumentsCreated( + status = Status.COMPLETED, + inputFile = inputFile, + entries = entries + ) + } + + private class VideoAndAudioSelector(val mediaStreams: ParsedMediaStreams, val preference: EncodingPreference) { + private var defaultVideoSelected: VideoStream? = mediaStreams.videoStream + .filter { (it.duration_ts ?: 0) > 0 } + .maxByOrNull { it.duration_ts ?: 0 } ?: mediaStreams.videoStream.minByOrNull { it.index } + private var defaultAudioSelected: AudioStream? = mediaStreams.audioStream + .filter { (it.duration_ts ?: 0) > 0 } + .maxByOrNull { it.duration_ts ?: 0 } ?: mediaStreams.audioStream.minByOrNull { it.index } + + fun getVideoStream(): VideoStream? { + return defaultVideoSelected + } + + fun getAudioStream(): AudioStream? { + val languageFiltered = mediaStreams.audioStream.filter { it.tags.language == preference.audio.language } + val channeledAndCodec = languageFiltered.find { + it.channels >= (preference.audio.channels ?: 2) && it.codec_name == preference.audio.codec.lowercase() + } + return channeledAndCodec ?: return languageFiltered.minByOrNull { it.index } ?: defaultAudioSelected + } + + } + + private class VideoArguments(val videoStream: VideoStream, val allStreams: ParsedMediaStreams, val preference: VideoPreference) { + fun isVideoCodecEqual() = getCodec(videoStream.codec_name) == getCodec(preference.codec.lowercase()) + protected fun getCodec(name: String): String { + return when (name) { + "hevc", "hevec", "h265", "h.265", "libx265" + -> "libx265" + + "h.264", "h264", "libx264" + -> "libx264" + + else -> name + } + } + + fun getVideoArguments(): VideoArgumentsDto { + val optionalParams = mutableListOf() + if (preference.pixelFormatPassthrough.none { it == videoStream.pix_fmt }) { + optionalParams.addAll(listOf("-pix_fmt", preference.pixelFormat)) + } + val codecParams = if (isVideoCodecEqual()) listOf("-vcodec", "copy") + else { + optionalParams.addAll(listOf("-crf", preference.threshold.toString())) + listOf("-c:v", getCodec(preference.codec.lowercase())) + } + + return VideoArgumentsDto( + index = allStreams.videoStream.indexOf(videoStream), + codecParameters = codecParams, + optionalParameters = optionalParams + ) + } + } + + private class AudioArguments(val audioStream: AudioStream, val allStreams: ParsedMediaStreams, val preference: AudioPreference) { + fun isAudioCodecEqual() = audioStream.codec_name.lowercase() == preference.codec.lowercase() + private fun shouldUseEAC3(): Boolean { + return (preference.defaultToEAC3OnSurroundDetected && audioStream.channels > 2 && audioStream.codec_name.lowercase() != "eac3") + } + + fun getAudioArguments(): AudioArgumentsDto { + val optionalParams = mutableListOf() + val codecParams = if (shouldUseEAC3()) + listOf("-c:a", "eac3") + else if (!isAudioCodecEqual()) { + listOf("-c:a", preference.codec) + } else + listOf("-acodec", "copy") + return AudioArgumentsDto( + index = allStreams.audioStream.indexOf(audioStream), + codecParameters = codecParams, + optionalParameters = optionalParams + ) + } + + } + + private class SubtitleArguments(val subtitleStreams: List) { + /** + * @property DEFAULT is default subtitle as dialog + * @property CC is Closed-Captions + * @property SHD is Hard of hearing + * @property NON_DIALOGUE is for Signs or Song (as in lyrics) + */ + private enum class SubtitleType { + DEFAULT, + CC, + SHD, + NON_DIALOGUE + } + + private fun SubtitleStream.isCC(): Boolean { + val title = this.tags.title?.lowercase() ?: return false + val keywords = listOf("cc", "closed caption") + return keywords.any { title.contains(it) } + } + private fun SubtitleStream.isSHD(): Boolean { + val title = this.tags.title?.lowercase() ?: return false + val keywords = listOf("shd", "hh", "Hard-of-Hearing", "Hard of Hearing") + return keywords.any { title.contains(it) } + } + private fun SubtitleStream.isSignOrSong(): Boolean { + val title = this.tags.title?.lowercase() ?: return false + val keywords = listOf("song", "songs", "sign", "signs") + return keywords.any { title.contains(it) } + } + private fun getSubtitleType(stream: SubtitleStream): SubtitleType { + return if (stream.isSignOrSong()) + SubtitleType.NON_DIALOGUE + else if (stream.isSHD()) { + SubtitleType.SHD + } else if (stream.isCC()) { + SubtitleType.CC + } else SubtitleType.DEFAULT + } + + fun getSubtitleArguments(): List { + val acceptable = subtitleStreams.filter { !it.isSignOrSong() } + val codecFiltered = acceptable.filter { getFormatToCodec(it.codec_name) != null } + val mappedToType = codecFiltered.map { getSubtitleType(it) to it }.filter { it.first in SubtitleType.entries } + .groupBy { it.second.tags.language ?: "eng" } + .mapValues { entry -> + val languageStreams = entry.value + val sortedStreams = languageStreams.sortedBy { SubtitleType.entries.indexOf(it.first) } + sortedStreams.firstOrNull()?.second + }.mapNotNull { it.value } + + return mappedToType.mapNotNull { stream -> + getFormatToCodec(stream.codec_name)?.let { format -> + SubtitleArgumentsDto( + index = subtitleStreams.indexOf(stream), + language = stream.tags.language ?: "eng", + format = format + ) + } + } + + } + + fun getFormatToCodec(codecName: String): String? { + return when(codecName) { + "ass" -> "ass" + "subrip" -> "srt" + "webvtt", "vtt" -> "vtt" + "smi" -> "smi" + "hdmv_pgs_subtitle" -> null + else -> null + } + } + + } + + + private fun toFfmpegWorkerArguments( + videoArguments: VideoArgumentsDto?, + audioArguments: AudioArgumentsDto? + ): List { + val arguments = mutableListOf( + *videoArguments?.codecParameters?.toTypedArray() ?: arrayOf(), + *videoArguments?.optionalParameters?.toTypedArray() ?: arrayOf(), + *audioArguments?.codecParameters?.toTypedArray() ?: arrayOf(), + *audioArguments?.optionalParameters?.toTypedArray() ?: arrayOf() + ) + videoArguments?.index?.let { + arguments.addAll(listOf("-map", "0:v:$it")) + } + audioArguments?.index?.let { + arguments.addAll(listOf("-map", "0:a:$it")) + } + return arguments + } +} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcherQueue.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/input/watcher/FileWatcherQueue.kt similarity index 91% rename from Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcherQueue.kt rename to apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/input/watcher/FileWatcherQueue.kt index ab142b95..1b8d0729 100644 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/fileWatcher/FileWatcherQueue.kt +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/input/watcher/FileWatcherQueue.kt @@ -1,10 +1,10 @@ -package no.iktdev.streamit.content.reader.fileWatcher +package no.iktdev.mediaprocessing.coordinator.tasks.input.watcher +import isFileAvailable import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.delay import kotlinx.coroutines.launch import no.iktdev.exfl.coroutines.Coroutines -import no.iktdev.streamit.content.common.FileAccess import java.io.File import java.util.UUID @@ -14,7 +14,7 @@ class FileWatcherQueue { fun addToQueue(file: File, onFilePending: (PendingFile) -> Unit, onFileAccessible: (PendingFile) -> Unit) { // Check if the file is accessible - if (FileAccess.isFileAvailable(file)) { + if (isFileAvailable(file)) { // If accessible, run the function immediately and return onFileAccessible(PendingFile(file = file)) return @@ -28,7 +28,7 @@ class FileWatcherQueue { while (true) { delay(500) val currentFile = fileChannel.receive() - if (FileAccess.isFileAvailable(currentFile.file)) { + if (isFileAvailable(currentFile.file)) { onFileAccessible(currentFile) // File is accessible, remove it from the queue removeFromQueue(currentFile.file) { /* Do nothing here as the operation is not intended to be performed here */ } diff --git a/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/input/watcher/InputDirectoryWatcher.kt b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/input/watcher/InputDirectoryWatcher.kt new file mode 100644 index 00000000..038d4b39 --- /dev/null +++ b/apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/input/watcher/InputDirectoryWatcher.kt @@ -0,0 +1,85 @@ +package no.iktdev.mediaprocessing.coordinator.tasks.input.watcher + +import dev.vishna.watchservice.KWatchEvent.Kind.Deleted +import dev.vishna.watchservice.asWatchChannel +import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.launch +import mu.KotlinLogging +import no.iktdev.exfl.coroutines.Coroutines +import no.iktdev.mediaprocessing.coordinator.Coordinator +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.contract.ProcessType +import no.iktdev.mediaprocessing.shared.extended.isSupportedVideoFile +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + + + +interface FileWatcherEvents { + fun onFileAvailable(file: PendingFile) + + /** + * If the file is being copied or incomplete, or in case a process currently owns the file, pending should be issued + */ + fun onFilePending(file: PendingFile) + + /** + * If the file is either removed or is not a valid file + */ + fun onFileFailed(file: PendingFile) + + + fun onFileRemoved(file: PendingFile) +} + + + +@Service +class InputDirectoryWatcher(@Autowired var coordinator: Coordinator): FileWatcherEvents { + private val logger = KotlinLogging.logger {} + val watcherChannel = SharedConfig.incomingContent.asWatchChannel() + val queue = FileWatcherQueue() + val io = Coroutines.io() + + init { + io.launch { + watcherChannel.consumeEach { + when (it.kind) { + Deleted -> queue.removeFromQueue(it.file, this@InputDirectoryWatcher::onFileRemoved) + else -> { + if (it.file.isFile && it.file.isSupportedVideoFile()) { + queue.addToQueue(it.file, this@InputDirectoryWatcher::onFilePending, this@InputDirectoryWatcher::onFileAvailable) + } else if (it.file.isDirectory) { + val supportedFiles = it.file.walkTopDown().filter { f -> f.isFile && f.isSupportedVideoFile() } + supportedFiles.forEach { sf -> + queue.addToQueue(sf, this@InputDirectoryWatcher::onFilePending, this@InputDirectoryWatcher::onFileAvailable) + } + } else { + logger.info { "Ignoring event kind: ${it.kind.name} for file ${it.file.name} as it is not a supported video file" } + } + } + } + } + } + } + + override fun onFileAvailable(file: PendingFile) { + logger.info { "File pending availability ${file.file.name}" } + + // This sens it to coordinator to start the process + coordinator.startProcess(file.file, ProcessType.FLOW) + } + + override fun onFilePending(file: PendingFile) { + logger.info { "File pending availability ${file.file.name}" } + } + + override fun onFileFailed(file: PendingFile) { + logger.warn { "File failed availability ${file.file.name}" } + } + + override fun onFileRemoved(file: PendingFile) { + logger.info { "File removed ${file.file.name} was removed" } + } + +} \ No newline at end of file diff --git a/apps/processer/build.gradle.kts b/apps/processer/build.gradle.kts new file mode 100644 index 00000000..70c5f0de --- /dev/null +++ b/apps/processer/build.gradle.kts @@ -0,0 +1,34 @@ +plugins { + id("java") + kotlin("jvm") + kotlin("plugin.spring") version "1.5.31" + id("org.springframework.boot") version "2.5.5" + id("io.spring.dependency-management") version "1.0.11.RELEASE" +} + +group = "no.iktdev.mediaprocessing.apps" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + implementation(kotlin("stdlib-jdk8")) + + implementation("org.springframework.boot:spring-boot-starter-web:3.0.4") + implementation("org.springframework.kafka:spring-kafka:2.8.5") + + implementation(project(mapOf("path" to ":shared:kafka"))) + implementation(project(mapOf("path" to ":shared"))) + implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") + implementation("com.google.code.gson:gson:2.9.0") + + + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") +} + +tasks.test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/apps/processer/src/main/kotlin/no/mediaprocessing/apps/processer/ProcesserApplication.kt b/apps/processer/src/main/kotlin/no/mediaprocessing/apps/processer/ProcesserApplication.kt new file mode 100644 index 00000000..bad6de26 --- /dev/null +++ b/apps/processer/src/main/kotlin/no/mediaprocessing/apps/processer/ProcesserApplication.kt @@ -0,0 +1,4 @@ +package no.mediaprocessing.apps.processer + +class ProcesserApplication { +} \ No newline at end of file diff --git a/pyMetadata/Dockerfile b/apps/pyMetadata/Dockerfile similarity index 100% rename from pyMetadata/Dockerfile rename to apps/pyMetadata/Dockerfile diff --git a/pyMetadata/__init__.py b/apps/pyMetadata/__init__.py similarity index 100% rename from pyMetadata/__init__.py rename to apps/pyMetadata/__init__.py diff --git a/pyMetadata/app.py b/apps/pyMetadata/app.py similarity index 86% rename from pyMetadata/app.py rename to apps/pyMetadata/app.py index 14fa8c88..60bbd83d 100644 --- a/pyMetadata/app.py +++ b/apps/pyMetadata/app.py @@ -32,19 +32,14 @@ logging.basicConfig( logger = logging.getLogger(__name__) class ProducerDataValueSchema: - def __init__(self, referenceId, statusType, errorMessage, data): + def __init__(self, referenceId, data): self.referenceId = referenceId - self.statusType = statusType - self.errorMessage = errorMessage self.data = data def to_dict(self): return { 'referenceId': self.referenceId, - 'status': { - 'statusType': self.statusType, - 'errorMessage': self.errorMessage - }, + 'eventId': uuid.uuid4(), 'data': self.data.to_dict() if self.data else None } @@ -52,15 +47,6 @@ class ProducerDataValueSchema: data_dict = self.to_dict() return json.dumps(data_dict) - @classmethod - def from_dict(cls, data_dict): - referenceId = data_dict.get('referenceId') - statusType = data_dict['status'].get('statusType') - errorMessage = data_dict['status'].get('errorMessage') - data = data_dict.get('data') - - return cls(referenceId, statusType, errorMessage, data) - def decode_key(key_bytes): return key_bytes.decode('utf-8') if key_bytes else None @@ -103,7 +89,7 @@ class KafkaConsumerThread(threading.Thread): # Sjekk om meldingen har målnøkkelen - if message.key == "request:metadata:obtain" or message.key == "event:reader:received-file": + if message.key == "request:metadata:obtain" or message.key == "event:media-read-base-info:performed": logger.info("Received message: key=%s, value=%s", message.key, message.value) # Opprett en ny tråd for å håndtere meldingen handler_thread = MessageHandlerThread(message) @@ -132,10 +118,10 @@ class MessageHandlerThread(threading.Thread): # Sjekk om meldingen har en Status if 'status' in self.message.value: - status_type = self.message.value['status']['statusType'] + status_type = self.message.value['data']['status'] - # Sjekk om statusen er SUCCESS - if status_type == 'SUCCESS': + # Sjekk om statusen er COMPLETED + if status_type == 'COMPLETED': baseName = self.message.value["data"]["sanitizedName"] title = self.message.value['data']["title"] @@ -154,7 +140,7 @@ class MessageHandlerThread(threading.Thread): key_serializer=lambda k: k.encode('utf-8') if isinstance(k, str) else None, value_serializer=lambda v: v.encode('utf-8') if isinstance(v, str) else None ) - producer.send(kafka_topic, key="event:metadata:obtained", value=result_json) + producer.send(kafka_topic, key="event:media-metadata-search:performed", value=result_json) producer.close() def get_metadata(self, name: str) -> Optional[DataResult]: @@ -168,7 +154,7 @@ class MessageHandlerThread(threading.Thread): logger.info("Not in cache: %s", name) logger.info("Searching in sources for information about %s", name) result: Optional[DataResult] = UseSource(title=name).select_result() - if (result.statusType == "SUCCESS"): + if (result.status == "SUCCESS"): logger.info("Storing response for %s in in-memory cache", name) ResultCache.add(name, result) return result @@ -177,8 +163,6 @@ class MessageHandlerThread(threading.Thread): def compose_message(self, referenceId: str, result: DataResult) -> ProducerDataValueSchema: return ProducerDataValueSchema( referenceId=referenceId, - statusType=result.statusType, - errorMessage=result.errorMessage, data=result.data ) diff --git a/pyMetadata/requirements.txt b/apps/pyMetadata/requirements.txt similarity index 100% rename from pyMetadata/requirements.txt rename to apps/pyMetadata/requirements.txt diff --git a/UI/web/src/App.css b/apps/pyMetadata/sources/__init__.py similarity index 100% rename from UI/web/src/App.css rename to apps/pyMetadata/sources/__init__.py diff --git a/pyMetadata/sources/anii.py b/apps/pyMetadata/sources/anii.py similarity index 82% rename from pyMetadata/sources/anii.py rename to apps/pyMetadata/sources/anii.py index 18899411..eaf9576e 100644 --- a/pyMetadata/sources/anii.py +++ b/apps/pyMetadata/sources/anii.py @@ -24,12 +24,12 @@ class metadata(): usedTitle=self.name ) if (meta.title is None) or (meta.type is None): - return DataResult("IGNORE", None, None) + return DataResult("SUCCESS", None, None) return DataResult("SUCCESS", None, meta) except IndexError as ingore: - return DataResult(statusType="IGNORE", errorMessage=f"No result for {self.name}") + return DataResult(statusType="SUCCESS", message=f"No result for {self.name}") except Exception as e: - return DataResult(statusType="ERROR", errorMessage=str(e)) + return DataResult(statusType="ERROR", message=str(e)) \ No newline at end of file diff --git a/pyMetadata/sources/cache.py b/apps/pyMetadata/sources/cache.py similarity index 100% rename from pyMetadata/sources/cache.py rename to apps/pyMetadata/sources/cache.py diff --git a/pyMetadata/sources/imdb.py b/apps/pyMetadata/sources/imdb.py similarity index 88% rename from pyMetadata/sources/imdb.py rename to apps/pyMetadata/sources/imdb.py index 45d80168..d60ddad5 100644 --- a/pyMetadata/sources/imdb.py +++ b/apps/pyMetadata/sources/imdb.py @@ -26,8 +26,8 @@ class metadata(): usedTitle=self.name ) if (meta.title is None) or (meta.type is None): - return DataResult("IGNORE", None, None) + return DataResult("SUCCESS", None, None) return DataResult("SUCCESS", None, meta) except Exception as e: - return DataResult(statusType="ERROR", errorMessage=str(e)) \ No newline at end of file + return DataResult(status="ERROR", data=None, message=str(e)) \ No newline at end of file diff --git a/pyMetadata/sources/mal.py b/apps/pyMetadata/sources/mal.py similarity index 82% rename from pyMetadata/sources/mal.py rename to apps/pyMetadata/sources/mal.py index cdf69540..48062010 100644 --- a/pyMetadata/sources/mal.py +++ b/apps/pyMetadata/sources/mal.py @@ -11,7 +11,7 @@ class metadata(): try: search = AnimeSearch(self.name) if (len(search.results) == 0): - return DataResult(statusType="IGNORE", errorMessage="No results") + return DataResult(status="SUCCESS", message="No results") anime = Anime(search.results[0].mal_id) meta = Metadata( title = anime.title, @@ -24,8 +24,8 @@ class metadata(): usedTitle=self.name ) if (meta.title is None) or (meta.type is None): - return DataResult("IGNORE", None, None) + return DataResult("SUCCESS", None, None) return DataResult("SUCCESS", None, meta) except Exception as e: - return DataResult(statusType="ERROR", errorMessage=str(e)) \ No newline at end of file + return DataResult(status="ERROR", message=str(e)) \ No newline at end of file diff --git a/pyMetadata/sources/result.py b/apps/pyMetadata/sources/result.py similarity index 53% rename from pyMetadata/sources/result.py rename to apps/pyMetadata/sources/result.py index 2527cf7f..a5bfcbbc 100644 --- a/pyMetadata/sources/result.py +++ b/apps/pyMetadata/sources/result.py @@ -17,19 +17,9 @@ class Metadata: @dataclass class DataResult: - statusType: str - errorMessage: str + status: str # COMPLETED / ERROR + message: str | None = None data: Metadata = None def to_dict(self): return asdict(self) - - @classmethod - def from_dict(cls, data_dict): - metadata_dict = data_dict.get('data') - metadata = Metadata(**metadata_dict) if metadata_dict else None - return cls( - statusType=data_dict['statusType'], - errorMessage=data_dict['errorMessage'], - data=metadata - ) diff --git a/pyMetadata/sources/select.py b/apps/pyMetadata/sources/select.py similarity index 88% rename from pyMetadata/sources/select.py rename to apps/pyMetadata/sources/select.py index 9e9f5694..1cd8c596 100644 --- a/pyMetadata/sources/select.py +++ b/apps/pyMetadata/sources/select.py @@ -28,11 +28,11 @@ class UseSource(): mal = MalMetadata(title).lookup() result: List[WeightedData] = [] - if (anii is not None) and (anii.statusType == "SUCCESS"): + if (anii is not None) and (anii.status == "SUCCESS" and anii.data is not None): result.append(WeightedData(anii, 4)) - if (imdb is not None) and (imdb.statusType == "SUCCESS"): + if (imdb is not None) and (imdb.status == "SUCCESS" and imdb.data is not None): result.append(WeightedData(imdb, 1)) - if (mal is not None) and (mal.statusType == "SUCCESS"): + if (mal is not None) and (mal.status == "SUCCESS" and mal.data is not None): result.append(WeightedData(mal, 8)) return result diff --git a/apps/pyMetadata/tests/__init__.py b/apps/pyMetadata/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyMetadata/tests/test_result.py b/apps/pyMetadata/tests/test_result.py similarity index 100% rename from pyMetadata/tests/test_result.py rename to apps/pyMetadata/tests/test_result.py diff --git a/apps/src/main/java/no/iktdev/mediaprocessing/Main.java b/apps/src/main/java/no/iktdev/mediaprocessing/Main.java new file mode 100644 index 00000000..a8ec1821 --- /dev/null +++ b/apps/src/main/java/no/iktdev/mediaprocessing/Main.java @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing; + +public class Main { + public static void main(String[] args) { + System.out.println("Hello world!"); + } +} \ No newline at end of file diff --git a/apps/ui/build.gradle.kts b/apps/ui/build.gradle.kts new file mode 100644 index 00000000..84e3f4e8 --- /dev/null +++ b/apps/ui/build.gradle.kts @@ -0,0 +1,53 @@ +plugins { + id("java") + kotlin("jvm") + kotlin("plugin.spring") version "1.5.31" + id("org.springframework.boot") version "2.5.5" + id("io.spring.dependency-management") version "1.0.11.RELEASE" +} + +group = "no.iktdev.mediaprocessing" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() + maven("https://jitpack.io") + maven { + url = uri("https://reposilite.iktdev.no/releases") + } + maven { + url = uri("https://reposilite.iktdev.no/snapshots") + } +} + +dependencies { + implementation(kotlin("stdlib-jdk8")) + + + implementation("org.springframework.boot:spring-boot-starter-web:3.0.4") + implementation("org.springframework.kafka:spring-kafka:2.8.5") + + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2") + implementation("org.jetbrains.kotlin:kotlin-reflect") + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + + implementation("com.google.code.gson:gson:2.9.0") + implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") + implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") + implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") + + + implementation("no.iktdev:exfl:0.0.13-SNAPSHOT") + implementation(project(mapOf("path" to ":shared:kafka"))) + implementation(project(mapOf("path" to ":shared"))) + + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") +} + +tasks.test { + useJUnitPlatform() +} +kotlin { + jvmToolchain(17) +} \ No newline at end of file diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/Configuration.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/Configuration.kt similarity index 79% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/Configuration.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/Configuration.kt index 9d1c3e1a..1faeace8 100644 --- a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/Configuration.kt +++ b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/Configuration.kt @@ -1,12 +1,12 @@ package no.iktdev.streamit.content.ui +import no.iktdev.mediaprocessing.shared.socket.SocketImplementation import org.springframework.beans.factory.annotation.Value import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory import org.springframework.boot.web.server.WebServerFactoryCustomizer import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.messaging.simp.config.MessageBrokerRegistry -import org.springframework.stereotype.Controller import org.springframework.web.bind.annotation.RestController import org.springframework.web.method.HandlerTypePredicate import org.springframework.web.servlet.config.annotation.CorsRegistry @@ -48,20 +48,7 @@ class WebConfig: WebMvcConfigurer { } } +class SocketImplemented: SocketImplementation() { -@Configuration -@EnableWebSocketMessageBroker -class WebSocketConfig : WebSocketMessageBrokerConfigurer { - - override fun registerStompEndpoints(registry: StompEndpointRegistry) { - registry.addEndpoint("/ws") - .setAllowedOrigins("*://localhost:*/*", "http://localhost:3000/") - .withSockJS() - } - - override fun configureMessageBroker(registry: MessageBrokerRegistry) { - registry.enableSimpleBroker("/topic") - registry.setApplicationDestinationPrefixes("/app") - } } diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/UIApplication.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/UIApplication.kt similarity index 94% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/UIApplication.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/UIApplication.kt index d8b7d61e..501f9d9b 100644 --- a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/UIApplication.kt +++ b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/UIApplication.kt @@ -6,11 +6,11 @@ import no.iktdev.exfl.coroutines.Coroutines import no.iktdev.exfl.observable.ObservableMap import no.iktdev.exfl.observable.Observables import no.iktdev.exfl.observable.observableMapOf -import no.iktdev.streamit.content.common.CommonConfig +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEnv import no.iktdev.streamit.content.ui.dto.EventDataObject import no.iktdev.streamit.content.ui.dto.ExplorerItem import no.iktdev.streamit.content.ui.dto.SimpleEventDataObject -import no.iktdev.streamit.library.kafka.KafkaEnv import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.AdminClientConfig import org.springframework.boot.autoconfigure.SpringBootApplication @@ -54,7 +54,7 @@ fun main(args: Array) { )) val go = admincli.listConsumerGroupOffsets("${KafkaEnv.consumerId}:UIDataComposer") go.partitionsToOffsetAndMetadata().whenComplete { result, throwable -> - val partitions = result.entries.filter { it.key.topic() == CommonConfig.kafkaTopic } + val partitions = result.entries.filter { it.key.topic() == SharedConfig.kafkaTopic } .map { it.key } val deleteResult = admincli.deleteConsumerGroupOffsets("${KafkaEnv.consumerId}:UIDataComposer", partitions.toSet()) deleteResult.all().whenComplete { result, throwable -> diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/UIEnv.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/UIEnv.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/UIEnv.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/UIEnv.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/dto/EventDataDto.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/dto/EventDataDto.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/dto/EventDataDto.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/dto/EventDataDto.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerAttr.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerAttr.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerAttr.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerAttr.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerCursor.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerCursor.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerCursor.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/dto/ExplorerCursor.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/explorer/ExplorerCore.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/explorer/ExplorerCore.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/explorer/ExplorerCore.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/explorer/ExplorerCore.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/EventConsumer.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/EventConsumer.kt similarity index 93% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/EventConsumer.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/EventConsumer.kt index 89d89c72..e79c94a0 100644 --- a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/EventConsumer.kt +++ b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/EventConsumer.kt @@ -4,11 +4,9 @@ import com.google.gson.Gson import mu.KotlinLogging import no.iktdev.streamit.content.common.CommonConfig import no.iktdev.streamit.content.common.DefaultKafkaReader -import no.iktdev.streamit.content.ui.dto.EventDataObject import no.iktdev.streamit.content.ui.kafka.converter.EventDataConverter import no.iktdev.streamit.library.kafka.dto.Message import no.iktdev.streamit.library.kafka.listener.ManualAcknowledgeMessageListener -import no.iktdev.streamit.library.kafka.listener.SimpleMessageListener import org.apache.kafka.clients.consumer.ConsumerRecord import org.springframework.beans.factory.annotation.Autowired import org.springframework.kafka.listener.ContainerProperties diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataConverter.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataConverter.kt similarity index 96% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataConverter.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataConverter.kt index 7f40213d..34116853 100644 --- a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataConverter.kt +++ b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataConverter.kt @@ -1,10 +1,10 @@ package no.iktdev.streamit.content.ui.kafka.converter +import no.iktdev.mediaprocessing.shared.kafka.dto.Message import no.iktdev.streamit.content.ui.dto.EventDataObject import no.iktdev.streamit.content.ui.memActiveEventMap import no.iktdev.streamit.content.ui.kafka.EventConsumer import no.iktdev.streamit.content.ui.memSimpleConvertedEventsMap -import no.iktdev.streamit.library.kafka.dto.Message import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataDetailsSubConverter.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataDetailsSubConverter.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataDetailsSubConverter.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataDetailsSubConverter.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataEncodeSubConverter.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataEncodeSubConverter.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataEncodeSubConverter.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataEncodeSubConverter.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataFilenameAndTypeDeterminerSubConverter.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataFilenameAndTypeDeterminerSubConverter.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataFilenameAndTypeDeterminerSubConverter.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataFilenameAndTypeDeterminerSubConverter.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataMetadataSubConverter.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataMetadataSubConverter.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataMetadataSubConverter.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataMetadataSubConverter.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataSubConverterBase.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataSubConverterBase.kt similarity index 89% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataSubConverterBase.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataSubConverterBase.kt index 7f5420e3..3413ce07 100644 --- a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataSubConverterBase.kt +++ b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/kafka/converter/EventDataSubConverterBase.kt @@ -1,7 +1,6 @@ package no.iktdev.streamit.content.ui.kafka.converter import no.iktdev.streamit.content.ui.dto.EventDataObject -import no.iktdev.streamit.content.ui.dto.SimpleEventDataObject import no.iktdev.streamit.library.kafka.dto.Message abstract class EventDataSubConverterBase { diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/service/FileRegisterService.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/service/FileRegisterService.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/service/FileRegisterService.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/service/FileRegisterService.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/ExplorerTopic.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/ExplorerTopic.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/ExplorerTopic.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/ExplorerTopic.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/RequestTopic.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/RequestTopic.kt similarity index 93% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/RequestTopic.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/RequestTopic.kt index b939d2ce..dc122a17 100644 --- a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/RequestTopic.kt +++ b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/RequestTopic.kt @@ -4,7 +4,6 @@ import no.iktdev.streamit.content.common.CommonConfig import no.iktdev.streamit.library.kafka.KafkaEvents import no.iktdev.streamit.library.kafka.dto.Message import no.iktdev.streamit.library.kafka.dto.Status -import no.iktdev.streamit.library.kafka.dto.StatusType import no.iktdev.streamit.library.kafka.producer.DefaultProducer import org.springframework.beans.factory.annotation.Autowired import org.springframework.messaging.handler.annotation.MessageMapping @@ -25,7 +24,7 @@ class RequestTopic( if (file.exists()) { try { val message = Message( - status = Status(StatusType.SUCCESS), + status = Status(Status.SUCCESS), data = fullName ) messageProducer.sendMessage(KafkaEvents.REQUEST_FILE_READ.event, message) diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/TopicSupport.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/TopicSupport.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/TopicSupport.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/TopicSupport.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/UISocketService.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/UISocketService.kt similarity index 100% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/UISocketService.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/UISocketService.kt diff --git a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/internal/EncoderReaderService.kt b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/internal/EncoderReaderService.kt similarity index 97% rename from UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/internal/EncoderReaderService.kt rename to apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/internal/EncoderReaderService.kt index 57537ec2..e8b4eed6 100644 --- a/UI/src/main/kotlin/no/iktdev/streamit/content/ui/socket/internal/EncoderReaderService.kt +++ b/apps/ui/src/main/kotlin/no/iktdev/streamit/content/ui/socket/internal/EncoderReaderService.kt @@ -6,7 +6,6 @@ import mu.KotlinLogging import no.iktdev.streamit.content.common.dto.WorkOrderItem import no.iktdev.streamit.content.ui.UIEnv import no.iktdev.streamit.content.ui.dto.EventDataObject -import no.iktdev.streamit.content.ui.dto.SimpleEventDataObject import no.iktdev.streamit.content.ui.memActiveEventMap import no.iktdev.streamit.content.ui.memSimpleConvertedEventsMap import org.springframework.messaging.simp.stomp.StompFrameHandler diff --git a/UI/web/.gitignore b/apps/ui/web/.gitignore similarity index 100% rename from UI/web/.gitignore rename to apps/ui/web/.gitignore diff --git a/UI/web/README.md b/apps/ui/web/README.md similarity index 100% rename from UI/web/README.md rename to apps/ui/web/README.md diff --git a/UI/web/package-lock.json b/apps/ui/web/package-lock.json similarity index 100% rename from UI/web/package-lock.json rename to apps/ui/web/package-lock.json diff --git a/UI/web/package.json b/apps/ui/web/package.json similarity index 100% rename from UI/web/package.json rename to apps/ui/web/package.json diff --git a/UI/web/public/favicon.ico b/apps/ui/web/public/favicon.ico similarity index 100% rename from UI/web/public/favicon.ico rename to apps/ui/web/public/favicon.ico diff --git a/UI/web/public/index.html b/apps/ui/web/public/index.html similarity index 100% rename from UI/web/public/index.html rename to apps/ui/web/public/index.html diff --git a/UI/web/public/logo192.png b/apps/ui/web/public/logo192.png similarity index 100% rename from UI/web/public/logo192.png rename to apps/ui/web/public/logo192.png diff --git a/UI/web/public/logo512.png b/apps/ui/web/public/logo512.png similarity index 100% rename from UI/web/public/logo512.png rename to apps/ui/web/public/logo512.png diff --git a/UI/web/public/manifest.json b/apps/ui/web/public/manifest.json similarity index 100% rename from UI/web/public/manifest.json rename to apps/ui/web/public/manifest.json diff --git a/UI/web/public/robots.txt b/apps/ui/web/public/robots.txt similarity index 100% rename from UI/web/public/robots.txt rename to apps/ui/web/public/robots.txt diff --git a/apps/ui/web/src/App.css b/apps/ui/web/src/App.css new file mode 100644 index 00000000..e69de29b diff --git a/UI/web/src/App.test.tsx b/apps/ui/web/src/App.test.tsx similarity index 100% rename from UI/web/src/App.test.tsx rename to apps/ui/web/src/App.test.tsx diff --git a/UI/web/src/App.tsx b/apps/ui/web/src/App.tsx similarity index 100% rename from UI/web/src/App.tsx rename to apps/ui/web/src/App.tsx diff --git a/UI/web/src/app/features/CategorySidebar.tsx b/apps/ui/web/src/app/features/CategorySidebar.tsx similarity index 100% rename from UI/web/src/app/features/CategorySidebar.tsx rename to apps/ui/web/src/app/features/CategorySidebar.tsx diff --git a/UI/web/src/app/features/UxTc.tsx b/apps/ui/web/src/app/features/UxTc.tsx similarity index 100% rename from UI/web/src/app/features/UxTc.tsx rename to apps/ui/web/src/app/features/UxTc.tsx diff --git a/UI/web/src/app/features/footer.tsx b/apps/ui/web/src/app/features/footer.tsx similarity index 100% rename from UI/web/src/app/features/footer.tsx rename to apps/ui/web/src/app/features/footer.tsx diff --git a/UI/web/src/app/features/table.tsx b/apps/ui/web/src/app/features/table.tsx similarity index 100% rename from UI/web/src/app/features/table.tsx rename to apps/ui/web/src/app/features/table.tsx diff --git a/UI/web/src/app/hooks.ts b/apps/ui/web/src/app/hooks.ts similarity index 100% rename from UI/web/src/app/hooks.ts rename to apps/ui/web/src/app/hooks.ts diff --git a/UI/web/src/app/page/ExplorePage.tsx b/apps/ui/web/src/app/page/ExplorePage.tsx similarity index 100% rename from UI/web/src/app/page/ExplorePage.tsx rename to apps/ui/web/src/app/page/ExplorePage.tsx diff --git a/UI/web/src/app/page/LaunchPage.tsx b/apps/ui/web/src/app/page/LaunchPage.tsx similarity index 100% rename from UI/web/src/app/page/LaunchPage.tsx rename to apps/ui/web/src/app/page/LaunchPage.tsx diff --git a/UI/web/src/app/store.ts b/apps/ui/web/src/app/store.ts similarity index 100% rename from UI/web/src/app/store.ts rename to apps/ui/web/src/app/store.ts diff --git a/UI/web/src/app/store/composed-slice.ts b/apps/ui/web/src/app/store/composed-slice.ts similarity index 100% rename from UI/web/src/app/store/composed-slice.ts rename to apps/ui/web/src/app/store/composed-slice.ts diff --git a/UI/web/src/app/store/explorer-slice.ts b/apps/ui/web/src/app/store/explorer-slice.ts similarity index 100% rename from UI/web/src/app/store/explorer-slice.ts rename to apps/ui/web/src/app/store/explorer-slice.ts diff --git a/UI/web/src/app/store/kafka-items-flat-slice.ts b/apps/ui/web/src/app/store/kafka-items-flat-slice.ts similarity index 100% rename from UI/web/src/app/store/kafka-items-flat-slice.ts rename to apps/ui/web/src/app/store/kafka-items-flat-slice.ts diff --git a/UI/web/src/app/ws/client.ts b/apps/ui/web/src/app/ws/client.ts similarity index 100% rename from UI/web/src/app/ws/client.ts rename to apps/ui/web/src/app/ws/client.ts diff --git a/UI/web/src/app/ws/subscriptions.ts b/apps/ui/web/src/app/ws/subscriptions.ts similarity index 100% rename from UI/web/src/app/ws/subscriptions.ts rename to apps/ui/web/src/app/ws/subscriptions.ts diff --git a/UI/web/src/index.css b/apps/ui/web/src/index.css similarity index 100% rename from UI/web/src/index.css rename to apps/ui/web/src/index.css diff --git a/UI/web/src/index.tsx b/apps/ui/web/src/index.tsx similarity index 100% rename from UI/web/src/index.tsx rename to apps/ui/web/src/index.tsx diff --git a/UI/web/src/logo.svg b/apps/ui/web/src/logo.svg similarity index 100% rename from UI/web/src/logo.svg rename to apps/ui/web/src/logo.svg diff --git a/UI/web/src/react-app-env.d.ts b/apps/ui/web/src/react-app-env.d.ts similarity index 100% rename from UI/web/src/react-app-env.d.ts rename to apps/ui/web/src/react-app-env.d.ts diff --git a/UI/web/src/reportWebVitals.ts b/apps/ui/web/src/reportWebVitals.ts similarity index 100% rename from UI/web/src/reportWebVitals.ts rename to apps/ui/web/src/reportWebVitals.ts diff --git a/UI/web/src/setupTests.ts b/apps/ui/web/src/setupTests.ts similarity index 100% rename from UI/web/src/setupTests.ts rename to apps/ui/web/src/setupTests.ts diff --git a/UI/web/src/theme.d.ts b/apps/ui/web/src/theme.d.ts similarity index 100% rename from UI/web/src/theme.d.ts rename to apps/ui/web/src/theme.d.ts diff --git a/UI/web/src/theme.ts b/apps/ui/web/src/theme.ts similarity index 100% rename from UI/web/src/theme.ts rename to apps/ui/web/src/theme.ts diff --git a/UI/web/src/types.d.ts b/apps/ui/web/src/types.d.ts similarity index 100% rename from UI/web/src/types.d.ts rename to apps/ui/web/src/types.d.ts diff --git a/UI/web/tsconfig.json b/apps/ui/web/tsconfig.json similarity index 100% rename from UI/web/tsconfig.json rename to apps/ui/web/tsconfig.json diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 00000000..26accb8f --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,25 @@ +plugins { + id("java") + kotlin("plugin.spring") version "1.5.31" + kotlin("jvm") version "1.9.20" +} + +group = "no.iktdev.mediaprocessing" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") + implementation(kotlin("stdlib-jdk8")) +} + +tasks.test { + useJUnitPlatform() +} +kotlin { + jvmToolchain(17) +} \ No newline at end of file diff --git a/CommonCode/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from CommonCode/gradle/wrapper/gradle-wrapper.jar rename to gradle/wrapper/gradle-wrapper.jar diff --git a/UI/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties similarity index 80% rename from UI/gradle/wrapper/gradle-wrapper.properties rename to gradle/wrapper/gradle-wrapper.properties index b6881dae..a0df75aa 100644 --- a/UI/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Sat Jul 15 22:33:37 CEST 2023 +#Sun Nov 19 18:13:55 CET 2023 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/CommonCode/gradlew b/gradlew similarity index 100% rename from CommonCode/gradlew rename to gradlew diff --git a/CommonCode/gradlew.bat b/gradlew.bat similarity index 100% rename from CommonCode/gradlew.bat rename to gradlew.bat diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 00000000..1cc5e456 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,22 @@ +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "0.5.0" +} +rootProject.name = "MediaProcessing" +include("apps") +include("shared") +include("shared:kafka") +findProject(":shared:kafka")?.name = "kafka" +include("apps:coordinator") +findProject(":apps:coordinator")?.name = "coordinator" +include("apps:ui") +findProject(":apps:ui")?.name = "ui" +include("apps:encoder") +findProject(":apps:encoder")?.name = "encoder" +include("apps:converter") +findProject(":apps:converter")?.name = "converter" +include("shared:contract") +findProject(":shared:contract")?.name = "contract" +include("shared:common") +findProject(":shared:common")?.name = "common" +include("apps:processer") +findProject(":apps:processer")?.name = "processer" diff --git a/CommonCode/build.gradle.kts b/shared/build.gradle.kts similarity index 50% rename from CommonCode/build.gradle.kts rename to shared/build.gradle.kts index d32030ce..29e6d923 100644 --- a/CommonCode/build.gradle.kts +++ b/shared/build.gradle.kts @@ -1,8 +1,9 @@ plugins { - kotlin("jvm") version "1.8.21" + id("java") + kotlin("jvm") } -group = "no.iktdev.streamit.content" +group = "no.iktdev.mediaprocessing" version = "1.0-SNAPSHOT" repositories { @@ -16,26 +17,34 @@ repositories { } } +val exposedVersion = "0.44.0" dependencies { + implementation(kotlin("stdlib-jdk8")) + implementation("com.github.pgreze:kotlin-process:1.3.1") implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") - - implementation("no.iktdev.streamit.library:streamit-library-kafka:0.0.2-alpha84") implementation("no.iktdev:exfl:0.0.13-SNAPSHOT") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") implementation("com.google.code.gson:gson:2.8.9") implementation("org.json:json:20230227") + implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3") + + implementation("org.jetbrains.exposed:exposed-core:$exposedVersion") + implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion") + implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") + implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion") + implementation ("mysql:mysql-connector-java:8.0.29") + implementation(project(mapOf("path" to ":shared:kafka"))) - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") - - testImplementation("junit:junit:4.13.2") + testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") - testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1") - testImplementation("org.assertj:assertj-core:3.4.1") } tasks.test { useJUnitPlatform() +} +kotlin { + jvmToolchain(17) } \ No newline at end of file diff --git a/shared/common/build.gradle.kts b/shared/common/build.gradle.kts new file mode 100644 index 00000000..cd827c41 --- /dev/null +++ b/shared/common/build.gradle.kts @@ -0,0 +1,21 @@ +plugins { + id("java") + kotlin("jvm") + +} + +group = "no.iktdev.mediaprocessing.shared" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") +} + +tasks.test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DeserializingRegistry.kt b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DeserializingRegistry.kt new file mode 100644 index 00000000..56251e6e --- /dev/null +++ b/shared/common/src/main/kotlin/no/iktdev/mediaprocessing/shared/common/DeserializingRegistry.kt @@ -0,0 +1,34 @@ +package no.iktdev.mediaprocessing.shared.common + +class DeserializingRegistry { + companion object { + val deserializables = mutableListOf?>>( + KafkaEvents.EVENT_PROCESS_STARTED to ProcessStarted::class, + KafkaEvents.EVENT_MEDIA_READ_STREAM_PERFORMED to ReaderPerformed::class, + KafkaEvents.EVENT_MEDIA_PARSE_STREAM_PERFORMED to MediaStreamsParsePerformed::class, + KafkaEvents.EVENT_MEDIA_READ_BASE_INFO_PERFORMED to BaseInfoPerformed::class, + KafkaEvents.EVENT_MEDIA_METADATA_SEARCH_PERFORMED to MetadataPerformed::class, + KafkaEvents.EVENT_MEDIA_READ_OUT_NAME_AND_TYPE to null, + KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED to null, + KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED to null, + KafkaEvents.EVENT_MEDIA_CONVERT_PARAMETER_CREATED to null, + KafkaEvents.EVENT_MEDIA_DOWNLOAD_COVER_PARAMETER_CREATED to null, + + KafkaEvents.EVENT_WORK_ENCODE_CREATED to null, + KafkaEvents.EVENT_WORK_EXTRACT_CREATED to null, + KafkaEvents.EVENT_WORK_CONVERT_CREATED to null, + + KafkaEvents.EVENT_WORK_ENCODE_PERFORMED to null, + KafkaEvents.EVENT_WORK_EXTRACT_PERFORMED to null, + KafkaEvents.EVENT_WORK_CONVERT_PERFORMED to null, + KafkaEvents.EVENT_WORK_DOWNLOAD_COVER_PERFORMED to null, + + KafkaEvents.EVENT_WORK_ENCODE_SKIPPED to null, + KafkaEvents.EVENT_WORK_EXTRACT_SKIPPED to null, + KafkaEvents.EVENT_WORK_CONVERT_SKIPPED to null, + + + + ) + } +} \ No newline at end of file diff --git a/shared/contract/build.gradle.kts b/shared/contract/build.gradle.kts new file mode 100644 index 00000000..e73a6692 --- /dev/null +++ b/shared/contract/build.gradle.kts @@ -0,0 +1,20 @@ +plugins { + id("java") + kotlin("jvm") +} + +group = "no.iktdev.mediaprocessing.shared" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") +} + +tasks.test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ProcessType.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ProcessType.kt new file mode 100644 index 00000000..d30b8407 --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ProcessType.kt @@ -0,0 +1,6 @@ +package no.iktdev.mediaprocessing.shared.contract + +enum class ProcessType { + FLOW, + MANUAL +} \ No newline at end of file diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/AudioArgumentsDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/AudioArgumentsDto.kt new file mode 100644 index 00000000..8b739d48 --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/AudioArgumentsDto.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.contract.ffmpeg + +data class AudioArgumentsDto( + val index: Int, + val codecParameters: List = listOf("-acodec", "copy"), + val optionalParameters: List = listOf() + +) \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/MediaStreams.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/MediaStreams.kt similarity index 95% rename from CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/MediaStreams.kt rename to shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/MediaStreams.kt index 4808d6ab..59fe8949 100644 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/streams/MediaStreams.kt +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/MediaStreams.kt @@ -1,4 +1,10 @@ -package no.iktdev.streamit.content.common.streams +package no.iktdev.mediaprocessing.shared.contract.ffmpeg + +data class ParsedMediaStreams( + val videoStream: List = listOf(), + val audioStream: List = listOf(), + val subtitleStream: List = listOf() +) data class MediaStreams( val streams: List diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/PreferenceDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/PreferenceDto.kt new file mode 100644 index 00000000..86069c4f --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/PreferenceDto.kt @@ -0,0 +1,43 @@ +package no.iktdev.mediaprocessing.shared.contract.ffmpeg + +data class PreferenceDto( + val encodePreference: EncodingPreference = EncodingPreference(video = VideoPreference(), audio = AudioPreference()), + val convertPreference: ConvertPreference = ConvertPreference() +) + + +data class EncodingPreference( + val video: VideoPreference, + val audio: AudioPreference +) + +enum class SubtitleTypes { + SRT, + VTT, + SMI +} + +data class ConvertPreference( + val cleanup: Boolean = true, + val merge: Boolean = false, + val subtitleTypes: List = listOf( + SubtitleTypes.SRT, SubtitleTypes.VTT, SubtitleTypes.SMI + ) +) + +data class VideoPreference( + val codec: String = "h264", + val pixelFormat: String = "yuv420p", + val pixelFormatPassthrough: List = listOf("yuv420p", "yuv420p10le"), + val threshold: Int = 16 +) + +data class AudioPreference( + val codec: String = "aac", + val sample_rate: Int? = null, + val channels: Int? = null, + val language: String = "eng", //ISO3 format + val preserveChannels: Boolean = true, + val defaultToEAC3OnSurroundDetected: Boolean = true, + val forceStereo: Boolean = false +) diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/SubtitleArgumentsDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/SubtitleArgumentsDto.kt new file mode 100644 index 00000000..1df684bc --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/SubtitleArgumentsDto.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.contract.ffmpeg + +data class SubtitleArgumentsDto( + val index: Int, + val language: String, + val format: String, // Extension as well + val codecParameters: List = listOf("-c:s", "copy"), + val optionalParameters: List = listOf() +) \ No newline at end of file diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/VideoAndAudioDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/VideoAndAudioDto.kt new file mode 100644 index 00000000..dc0d832d --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/VideoAndAudioDto.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.contract.ffmpeg + + +data class VideoAndAudioDto( + val video: VideoArgumentsDto, + val audio: AudioArgumentsDto, + val outFile: String // Absolute path to file +) \ No newline at end of file diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/VideoArgumentsDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/VideoArgumentsDto.kt new file mode 100644 index 00000000..f98ec404 --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/ffmpeg/VideoArgumentsDto.kt @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing.shared.contract.ffmpeg + +data class VideoArgumentsDto( + val index: Int, + val codecParameters: List = listOf("-vcodec", "copy"), + val optionalParameters: List = listOf() +) \ No newline at end of file diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/MediaProcessedDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/MediaProcessedDto.kt new file mode 100644 index 00000000..74a8b9c6 --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/MediaProcessedDto.kt @@ -0,0 +1,11 @@ +package no.iktdev.mediaprocessing.shared.contract.reader + +import no.iktdev.mediaprocessing.shared.contract.ProcessType + +data class MediaProcessedDto( + val referenceId: String, + val process: ProcessType?, + val inputFile: String?, + val metadata: MetadataDto?, + val outputFiles: OutputFilesDto? +) \ No newline at end of file diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/MetadataDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/MetadataDto.kt new file mode 100644 index 00000000..11af4a1b --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/MetadataDto.kt @@ -0,0 +1,15 @@ +package no.iktdev.mediaprocessing.shared.contract.reader + +data class MetadataDto( + val title: String, + val type: String, + val cover: MetadataCoverDto, + val summary: String, + val genres: List +) + +data class MetadataCoverDto( + val cover: String, + val coverUrl: String, + val coverFile: String +) \ No newline at end of file diff --git a/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/OutputFilesDto.kt b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/OutputFilesDto.kt new file mode 100644 index 00000000..af00615a --- /dev/null +++ b/shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/OutputFilesDto.kt @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing.shared.contract.reader + +class OutputFilesDto( + val videoFile: String, + val videoArguments: List, + val subtitleFiles: List +) \ No newline at end of file diff --git a/shared/kafka/build.gradle.kts b/shared/kafka/build.gradle.kts new file mode 100644 index 00000000..c772f154 --- /dev/null +++ b/shared/kafka/build.gradle.kts @@ -0,0 +1,44 @@ +plugins { + id("java") + kotlin("jvm") +} + +group = "no.iktdev.mediaprocessing.shared" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + implementation(kotlin("stdlib-jdk8")) + implementation("com.google.code.gson:gson:2.8.9") + implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") + + implementation("org.springframework.kafka:spring-kafka:3.0.1") + implementation("com.fasterxml.jackson.core:jackson-databind:2.13.0") + implementation(project(mapOf("path" to ":shared:contract"))) + + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + implementation("org.jetbrains.kotlin:kotlin-reflect") + implementation("io.github.classgraph:classgraph:4.8.165") + + testImplementation("org.springframework.kafka:spring-kafka-test:3.0.1") + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("junit:junit:4.13.2") + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") + testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1") + testImplementation("org.assertj:assertj-core:3.4.1") + testImplementation("org.mockito:mockito-core:3.+") + testImplementation("org.assertj:assertj-core:3.4.1") + +} + +tasks.test { + useJUnitPlatform() +} +kotlin { + jvmToolchain(17) +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/AnnotationFinder.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/AnnotationFinder.kt new file mode 100644 index 00000000..e8f19ee8 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/AnnotationFinder.kt @@ -0,0 +1,39 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +import java.io.File +import kotlin.reflect.KClass + +class AnnotationFinder { + fun getClassesWithAnnotation(packageName: String, annotation: KClass): List> { + val packageToScan = packageName.replace('.', '/') + val classLoader = Thread.currentThread().contextClassLoader + val resources = classLoader.getResources(packageToScan) + + val classes = mutableListOf>() + + while (resources.hasMoreElements()) { + val resource = resources.nextElement() + if (resource.protocol == "file") { + val file = File(resource.file) + if (file.isDirectory) { + val classNames = file.walkTopDown().filter { it.isFile && it.extension == "class" } + .map { it.toRelativeString(file).removeSuffix(".class").replace('/', '.') } + .toList() + + classNames.forEach { className -> + try { + val loadedClass = Class.forName(className).kotlin + if (loadedClass.annotations.any { it.annotationClass == annotation }) { + classes.add(loadedClass) + } + } catch (e: ClassNotFoundException) { + // Handle exception if needed + } + } + } + } + } + + return classes + } +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultConsumer.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultConsumer.kt new file mode 100644 index 00000000..9c8e47d1 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultConsumer.kt @@ -0,0 +1,97 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import mu.KotlinLogging +import no.iktdev.mediaprocessing.shared.contract.ffmpeg.ParsedMediaStreams +import no.iktdev.mediaprocessing.shared.kafka.dto.Message +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.* +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.common.serialization.StringDeserializer +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory +import org.springframework.kafka.core.DefaultKafkaConsumerFactory +import org.springframework.kafka.listener.ContainerProperties.AckMode +import kotlin.reflect.full.findAnnotation +import java.util.UUID +import kotlin.reflect.KClass + +open class DefaultConsumer(val subId: String = UUID.randomUUID().toString()) { + val log = KotlinLogging.logger {} + + var autoCommit: Boolean = true + var ackModeOverride: AckMode? = null + + fun consumerFactory(): DefaultKafkaConsumerFactory { + val config: MutableMap = HashMap() + config[ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG] = KafkaEnv.servers + config[ConsumerConfig.GROUP_ID_CONFIG] = "${KafkaEnv.consumerId}:$subId" + config[ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG] = StringDeserializer::class.java + config[ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG] = StringDeserializer::class.java + config[ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG] = autoCommit + config[ConsumerConfig.AUTO_OFFSET_RESET_CONFIG] = KafkaEnv.loadMessages + + return DefaultKafkaConsumerFactory(config, StringDeserializer(), StringDeserializer()) + + } + + fun consumerFactoryListener(): ConcurrentKafkaListenerContainerFactory { + val factory = ConcurrentKafkaListenerContainerFactory() + factory.consumerFactory = consumerFactory() + ackModeOverride?.let { + factory.containerProperties.ackMode = it + } + + return factory + } + + class GsonDeserializer : org.apache.kafka.common.serialization.Deserializer> { + private val gson = Gson() + val log = KotlinLogging.logger {} + + + fun getAnnotatedClasses(): List>> { + val classesWithAnnotation = AnnotationFinder().getClassesWithAnnotation("no.iktdev.mediaprocessing.shared.kafka.dto.events_result", KafkaBelongsToEvent::class) + .mapNotNull { clazz -> + val annotation = clazz.findAnnotation() + annotation?.event?.let { kafkaEvent -> + kafkaEvent to clazz + } + } + + classesWithAnnotation.forEach { (event, clazz) -> + println("Event: $event, Class: $clazz") + } + return classesWithAnnotation + + } + + override fun configure(configs: MutableMap?, isKey: Boolean) { + // Ingen ekstra konfigurasjon kreves + } + + override fun deserialize(topic: String, data: ByteArray): Message { + val jsonString = try { String(data) } catch (e: Exception) {e.printStackTrace(); null} + return deserialiseJsonString(jsonString) + } + + fun deserialiseJsonString(json: String?): Message { + if (json.isNullOrBlank()) { + log.error { "Data is null or empty" } + } + try { + val type = object : TypeToken>() {}.type + return gson.fromJson>(json, Message::class.java) + } catch (e: Exception) { + e.printStackTrace() + } + val type = object : TypeToken>() {}.type + return gson.fromJson(json, type) + } + + override fun close() { + // Ingen ressurser å lukke + } + } + +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultMessageListener.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultMessageListener.kt new file mode 100644 index 00000000..fc371420 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultMessageListener.kt @@ -0,0 +1,75 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +import mu.KotlinLogging +import no.iktdev.mediaprocessing.shared.kafka.dto.DeserializedConsumerRecord +import no.iktdev.mediaprocessing.shared.kafka.dto.Message +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.springframework.kafka.listener.ContainerProperties +import org.springframework.kafka.listener.KafkaMessageListenerContainer +import org.springframework.kafka.listener.MessageListener +import java.lang.IllegalArgumentException +import java.util.* + +open class DefaultMessageListener( + open val topic: String, + open val consumer: DefaultConsumer = DefaultConsumer(subId = UUID.randomUUID().toString()), + open var onMessageReceived: (DeserializedConsumerRecord>) -> Unit = {} +) + : MessageListener { + + + private val logger = KotlinLogging.logger {} + private val deserializer = DeserializingRegistry() + + protected var container: KafkaMessageListenerContainer? = null + + fun listen() { + val listener = consumer.consumerFactoryListener() + val containerProperties = ContainerProperties(topic).apply { + messageListener = this@DefaultMessageListener + } + container = KafkaMessageListenerContainer(listener.consumerFactory, containerProperties) + container?.start() + logger.info { "Listening to topic $topic" } + } + + fun stop() { + container?.stop() + container = null + } + + fun resume() = container?.resume() + fun pause() = container?.pause() + fun isPaused() = container?.isContainerPaused + fun isRunning() = container?.isRunning + + override fun onMessage(data: ConsumerRecord) { + val event = try { + KafkaEvents.valueOf(data.key()) + } catch (e: IllegalArgumentException) { + logger.error { "${data.key()} is not a member of KafkaEvents" } + null + } + event?.let { + val deserialized = deserializer.deserialize(it, data.value()) + val dz = data.toDeserializedConsumerRecord(it, deserialized) + onMessageReceived(dz) + } + } + +} + +private fun ConsumerRecord.toDeserializedConsumerRecord(keyzz: KDez, valuezz: VDez): DeserializedConsumerRecord { + return DeserializedConsumerRecord( + topic = this.topic(), + partition = this.partition(), + offset = this.offset(), + timestamp = this.timestamp(), + timestampType = this.timestampType(), + headers = this.headers(), + key = keyzz, + value = valuezz, + leaderEpoch = this.leaderEpoch().orElse(null) + ) +} diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultProducer.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultProducer.kt new file mode 100644 index 00000000..49140478 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DefaultProducer.kt @@ -0,0 +1,39 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +import com.google.gson.Gson +import no.iktdev.mediaprocessing.shared.kafka.dto.Message +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import org.apache.kafka.clients.producer.ProducerConfig +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.serialization.StringSerializer +import org.springframework.kafka.core.DefaultKafkaProducerFactory +import org.springframework.kafka.core.KafkaTemplate +import org.springframework.kafka.core.ProducerFactory + +open class DefaultProducer(val topic: String) { + private val producerFactory: ProducerFactory + + init { + val config: MutableMap = HashMap() + config[ProducerConfig.BOOTSTRAP_SERVERS_CONFIG] = KafkaEnv.servers + config[ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java + config[ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java + + producerFactory = DefaultKafkaProducerFactory(config) + } + + fun createKafkaTemplate(): KafkaTemplate { + return KafkaTemplate(producerFactory) + } + + fun sendMessage(key: String, message: Message) { + val kafkaTemplate = createKafkaTemplate() + val serializedMessage = serializeMessage(message) + kafkaTemplate.send(ProducerRecord(topic, key, serializedMessage)) + } + + private fun serializeMessage(message: Message): String { + val gson = Gson() + return gson.toJson(message) + } +} diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DeserializingRegistry.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DeserializingRegistry.kt new file mode 100644 index 00000000..4d015808 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/DeserializingRegistry.kt @@ -0,0 +1,72 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import no.iktdev.mediaprocessing.shared.kafka.dto.Message +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.* +import java.lang.reflect.Type +import kotlin.reflect.KClass + +class DeserializingRegistry { + companion object { + val deserializables = mutableMapOf( + KafkaEvents.EVENT_PROCESS_STARTED to ProcessStarted::class.java, + KafkaEvents.EVENT_MEDIA_READ_STREAM_PERFORMED to ReaderPerformed::class.java, + KafkaEvents.EVENT_MEDIA_PARSE_STREAM_PERFORMED to MediaStreamsParsePerformed::class.java, + KafkaEvents.EVENT_MEDIA_READ_BASE_INFO_PERFORMED to BaseInfoPerformed::class.java, + KafkaEvents.EVENT_MEDIA_METADATA_SEARCH_PERFORMED to MetadataPerformed::class.java, + KafkaEvents.EVENT_MEDIA_READ_OUT_NAME_AND_TYPE to null, + KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED to null, + KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED to null, + KafkaEvents.EVENT_MEDIA_CONVERT_PARAMETER_CREATED to null, + KafkaEvents.EVENT_MEDIA_DOWNLOAD_COVER_PARAMETER_CREATED to null, + + KafkaEvents.EVENT_WORK_ENCODE_CREATED to null, + KafkaEvents.EVENT_WORK_EXTRACT_CREATED to null, + KafkaEvents.EVENT_WORK_CONVERT_CREATED to null, + + KafkaEvents.EVENT_WORK_ENCODE_PERFORMED to null, + KafkaEvents.EVENT_WORK_EXTRACT_PERFORMED to null, + KafkaEvents.EVENT_WORK_CONVERT_PERFORMED to null, + KafkaEvents.EVENT_WORK_DOWNLOAD_COVER_PERFORMED to null, + + KafkaEvents.EVENT_WORK_ENCODE_SKIPPED to null, + KafkaEvents.EVENT_WORK_EXTRACT_SKIPPED to null, + KafkaEvents.EVENT_WORK_CONVERT_SKIPPED to null, + ) + } + + fun deserialize(event: KafkaEvents, json: String): Message { + val gson = Gson() + val dezClazz = deserializables[event] + dezClazz?.let { eventClass -> + try { + val type = TypeToken.getParameterized(Message::class.java, eventClass).type + return gson.fromJson>(json, type) + } catch (e: Exception) { + e.printStackTrace() + } + } + // Fallback + val type = object : TypeToken>() {}.type + return gson.fromJson>(json, type) + } + + fun deserializeData(event: KafkaEvents, json: String): MessageDataWrapper { + val gson = Gson() + val dezClazz = deserializables[event] + dezClazz?.let { eventClass -> + try { + val type = TypeToken.getParameterized(eventClass).type + return gson.fromJson(json, type) + } catch (e: Exception) { + e.printStackTrace() + } + } + // Fallback + val type = object : TypeToken() {}.type + return gson.fromJson(json, type) + } + +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaBelongsToEvent.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaBelongsToEvent.kt new file mode 100644 index 00000000..dac3e4ed --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaBelongsToEvent.kt @@ -0,0 +1,8 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import java.lang.annotation.ElementType + +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.CLASS) +annotation class KafkaBelongsToEvent(vararg val event: KafkaEvents) diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaEnv.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaEnv.kt new file mode 100644 index 00000000..bdf01c91 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaEnv.kt @@ -0,0 +1,12 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +import java.util.UUID + +class KafkaEnv { + companion object { + val servers: String = System.getenv("KAFKA_BOOTSTRAP_SERVER") ?: "127.0.0.1:9092" + var consumerId: String = System.getenv("KAFKA_CONSUMER_ID") ?: "LibGenerated-${UUID.randomUUID()}" + var enabled: Boolean = System.getenv("KAFKA_ENABLED").toBoolean() + val loadMessages: String = System.getenv("KAFKA_MESSAGES_USE") ?: "earliest" + } +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaEvents.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaEvents.kt new file mode 100644 index 00000000..2b071c83 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/core/KafkaEvents.kt @@ -0,0 +1,41 @@ +package no.iktdev.mediaprocessing.shared.kafka.core + +enum class KafkaEvents(val event: String) { + EVENT_PROCESS_STARTED("event:process:started"), + + EVENT_MEDIA_READ_STREAM_PERFORMED("event:media-read-stream:performed"), + EVENT_MEDIA_PARSE_STREAM_PERFORMED("event:media-parse-stream:performed"), + EVENT_MEDIA_READ_BASE_INFO_PERFORMED("event:media-read-base-info:performed"), + EVENT_MEDIA_METADATA_SEARCH_PERFORMED("event:media-metadata-search:performed"), + EVENT_MEDIA_READ_OUT_NAME_AND_TYPE("event:media-read-out-name-and-type:performed"), + + EVENT_MEDIA_ENCODE_PARAMETER_CREATED("event:media-encode-parameter:created"), + EVENT_MEDIA_EXTRACT_PARAMETER_CREATED("event:media-extract-parameter:created"), + EVENT_MEDIA_CONVERT_PARAMETER_CREATED("event:media-convert-parameter:created"), + EVENT_MEDIA_DOWNLOAD_COVER_PARAMETER_CREATED("event:media-download-cover-parameter:created"), + + EVENT_WORK_ENCODE_CREATED("event:work-encode:created"), + EVENT_WORK_EXTRACT_CREATED("event:work-extract:created"), + EVENT_WORK_CONVERT_CREATED("event:work-convert:created"), + + EVENT_WORK_ENCODE_PERFORMED("event:work-encode:performed"), + EVENT_WORK_EXTRACT_PERFORMED("event:work-extract:performed"), + EVENT_WORK_CONVERT_PERFORMED("event:work-convert:performed"), + EVENT_WORK_DOWNLOAD_COVER_PERFORMED("event:work-download-cover:performed"), + + + EVENT_WORK_ENCODE_SKIPPED("event:work-encode:skipped"), + EVENT_WORK_EXTRACT_SKIPPED("event:work-extract:skipped"), + EVENT_WORK_CONVERT_SKIPPED("event:work-convert:skipped"), + + + EVENT_STORE_VIDEO_PERFORMED("event:store-video:performed"), + EVENT_STORE_SUBTITLE_PERFORMED("event:store-subtitle:performed"), + EVENT_STORE_COVER_PERFORMED("event:store-cover:performed"), + EVENT_STORE_METADATA_PERFORMED("event:store-metadata:performed"), + + EVENT_PROCESS_COMPLETED("event:process:completed"), +} +fun toEvent(event: String): KafkaEvents? { + return KafkaEvents.entries.find { it.event == event } +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/CollectionReference.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/CollectionReference.kt new file mode 100644 index 00000000..453d6543 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/CollectionReference.kt @@ -0,0 +1,9 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto + +import java.util.* + +open class CollectionReference( + @Transient open val referenceId: String = UUID.randomUUID().toString(), +) { + +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/DeserializedConsumerRecord.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/DeserializedConsumerRecord.kt new file mode 100644 index 00000000..e782efd9 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/DeserializedConsumerRecord.kt @@ -0,0 +1,16 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto + +import org.apache.kafka.common.header.Headers +import org.apache.kafka.common.record.TimestampType + +data class DeserializedConsumerRecord( + val topic: String, + val partition: Int, + val offset: Long, + val timestamp: Long, + val timestampType: TimestampType, + val headers: Headers, + val key: K, + val value: V, + val leaderEpoch: Int? +) diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/Message.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/Message.kt new file mode 100644 index 00000000..8d7b55dd --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/Message.kt @@ -0,0 +1,38 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto + +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import no.iktdev.streamit.library.kafka.dto.Status +import java.lang.reflect.Type +import java.util.* + +open class Message( + override val referenceId: String = UUID.randomUUID().toString(), + val eventId: String = UUID.randomUUID().toString(), + val data: C? = null +): CollectionReference() { + + fun dataAsJson(): String = Gson().toJson(this.data) + + fun dataAs(clazz: C): C? { + return try { + val typeToken = object : TypeToken() {}.type + val gson = Gson() + val json: String = gson.toJson(data) + gson.fromJson(json, typeToken) + } catch (e: Exception) { + e.printStackTrace() + null + } + } + + fun dataAs(type: Type): C? { + return try { + val gson = Gson() + val json = dataAsJson() + gson.fromJson(json, type) + } catch (e: Exception) { + null + } + } +} diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/MessageDataWrapper.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/MessageDataWrapper.kt new file mode 100644 index 00000000..efd7af47 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/MessageDataWrapper.kt @@ -0,0 +1,19 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto + +import com.google.gson.Gson +import no.iktdev.streamit.library.kafka.dto.Status +import java.io.Serializable +import java.lang.reflect.Type +import java.util.* + + +open class MessageDataWrapper( + @Transient open val status: Status = Status.ERROR, + @Transient open val message: String? = null +) + + + +fun MessageDataWrapper?.isSuccess(): Boolean { + return this != null && this.status != Status.ERROR +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/Status.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/Status.kt new file mode 100644 index 00000000..84544f6b --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/Status.kt @@ -0,0 +1,7 @@ +package no.iktdev.streamit.library.kafka.dto + +enum class Status { + STARTED, + COMPLETED, + ERROR +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/BaseInfoPerformed.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/BaseInfoPerformed.kt new file mode 100644 index 00000000..adb3cc36 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/BaseInfoPerformed.kt @@ -0,0 +1,17 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_MEDIA_READ_BASE_INFO_PERFORMED) +data class BaseInfoPerformed( + override val status: Status, + val title: String, + val sanitizedName: String +) : MessageDataWrapper(status) + +fun BaseInfoPerformed?.hasValidData(): Boolean { + return this != null && this.title.isNotBlank() && this.sanitizedName.isNotBlank() +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ConvertWorkerRequest.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ConvertWorkerRequest.kt new file mode 100644 index 00000000..90931a9d --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ConvertWorkerRequest.kt @@ -0,0 +1,14 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper + +@KafkaBelongsToEvent(KafkaEvents.EVENT_WORK_CONVERT_CREATED) +data class ConvertWorkerRequest( + val requiresEventId: String, + val inputFile: String, + val allowOverwrite: Boolean, + val outFileBaseName: String, + val outDirectory: String +): MessageDataWrapper() \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/CoverInfoPerformed.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/CoverInfoPerformed.kt new file mode 100644 index 00000000..0fe775e7 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/CoverInfoPerformed.kt @@ -0,0 +1,12 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +data class CoverInfoPerformed( + override val status: Status, + val url: String, + val outDir: String, + val outFileBaseName: String +) + : MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/FfmpegWorkRequestCreated.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/FfmpegWorkRequestCreated.kt new file mode 100644 index 00000000..b7c968ce --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/FfmpegWorkRequestCreated.kt @@ -0,0 +1,15 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper + +@KafkaBelongsToEvent( + KafkaEvents.EVENT_WORK_ENCODE_CREATED, + KafkaEvents.EVENT_WORK_EXTRACT_CREATED +) +data class FfmpegWorkRequestCreated( + val inputFile: String, + val arguments: List, + val outFile: String +): MessageDataWrapper() \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/FfmpegWorkerArgumentsCreated.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/FfmpegWorkerArgumentsCreated.kt new file mode 100644 index 00000000..50dec32d --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/FfmpegWorkerArgumentsCreated.kt @@ -0,0 +1,28 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +/** + * @param status Status type + * @param inputFile File.absolutePath + * @param outputFile File.absolutePath + * @param arguments Requires arguments, instructions for what ffmpeg should do + */ +@KafkaBelongsToEvent( + KafkaEvents.EVENT_MEDIA_ENCODE_PARAMETER_CREATED, + KafkaEvents.EVENT_MEDIA_EXTRACT_PARAMETER_CREATED +) +data class FfmpegWorkerArgumentsCreated( + override val status: Status, + val inputFile: String, // absolutePath + val entries: List +): + MessageDataWrapper(status) + +data class FfmpegWorkerArgument( + val outputFile: String, + val arguments: List +) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaConvertInfo.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaConvertInfo.kt new file mode 100644 index 00000000..dc7c23ae --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaConvertInfo.kt @@ -0,0 +1,13 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_WORK_CONVERT_CREATED) +data class MediaConvertInfo( + override val status: Status, + val arguments: List> + +): MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaEncodeInfo.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaEncodeInfo.kt new file mode 100644 index 00000000..20f1c5f7 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaEncodeInfo.kt @@ -0,0 +1,13 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_WORK_ENCODE_CREATED) +data class MediaEncodeInfo( + override val status: Status, + val arguments: List +) : + MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaExtractInfo.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaExtractInfo.kt new file mode 100644 index 00000000..9fd7b49a --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaExtractInfo.kt @@ -0,0 +1,12 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_WORK_EXTRACT_CREATED) +data class MediaExtractInfo( + override val status: Status, + val arguments: List> +) : MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaStreamsParsePerformed.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaStreamsParsePerformed.kt new file mode 100644 index 00000000..e8ee2a9a --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaStreamsParsePerformed.kt @@ -0,0 +1,13 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_MEDIA_PARSE_STREAM_PERFORMED) +data class MediaStreamsParsePerformed( + override val status: Status, + val parsedAsJson: String + +): MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MetadataPerformed.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MetadataPerformed.kt new file mode 100644 index 00000000..b9c41a2a --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MetadataPerformed.kt @@ -0,0 +1,22 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_MEDIA_METADATA_SEARCH_PERFORMED) +data class MetadataPerformed( + override val status: Status, + override val message: String? = null, + val data: pyMetadata? = null + ) : MessageDataWrapper(status, message) + +data class pyMetadata( + val title: String, + val altTitle: List = emptyList(), + val cover: String? = null, + val type: String, + val summary: String? = null, + val genres: List = emptyList() +) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ProcessCompleted.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ProcessCompleted.kt new file mode 100644 index 00000000..4e12bc5a --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ProcessCompleted.kt @@ -0,0 +1,10 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_PROCESS_COMPLETED) +data class ProcessCompleted(override val status: Status) : MessageDataWrapper(status) { +} \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ProcessStarted.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ProcessStarted.kt new file mode 100644 index 00000000..0f250c2e --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ProcessStarted.kt @@ -0,0 +1,14 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.contract.ProcessType +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_PROCESS_STARTED) +data class ProcessStarted( + override val status: Status, + val type: ProcessType = ProcessType.FLOW, + val file: String // AbsolutePath +) : MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ReaderPerformed.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ReaderPerformed.kt new file mode 100644 index 00000000..033e97ff --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/ReaderPerformed.kt @@ -0,0 +1,13 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaBelongsToEvent +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +@KafkaBelongsToEvent(KafkaEvents.EVENT_MEDIA_READ_STREAM_PERFORMED) +data class ReaderPerformed( + override val status: Status, + val file: String, //AbsolutePath + val output: String +) : MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/VideoInfoPerformed.kt b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/VideoInfoPerformed.kt new file mode 100644 index 00000000..e94ce7d9 --- /dev/null +++ b/shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/VideoInfoPerformed.kt @@ -0,0 +1,34 @@ +package no.iktdev.mediaprocessing.shared.kafka.dto.events_result + +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +data class VideoInfoPerformed( + override val status: Status, + val data: VideoInfo +) + : MessageDataWrapper(status) + + +data class EpisodeInfo( + val title: String, + val episode: Int, + val season: Int, + val episodeTitle: String?, + override val fullName: String +): VideoInfo(fullName) + +data class MovieInfo( + val title: String, + override val fullName: String +) : VideoInfo(fullName) + +data class SubtitleInfo( + val inputFile: String, + val collection: String, + val language: String +) + +abstract class VideoInfo( + @Transient open val fullName: String +) \ No newline at end of file diff --git a/shared/kafka/src/test/kotlin/SerializationTest.kt b/shared/kafka/src/test/kotlin/SerializationTest.kt new file mode 100644 index 00000000..c587e964 --- /dev/null +++ b/shared/kafka/src/test/kotlin/SerializationTest.kt @@ -0,0 +1,47 @@ +import com.fasterxml.jackson.databind.ObjectMapper +import com.google.gson.Gson +import no.iktdev.mediaprocessing.shared.kafka.core.DefaultConsumer +import no.iktdev.mediaprocessing.shared.kafka.dto.Message +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status +import org.junit.jupiter.api.Test +import org.assertj.core.api.Assertions.assertThat + + +class SerializationTest { + + @Test + fun serialize() { + val gson = Gson() + val message = Message( + "d2fb1472-ebdd-4fce-9ffd-7202a1ad911d", + "01e4420d-f7ab-49b5-ac5b-8b0f4f4a600e", + data = MockData( + Status.COMPLETED, + "Test" + )) + + val json = gson.toJson(message) + val objectMapper = ObjectMapper() + val result = objectMapper.readValue(json, Message::class.java) + assertThat(result.data).isInstanceOf(MockData::class.java) + + + } + + @Test + fun getAnnotatedClasses() { + val serializer = DefaultConsumer.GsonDeserializer() + val result = serializer.getAnnotatedClasses() + assertThat(result).isNotEmpty() + } + + + +} + +data class MockData( + override val status: Status, + val tekst: String + +): MessageDataWrapper(status) \ No newline at end of file diff --git a/shared/src/main/kotlin/Utils.kt b/shared/src/main/kotlin/Utils.kt new file mode 100644 index 00000000..afd0d955 --- /dev/null +++ b/shared/src/main/kotlin/Utils.kt @@ -0,0 +1,19 @@ +import mu.KotlinLogging +import java.io.File +import java.io.RandomAccessFile + +private val logger = KotlinLogging.logger {} + +fun isFileAvailable(file: File): Boolean { + if (!file.exists()) return false + var stream: RandomAccessFile? = null + try { + stream = RandomAccessFile(file, "rw") + stream.close() + logger.info { "File ${file.name} is read and writable" } + return true + } catch (e: Exception) { + stream?.close() + } + return false +} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/Downloader.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/DownloadClient.kt similarity index 95% rename from CommonCode/src/main/java/no/iktdev/streamit/content/common/Downloader.kt rename to shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/DownloadClient.kt index 19023a23..82687d86 100644 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/Downloader.kt +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/DownloadClient.kt @@ -1,13 +1,12 @@ -package no.iktdev.streamit.content.common +package no.iktdev.mediaprocessing.shared import no.iktdev.exfl.using import java.io.File import java.io.FileOutputStream import java.net.HttpURLConnection import java.net.URL -import kotlin.math.sign -open class Downloader(val url: String, val outDir: File, val baseName: String) { +open class DownloadClient(val url: String, val outDir: File, val baseName: String) { protected val http: HttpURLConnection = openConnection() private val BUFFER_SIZE = 4096 diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/Preference.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/Preference.kt new file mode 100644 index 00000000..7236054e --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/Preference.kt @@ -0,0 +1,55 @@ +package no.iktdev.mediaprocessing.shared + +import com.google.gson.Gson +import mu.KotlinLogging +import no.iktdev.mediaprocessing.shared.dto.PreferenceDto +import org.slf4j.LoggerFactory + +private val log = KotlinLogging.logger {} +class Preference { + + companion object { + fun getPreference(): PreferenceDto { + val preference = readPreferenceFromFile() ?: PreferenceDto() + log.info { "[Audio]: Codec = " + preference.encodePreference.audio.codec } + log.info { "[Audio]: Language = " + preference.encodePreference.audio.language } + log.info { "[Audio]: Channels = " + preference.encodePreference.audio.channels } + log.info { "[Audio]: Sample rate = " + preference.encodePreference.audio.sample_rate } + log.info { "[Audio]: Use EAC3 for surround = " + preference.encodePreference.audio.defaultToEAC3OnSurroundDetected } + + log.info { "[Video]: Codec = " + preference.encodePreference.video.codec } + log.info { "[Video]: Pixel format = " + preference.encodePreference.video.pixelFormat } + log.info { "[Video]: Pixel format pass-through = " + preference.encodePreference.video.pixelFormatPassthrough.joinToString(", ") } + log.info { "[Video]: Threshold = " + preference.encodePreference.video.threshold } + + return preference + } + + private fun readPreferenceFromFile(): PreferenceDto? { + val prefFile = SharedConfig.preference + if (!prefFile.exists()) { + log.info("Preference file: ${prefFile.absolutePath} does not exists...") + log.info("Using default configuration") + return null + } + else { + log.info("Preference file: ${prefFile.absolutePath} found") + } + + return try { + val instr = prefFile.inputStream() + val text = instr.bufferedReader().use { it.readText() } + Gson().fromJson(text, PreferenceDto::class.java) + } + catch (e: Exception) { + log.error("Failed to read preference file: ${prefFile.absolutePath}.. Will use default configuration") + null + } + } + } + + + + + +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/SharedConfig.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/SharedConfig.kt new file mode 100644 index 00000000..2067a4c2 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/SharedConfig.kt @@ -0,0 +1,22 @@ +package no.iktdev.mediaprocessing.shared + +import java.io.File + +object SharedConfig { + var kafkaTopic: String = System.getenv("KAFKA_TOPIC") ?: "contentEvents" + var incomingContent: File = if (!System.getenv("DIRECTORY_CONTENT_INCOMING").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_INCOMING")) else File("/src/input") + val outgoingContent: File = if (!System.getenv("DIRECTORY_CONTENT_OUTGOING").isNullOrBlank()) File(System.getenv("DIRECTORY_CONTENT_OUTGOING")) else File("/src/output") + + val ffprobe: String = System.getenv("SUPPORTING_EXECUTABLE_FFPROBE") ?: "ffprobe" + val ffmpeg: String = System.getenv("SUPPORTING_EXECUTABLE_FFMPEG") ?: "no/iktdev/mediaprocessing/shared/contract/ffmpeg" + + val preference: File = File("/data/config/preference.json") +} + +object DatabaseConfig { + val address: String? = System.getenv("DATABASE_ADDRESS") + val port: String? = System.getenv("DATABASE_PORT") + val username: String? = System.getenv("DATABASE_USERNAME") + val password: String? = System.getenv("DATABASE_PASSWORD") + val database: String? = System.getenv("DATABASE_NAME") +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/DataSource.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/DataSource.kt new file mode 100644 index 00000000..254a261c --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/DataSource.kt @@ -0,0 +1,34 @@ +package no.iktdev.mediaprocessing.shared.datasource + +import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.Table +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneId +import java.time.ZoneOffset + +abstract class DataSource(val databaseName: String, val address: String, val port: String?, val username: String, val password: String) { + + abstract fun createDatabase(): Database? + + abstract fun createTables(vararg tables: Table) + + abstract fun createDatabaseStatement(): String + + abstract fun toConnectionUrl(): String + + fun toPortedAddress(): String { + return if (!address.contains(":") && port?.isBlank() != true) { + "$address:$port" + } else address + } + +} + +fun timestampToLocalDateTime(timestamp: Int): LocalDateTime { + return Instant.ofEpochSecond(timestamp.toLong()).atZone(ZoneId.systemDefault()).toLocalDateTime() +} + +fun LocalDateTime.toEpochSeconds(): Long { + return this.toEpochSecond(ZoneOffset.ofTotalSeconds(ZoneOffset.systemDefault().rules.getOffset(LocalDateTime.now()).totalSeconds)) +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/MySqlDataSource.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/MySqlDataSource.kt new file mode 100644 index 00000000..10b38bc5 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/MySqlDataSource.kt @@ -0,0 +1,84 @@ +package no.iktdev.mediaprocessing.shared.datasource + +import no.iktdev.mediaprocessing.shared.DatabaseConfig +import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.SchemaUtils +import org.jetbrains.exposed.sql.Table +import org.jetbrains.exposed.sql.transactions.TransactionManager +import org.jetbrains.exposed.sql.transactions.transaction + +open class MySqlDataSource(databaseName: String, address: String, port: String = "", username: String, password: String): DataSource(databaseName = databaseName, address = address, port = port, username = username, password = password) { + companion object { + fun fromDatabaseEnv(): MySqlDataSource { + if (DatabaseConfig.database.isNullOrBlank()) throw RuntimeException("Database name is not defined in 'DATABASE_NAME'") + if (DatabaseConfig.username.isNullOrBlank()) throw RuntimeException("Database username is not defined in 'DATABASE_USERNAME'") + if (DatabaseConfig.address.isNullOrBlank()) throw RuntimeException("Database address is not defined in 'DATABASE_ADDRESS'") + return MySqlDataSource( + databaseName = DatabaseConfig.database, + address = DatabaseConfig.address, + port = DatabaseConfig.port ?: "", + username = DatabaseConfig.username, + password = DatabaseConfig.password ?: "" + ) + } + } + + override fun createDatabase(): Database? { + val ok = transaction(toDatabaseServerConnection()) { + val tmc = TransactionManager.current().connection + val query = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '$databaseName'" + val stmt = tmc.prepareStatement(query, true) + + val resultSet = stmt.executeQuery() + val databaseExists = resultSet.next() + + if (!databaseExists) { + try { + exec(createDatabaseStatement()) + println("Database $databaseName created.") + true + } catch (e: Exception) { + e.printStackTrace() + false + } + } else { + println("Database $databaseName already exists.") + true + } + } + + return if (ok) toDatabase() else null + } + + override fun createTables(vararg tables: Table) { + transaction { + SchemaUtils.createMissingTablesAndColumns(*tables) + println("Database transaction completed") + } + } + + override fun createDatabaseStatement(): String { + return "CREATE DATABASE $databaseName" + } + + protected fun toDatabaseServerConnection(): Database { + return Database.connect( + toConnectionUrl(), + user = username, + password = password + ) + } + + fun toDatabase(): Database { + return Database.connect( + "${toConnectionUrl()}/$databaseName", + user = username, + password = password + ) + } + + override fun toConnectionUrl(): String { + return "jdbc:mysql://${toPortedAddress()}" + } + +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/TableDefaultOperations.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/TableDefaultOperations.kt new file mode 100644 index 00000000..0d7a0fb5 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/datasource/TableDefaultOperations.kt @@ -0,0 +1,72 @@ +package no.iktdev.mediaprocessing.shared.datasource + +import org.jetbrains.exposed.dao.id.EntityID +import org.jetbrains.exposed.sql.Column +import org.jetbrains.exposed.sql.Table +import org.jetbrains.exposed.sql.insert +import org.jetbrains.exposed.sql.statements.InsertStatement +import org.jetbrains.exposed.sql.update + +import org.jetbrains.exposed.sql.transactions.transaction + +open class TableDefaultOperations { + +} + +fun withTransaction(block: () -> T): T? { + return try { + transaction { + try { + block() + } catch (e: Exception) { + e.printStackTrace() + // log the error here or handle the exception as needed + throw e // Optionally, you can rethrow the exception if needed + } + } + } catch (e: Exception) { + e.printStackTrace() + // log the error here or handle the exception as needed + null + } +} + +fun insertWithSuccess(block: () -> T): Boolean { + return try { + transaction { + try { + block() + } catch (e: Exception) { + e.printStackTrace() + // log the error here or handle the exception as needed + throw e // Optionally, you can rethrow the exception if needed + } + } + true + } catch (e: Exception) { + e.printStackTrace() + false + } +} + +fun executeWithStatus(block: () -> T): Boolean { + return try { + transaction { + try { + block() + } catch (e: Exception) { + e.printStackTrace() + // log the error here or handle the exception as needed + throw e // Optionally, you can rethrow the exception if needed + } + } + true + } catch (e: Exception) { + e.printStackTrace() + false + } +} + + + + diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/extended/FileExt.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/extended/FileExt.kt new file mode 100644 index 00000000..6c1fc953 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/extended/FileExt.kt @@ -0,0 +1,100 @@ +package no.iktdev.mediaprocessing.shared.extended + +import java.io.File + +val validVideoFiles = listOf( + "mkv", + "avi", + "mp4", + "wmv", + "webm", + "mov" +) + +fun File.isSupportedVideoFile(): Boolean { + return this.isFile && validVideoFiles.contains(this.extension) +} + +fun getSanitizedFileName(name: String): String { + /** + * Modifies the input value and removes "[Text]" + * @param text "[TEST] Dummy - 01 [AZ 1080p] " + */ + fun removeBracketedText(text: String): String { + return Regex("\\[.*?]").replace(text, " ") + } + + /** + * + */ + fun removeParenthesizedText(text: String): String { + return Regex("\\(.*?\\)").replace(text, " ") + } + + /** + * + */ + fun removeResolutionAndTags(text: String): String { + return Regex("(.*?)(?=\\d+[pk]\\b)").replace(text, " ") + } + + fun removeInBetweenCharacters(text: String): String { + return Regex("[.]").replace(text, " ") + } + + /** + * @param text "example text with extra spaces" + * @return example text with extra spaces + */ + fun removeExtraWhiteSpace(text: String): String { + return Regex("\\s{2,}").replace(text, " ") + } + + return name + .let { removeBracketedText(it) } + .let { removeParenthesizedText(it) } + .let { removeResolutionAndTags(it) } + .let { removeInBetweenCharacters(it) } + .let { removeExtraWhiteSpace(it) } +} + + +fun File.getDesiredVideoFileName(): String? { + if (!this.isSupportedVideoFile()) return null + val cleanedFileName = getSanitizedFileName(this.nameWithoutExtension) + val parts = cleanedFileName.split(" - ") + return when { + parts.size == 2 && parts[1].matches(Regex("\\d{4}")) -> { + val title = parts[0] + val year = parts[1] + "$title ($year)" + } + + parts.size >= 3 && parts[1].matches(Regex("S\\d+")) && parts[2].matches(Regex("\\d+[vV]\\d+")) -> { + val title = parts[0] + val episodeWithRevision = parts[2] + val episodeParts = episodeWithRevision.split("v", "V") + val episodeNumber = episodeParts[0].toInt() + val revisionNumber = episodeParts[1].toInt() + val seasonEpisode = + "S${episodeNumber.toString().padStart(2, '0')}E${revisionNumber.toString().padStart(2, '0')}" + val episodeTitle = if (parts.size > 3) parts[3] else "" + "$title - $seasonEpisode - $episodeTitle" + } + + else -> cleanedFileName + }.trim() +} + +fun File.getGuessedVideoTitle(): String? { + val desiredFileName = getDesiredVideoFileName() ?: return null + val seasonRegex = Regex("\\sS[0-9]+(\\s- [0-9]+|\\s[0-9]+)", RegexOption.IGNORE_CASE) + if (seasonRegex.containsMatchIn(desiredFileName)) { + return seasonRegex.replace(desiredFileName, "").trim() + } else { + val result = if (desiredFileName.contains(" - ")) { + return desiredFileName.split(" - ").firstOrNull() ?: desiredFileName + } else desiredFileName + return result.trim() + } +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/CoordinatorProducer.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/CoordinatorProducer.kt new file mode 100644 index 00000000..8f9431c3 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/CoordinatorProducer.kt @@ -0,0 +1,24 @@ +package no.iktdev.mediaprocessing.shared.kafka + +import no.iktdev.mediaprocessing.shared.SharedConfig +import no.iktdev.mediaprocessing.shared.kafka.core.DefaultProducer +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.Message +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import no.iktdev.streamit.library.kafka.dto.Status + +class CoordinatorProducer(): DefaultProducer(SharedConfig.kafkaTopic) { + fun sendMessage(referenceId: String, event: KafkaEvents, data: MessageDataWrapper) { + super.sendMessage(event.event, Message( + referenceId = referenceId, + data = data + )) + } + fun sendMessage(referenceId: String, event: KafkaEvents, eventId: String, data: MessageDataWrapper) { + super.sendMessage(event.event, Message( + referenceId = referenceId, + eventId = eventId, + data = data + )) + } +} \ No newline at end of file diff --git a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/FileNameDeterminate.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/parsing/FileNameDeterminate.kt similarity index 95% rename from Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/FileNameDeterminate.kt rename to shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/parsing/FileNameDeterminate.kt index 5f530021..b69be8bc 100644 --- a/Reader/src/main/kotlin/no/iktdev/streamit/content/reader/analyzer/contentDeterminator/FileNameDeterminate.kt +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/parsing/FileNameDeterminate.kt @@ -1,8 +1,9 @@ -package no.iktdev.streamit.content.reader.analyzer.contentDeterminator +package no.iktdev.mediaprocessing.shared.parsing + +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.EpisodeInfo +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.MovieInfo +import no.iktdev.mediaprocessing.shared.kafka.dto.events_result.VideoInfo -import no.iktdev.streamit.content.common.dto.reader.EpisodeInfo -import no.iktdev.streamit.content.common.dto.reader.MovieInfo -import no.iktdev.streamit.content.common.dto.reader.VideoInfo class FileNameDeterminate(val title: String, val sanitizedName: String, val ctype: ContentType = ContentType.UNDEFINED) { @@ -156,4 +157,4 @@ class FileNameDeterminate(val title: String, val sanitizedName: String, val ctyp return cleanedEpisodeTitle } } -} +} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/Naming.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/parsing/FileNameParser.kt similarity index 96% rename from CommonCode/src/main/java/no/iktdev/streamit/content/common/Naming.kt rename to shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/parsing/FileNameParser.kt index d28f406a..6c0714ce 100644 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/Naming.kt +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/parsing/FileNameParser.kt @@ -1,6 +1,6 @@ -package no.iktdev.streamit.content.common +package no.iktdev.mediaprocessing.shared.parsing -class Naming(val fileName: String) { +class FileNameParser(val fileName: String) { var cleanedFileName: String private set diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentDataReader.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentDataReader.kt new file mode 100644 index 00000000..fa6ba700 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentDataReader.kt @@ -0,0 +1,33 @@ +package no.iktdev.mediaprocessing.shared.persistance + +import com.google.gson.Gson +import no.iktdev.mediaprocessing.shared.datasource.withTransaction +import no.iktdev.mediaprocessing.shared.kafka.core.DeserializingRegistry +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import org.jetbrains.exposed.sql.ResultRow +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.selectAll +import java.time.LocalDateTime + +class PersistentDataReader { + val dzz = DeserializingRegistry() + + fun getAllMessages(): List> { + val events = withTransaction { + events.selectAll() + .groupBy { it[events.referenceId] } + } + return events?.mapNotNull { it.value.mapNotNull { v -> fromRowToPersistentMessage(v, dzz) } } ?: emptyList() + } + + fun getMessagesFor(referenceId: String): List { + return withTransaction { + events.select { events.referenceId eq referenceId } + .orderBy(events.created, SortOrder.ASC) + .mapNotNull { fromRowToPersistentMessage(it, dzz) } + } ?: emptyList() + } + +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentDataStore.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentDataStore.kt new file mode 100644 index 00000000..4d1268ad --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentDataStore.kt @@ -0,0 +1,20 @@ +package no.iktdev.mediaprocessing.shared.persistance + +import no.iktdev.mediaprocessing.shared.datasource.executeWithStatus +import no.iktdev.mediaprocessing.shared.datasource.withTransaction +import no.iktdev.mediaprocessing.shared.kafka.dto.Message +import org.jetbrains.exposed.sql.insert + +open class PersistentDataStore { + fun storeMessage(event: String, message: Message<*>): Boolean { + return executeWithStatus { + events.insert { + it[referenceId] = message.referenceId + it[eventId] = message.eventId + it[events.event] = event + it[data] = message.dataAsJson() + } + } + } + +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentMessage.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentMessage.kt new file mode 100644 index 00000000..d07e6b44 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/PersistentMessage.kt @@ -0,0 +1,33 @@ +package no.iktdev.mediaprocessing.shared.persistance + +import com.google.gson.Gson +import no.iktdev.mediaprocessing.shared.kafka.core.DeserializingRegistry +import no.iktdev.mediaprocessing.shared.kafka.core.KafkaEvents +import no.iktdev.mediaprocessing.shared.kafka.dto.MessageDataWrapper +import org.jetbrains.exposed.sql.ResultRow +import java.time.LocalDateTime + +data class PersistentMessage( + val referenceId: String, + val eventId: String, + val event: KafkaEvents, + val data: MessageDataWrapper, + val created: LocalDateTime +) + +fun fromRowToPersistentMessage(row: ResultRow, dez: DeserializingRegistry): PersistentMessage? { + val kev = try { + KafkaEvents.valueOf(row[events.event]) + } catch (e: IllegalArgumentException) { + e.printStackTrace() + return null + } + val dzdata = dez.deserializeData(kev, row[events.data]) + return PersistentMessage( + referenceId = row[events.referenceId], + eventId = row[events.eventId], + event = kev, + data = dzdata, + created = row[events.created] + ) +} \ No newline at end of file diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/events.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/events.kt new file mode 100644 index 00000000..f6eca0b3 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/persistance/events.kt @@ -0,0 +1,19 @@ +package no.iktdev.mediaprocessing.shared.persistance + +import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.Column +import org.jetbrains.exposed.sql.javatime.CurrentDateTime +import org.jetbrains.exposed.sql.javatime.datetime +import java.time.LocalDateTime + +object events: IntIdTable() { + val referenceId: Column = varchar("referenceId", 50) + val eventId: Column = varchar("referenceId", 50) + val event: Column = varchar("event1",100) + val data: Column = text("data") + val created: Column = datetime("created").defaultExpression(CurrentDateTime) + + init { + uniqueIndex(referenceId, eventId, event) + } +} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deamon/IDaemon.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/IRunner.kt similarity index 63% rename from CommonCode/src/main/java/no/iktdev/streamit/content/common/deamon/IDaemon.kt rename to shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/IRunner.kt index 0f8316e6..4d48f51c 100644 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deamon/IDaemon.kt +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/IRunner.kt @@ -1,6 +1,6 @@ -package no.iktdev.streamit.content.common.deamon +package no.iktdev.mediaprocessing.shared.runner -interface IDaemon { +interface IRunner { fun onStarted() {} diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/ResultRunner.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/ResultRunner.kt new file mode 100644 index 00000000..f5b81e38 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/ResultRunner.kt @@ -0,0 +1,20 @@ +package no.iktdev.mediaprocessing.shared.runner + +import com.github.pgreze.process.Redirect +import com.github.pgreze.process.process + +data class CodeToOutput( + val statusCode: Int, + val output: List +) + +suspend fun getOutputUsing(executable: String, vararg arguments: String): CodeToOutput { + val result: MutableList = mutableListOf() + val code = process(executable, *arguments, + stderr = Redirect.CAPTURE, + stdout = Redirect.CAPTURE, + consumer = { + result.add(it) + }).resultCode + return CodeToOutput(statusCode = code, result) +} \ No newline at end of file diff --git a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deamon/Daemon.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/Runner.kt similarity index 58% rename from CommonCode/src/main/java/no/iktdev/streamit/content/common/deamon/Daemon.kt rename to shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/Runner.kt index fea48205..bfe8943b 100644 --- a/CommonCode/src/main/java/no/iktdev/streamit/content/common/deamon/Daemon.kt +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/runner/Runner.kt @@ -1,26 +1,25 @@ -package no.iktdev.streamit.content.common.deamon +package no.iktdev.mediaprocessing.shared.runner -import com.github.pgreze.process.ProcessResult -import com.github.pgreze.process.Redirect -import com.github.pgreze.process.process -import com.google.gson.Gson -import kotlinx.coroutines.* +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch import mu.KotlinLogging import no.iktdev.exfl.coroutines.Coroutines -private val logger = KotlinLogging.logger {} +open class Runner(open val executable: String, val daemonInterface: IRunner) { + private val logger = KotlinLogging.logger {} -open class Daemon(open val executable: String, val daemonInterface: IDaemon) { val scope = Coroutines.io() var job: Job? = null - var executor: ProcessResult? = null + var executor: com.github.pgreze.process.ProcessResult? = null open suspend fun run(parameters: List): Int { daemonInterface.onStarted() logger.info { "\nDaemon arguments: $executable \nParamters:\n${parameters.joinToString(" ")}" } job = scope.launch { - executor = process(executable, *parameters.toTypedArray(), - stdout = Redirect.CAPTURE, - stderr = Redirect.CAPTURE, + executor = com.github.pgreze.process.process(executable, *parameters.toTypedArray(), + stdout = com.github.pgreze.process.Redirect.CAPTURE, + stderr = com.github.pgreze.process.Redirect.CAPTURE, consumer = { daemonInterface.onOutputChanged(it) }) diff --git a/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/socket/SocketImplementation.kt b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/socket/SocketImplementation.kt new file mode 100644 index 00000000..3b614421 --- /dev/null +++ b/shared/src/main/kotlin/no/iktdev/mediaprocessing/shared/socket/SocketImplementation.kt @@ -0,0 +1,23 @@ +package no.iktdev.mediaprocessing.shared.socket + +import org.springframework.context.annotation.Configuration +import org.springframework.messaging.simp.config.MessageBrokerRegistry +import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker +import org.springframework.web.socket.config.annotation.StompEndpointRegistry +import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer + +@Configuration +@EnableWebSocketMessageBroker +open class SocketImplementation: WebSocketMessageBrokerConfigurer { + + override fun registerStompEndpoints(registry: StompEndpointRegistry) { + registry.addEndpoint("/ws") + .setAllowedOrigins("*://localhost:*/*", "http://localhost:3000/") + .withSockJS() + } + + override fun configureMessageBroker(registry: MessageBrokerRegistry) { + registry.enableSimpleBroker("/topic") + registry.setApplicationDestinationPrefixes("/app") + } +} \ No newline at end of file diff --git a/src/main/java/no/iktdev/mediaprocessing/Main.java b/src/main/java/no/iktdev/mediaprocessing/Main.java new file mode 100644 index 00000000..a8ec1821 --- /dev/null +++ b/src/main/java/no/iktdev/mediaprocessing/Main.java @@ -0,0 +1,7 @@ +package no.iktdev.mediaprocessing; + +public class Main { + public static void main(String[] args) { + System.out.println("Hello world!"); + } +} \ No newline at end of file