Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6527e68e52 |
100
.github/workflows/build-java-app.yml
vendored
100
.github/workflows/build-java-app.yml
vendored
@ -1,100 +0,0 @@
|
||||
name: Build Java App
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
app:
|
||||
required: true
|
||||
type: string
|
||||
dockerTag:
|
||||
required: true
|
||||
type: string
|
||||
enabled:
|
||||
required: true
|
||||
type: boolean
|
||||
shouldBuild:
|
||||
required: true
|
||||
type: boolean
|
||||
|
||||
|
||||
jobs:
|
||||
build-java:
|
||||
if: ${{ inputs.enabled && inputs.shouldBuild }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Detect frontend
|
||||
id: detect_web
|
||||
run: |
|
||||
if [ -d "apps/${{ inputs.app }}/web" ]; then
|
||||
echo "has_web=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "has_web=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Build React frontend
|
||||
if: ${{ steps.detect_web.outputs.has_web == 'true' }}
|
||||
run: |
|
||||
cd apps/${{ inputs.app }}/web
|
||||
npm install
|
||||
export CI=false
|
||||
npm run build
|
||||
|
||||
- name: Copy React build into Spring Boot resources
|
||||
if: ${{ steps.detect_web.outputs.has_web == 'true' }}
|
||||
run: |
|
||||
rm -rf apps/${{ inputs.app }}/src/main/resources/static
|
||||
mkdir -p apps/${{ inputs.app }}/src/main/resources/static
|
||||
cp -r apps/${{ inputs.app }}/web/build/* apps/${{ inputs.app }}/src/main/resources/static
|
||||
|
||||
- name: Extract version
|
||||
run: |
|
||||
VERSION=$(grep '^version' apps/${{ inputs.app }}/build.gradle.kts | sed 's/.*"\(.*\)".*/\1/')
|
||||
echo "VERSION=$VERSION" >> $GITHUB_ENV
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
chmod +x ./gradlew
|
||||
./gradlew :apps:${{ inputs.app }}:test --info --stacktrace
|
||||
|
||||
|
||||
- name: Build Java module
|
||||
run: |
|
||||
chmod +x ./gradlew
|
||||
./gradlew :apps:${{ inputs.app }}:bootJar --info --stacktrace
|
||||
|
||||
- name: Build Docker image locally
|
||||
run: |
|
||||
docker build \
|
||||
-f ./dockerfiles/DebianJava \
|
||||
-t local-${{ inputs.app }}:${{ inputs.dockerTag }} \
|
||||
--build-arg MODULE_NAME=${{ inputs.app }} \
|
||||
--build-arg PASS_APP_VERSION=${{ env.VERSION }} \
|
||||
.
|
||||
|
||||
- name: Test Docker container
|
||||
run: |
|
||||
docker run --rm local-${{ inputs.app }}:${{ inputs.dockerTag }} /bin/sh -c "echo 'Smoke test OK'"
|
||||
|
||||
- name: Docker login
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
|
||||
- name: Push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./dockerfiles/DebianJavaFfmpeg
|
||||
build-args: |
|
||||
MODULE_NAME=${{ inputs.app }}
|
||||
PASS_APP_VERSION=${{ env.VERSION }}
|
||||
push: true
|
||||
tags: |
|
||||
bskjon/mediaprocessing-${{ inputs.app }}:v5
|
||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ inputs.dockerTag }}
|
||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ github.sha }}
|
||||
127
.github/workflows/build-python-app.yml
vendored
127
.github/workflows/build-python-app.yml
vendored
@ -1,127 +0,0 @@
|
||||
name: Build Python App
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
app:
|
||||
required: true
|
||||
type: string
|
||||
dockerTag:
|
||||
required: true
|
||||
type: string
|
||||
enabled:
|
||||
required: true
|
||||
type: boolean
|
||||
shouldBuild:
|
||||
required: true
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
build-python:
|
||||
if: ${{ inputs.enabled && inputs.shouldBuild }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# -----------------------------
|
||||
# Cache pip per app
|
||||
# -----------------------------
|
||||
- name: Cache pip
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ inputs.app }}-${{ hashFiles('apps/${{ inputs.app }}/requirements*.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-${{ inputs.app }}-
|
||||
|
||||
# -----------------------------
|
||||
# Install Python deps
|
||||
# -----------------------------
|
||||
- name: Install dependencies
|
||||
working-directory: apps/${{ inputs.app }}
|
||||
run: |
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
|
||||
|
||||
# -----------------------------
|
||||
# Run tests
|
||||
# -----------------------------
|
||||
- name: Run Python tests
|
||||
working-directory: apps/${{ inputs.app }}
|
||||
run: python -m pytest -q
|
||||
|
||||
# -----------------------------
|
||||
# Setup Buildx
|
||||
# -----------------------------
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# -----------------------------
|
||||
# Cache Docker layers per app
|
||||
# -----------------------------
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache-${{ inputs.app }}
|
||||
key: ${{ runner.os }}-buildx-${{ inputs.app }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-${{ inputs.app }}-
|
||||
|
||||
# -----------------------------
|
||||
# Build image (load locally for smoke test)
|
||||
# -----------------------------
|
||||
- name: Build Docker image (local load)
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./dockerfiles/Python
|
||||
build-args: |
|
||||
MODULE_NAME=${{ inputs.app }}
|
||||
load: true
|
||||
tags: local-${{ inputs.app }}:${{ inputs.dockerTag }}
|
||||
cache-from: type=local,src=/tmp/.buildx-cache-${{ inputs.app }}
|
||||
cache-to: type=local,dest=/tmp/.buildx-cache-${{ inputs.app }}-new
|
||||
|
||||
# -----------------------------
|
||||
# Smoke test
|
||||
# -----------------------------
|
||||
- name: Smoke test container
|
||||
run: |
|
||||
docker run --rm local-${{ inputs.app }}:${{ inputs.dockerTag }} \
|
||||
/bin/sh -c "echo 'Smoke test OK'"
|
||||
|
||||
# -----------------------------
|
||||
# Docker login
|
||||
# -----------------------------
|
||||
- name: Docker login
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
# -----------------------------
|
||||
# Push final image (no rebuild)
|
||||
# -----------------------------
|
||||
- name: Push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./dockerfiles/Python
|
||||
build-args: |
|
||||
MODULE_NAME=${{ inputs.app }}
|
||||
push: true
|
||||
tags: |
|
||||
bskjon/mediaprocessing-${{ inputs.app }}:v5
|
||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ inputs.dockerTag }}
|
||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ github.sha }}
|
||||
cache-from: type=local,src=/tmp/.buildx-cache-${{ inputs.app }}
|
||||
cache-to: type=local,dest=/tmp/.buildx-cache-${{ inputs.app }}-new
|
||||
|
||||
# -----------------------------
|
||||
# Move Docker cache
|
||||
# -----------------------------
|
||||
- name: Move Docker cache
|
||||
run: |
|
||||
rm -rf /tmp/.buildx-cache-${{ inputs.app }}
|
||||
mv /tmp/.buildx-cache-${{ inputs.app }}-new /tmp/.buildx-cache-${{ inputs.app }}
|
||||
26
.github/workflows/build-shared.yml
vendored
26
.github/workflows/build-shared.yml
vendored
@ -1,26 +0,0 @@
|
||||
name: Build Shared
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
dockerTag:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
build-shared:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Cache Gradle
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.gradle/caches
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
|
||||
|
||||
- name: Build Shared module
|
||||
run: |
|
||||
chmod +x ./gradlew
|
||||
./gradlew :shared:build --info --stacktrace
|
||||
115
.github/workflows/build-v5.yml
vendored
115
.github/workflows/build-v5.yml
vendored
@ -1,115 +0,0 @@
|
||||
name: Build v5
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ v5 ]
|
||||
pull_request:
|
||||
branches: [ v5 ]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
pre-check:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
sharedDefinitions: ${{ steps.filter.outputs.sharedDefinitions }}
|
||||
shared: ${{ steps.filter.outputs.shared }}
|
||||
processer: ${{ steps.filter.outputs.processer }}
|
||||
converter: ${{ steps.filter.outputs.converter }}
|
||||
coordinator: ${{ steps.filter.outputs.coordinator }}
|
||||
ui: ${{ steps.filter.outputs.ui }}
|
||||
py-metadata: ${{ steps.filter.outputs.metadata }}
|
||||
py-watcher: ${{ steps.filter.outputs.watcher }}
|
||||
dockerTag: ${{ steps.tag.outputs.tag }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
base: ${{ github.event.before }}
|
||||
|
||||
- name: Detect changes
|
||||
id: filter
|
||||
uses: dorny/paths-filter@v3
|
||||
with:
|
||||
filters: |
|
||||
shared:
|
||||
- 'shared/**'
|
||||
sharedDefinitions:
|
||||
- 'gradle/**'
|
||||
processer:
|
||||
- 'apps/processer/**'
|
||||
converter:
|
||||
- 'apps/converter/**'
|
||||
coordinator:
|
||||
- 'apps/coordinator/**'
|
||||
ui:
|
||||
- 'apps/ui/**'
|
||||
metadata:
|
||||
- 'apps/py-metadata/**'
|
||||
watcher:
|
||||
- 'apps/py-watcher/**'
|
||||
|
||||
- name: Generate docker tag
|
||||
id: tag
|
||||
run: echo "tag=$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" >> $GITHUB_OUTPUT
|
||||
|
||||
|
||||
build-shared:
|
||||
needs: pre-check
|
||||
if: ${{
|
||||
needs.pre-check.outputs.shared == 'true'
|
||||
|| needs.pre-check.outputs.sharedDefinitions == 'true'
|
||||
|| needs.pre-check.outputs.processer == 'true'
|
||||
|| needs.pre-check.outputs.converter == 'true'
|
||||
|| needs.pre-check.outputs.coordinator == 'true'
|
||||
|| needs.pre-check.outputs.ui == 'true'
|
||||
|| github.event_name == 'workflow_dispatch'
|
||||
}}
|
||||
uses: ./.github/workflows/build-shared.yml
|
||||
with:
|
||||
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
|
||||
|
||||
build-java:
|
||||
needs: [pre-check, build-shared]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- app: processer
|
||||
enabled: true
|
||||
- app: converter
|
||||
enabled: true
|
||||
- app: coordinator
|
||||
enabled: true
|
||||
- app: ui
|
||||
enabled: false
|
||||
|
||||
uses: ./.github/workflows/build-java-app.yml
|
||||
with:
|
||||
app: ${{ matrix.app }}
|
||||
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
|
||||
enabled: ${{ matrix.enabled }}
|
||||
shouldBuild: ${{ needs.pre-check.outputs[matrix.app] == 'true'
|
||||
|| needs.pre-check.outputs.shared == 'true'
|
||||
|| needs.pre-check.outputs.sharedDefinitions == 'true'
|
||||
|| github.event_name == 'workflow_dispatch' }}
|
||||
secrets: inherit
|
||||
|
||||
|
||||
build-python:
|
||||
needs: [pre-check]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- app: py-metadata
|
||||
enabled: true
|
||||
- app: py-watcher
|
||||
enabled: true
|
||||
|
||||
uses: ./.github/workflows/build-python-app.yml
|
||||
with:
|
||||
app: ${{ matrix.app }}
|
||||
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
|
||||
enabled: ${{ matrix.enabled }}
|
||||
shouldBuild: ${{ needs.pre-check.outputs[matrix.app] == 'true'
|
||||
|| github.event_name == 'workflow_dispatch' }}
|
||||
secrets: inherit
|
||||
9
.gitignore
vendored
9
.gitignore
vendored
@ -36,12 +36,7 @@ bin/
|
||||
/.nb-gradle/
|
||||
|
||||
### VS Code ###
|
||||
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
|
||||
|
||||
.idea/runConfigurations
|
||||
/apps/py-metadata/venv/
|
||||
/apps/py-watcher/venv/
|
||||
.DS_Store
|
||||
6
.idea/copilot.data.migration.agent.xml
generated
6
.idea/copilot.data.migration.agent.xml
generated
@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="AgentMigrationStateService">
|
||||
<option name="migrationStatus" value="COMPLETED" />
|
||||
</component>
|
||||
</project>
|
||||
6
.idea/copilot.data.migration.ask.xml
generated
6
.idea/copilot.data.migration.ask.xml
generated
@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="AskMigrationStateService">
|
||||
<option name="migrationStatus" value="COMPLETED" />
|
||||
</component>
|
||||
</project>
|
||||
6
.idea/copilot.data.migration.ask2agent.xml
generated
6
.idea/copilot.data.migration.ask2agent.xml
generated
@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Ask2AgentMigrationStateService">
|
||||
<option name="migrationStatus" value="COMPLETED" />
|
||||
</component>
|
||||
</project>
|
||||
6
.idea/copilot.data.migration.edit.xml
generated
6
.idea/copilot.data.migration.edit.xml
generated
@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="EditMigrationStateService">
|
||||
<option name="migrationStatus" value="COMPLETED" />
|
||||
</component>
|
||||
</project>
|
||||
4
.idea/gradle.xml
generated
4
.idea/gradle.xml
generated
@ -5,7 +5,6 @@
|
||||
<option name="linkedExternalProjectsSettings">
|
||||
<GradleProjectSettings>
|
||||
<option name="externalProjectPath" value="$PROJECT_DIR$" />
|
||||
<option name="gradleJvm" value="17" />
|
||||
<option name="modules">
|
||||
<set>
|
||||
<option value="$PROJECT_DIR$" />
|
||||
@ -16,8 +15,7 @@
|
||||
<option value="$PROJECT_DIR$/apps/ui" />
|
||||
<option value="$PROJECT_DIR$/shared" />
|
||||
<option value="$PROJECT_DIR$/shared/common" />
|
||||
<option value="$PROJECT_DIR$/shared/event-task-contract" />
|
||||
<option value="$PROJECT_DIR$/shared/ffmpeg" />
|
||||
<option value="$PROJECT_DIR$/shared/eventi" />
|
||||
</set>
|
||||
</option>
|
||||
</GradleProjectSettings>
|
||||
|
||||
2
.idea/kotlinc.xml
generated
2
.idea/kotlinc.xml
generated
@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="KotlinJpsPluginSettings">
|
||||
<option name="version" value="2.1.0" />
|
||||
<option name="version" value="1.9.20" />
|
||||
</component>
|
||||
</project>
|
||||
2
.idea/misc.xml
generated
2
.idea/misc.xml
generated
@ -1,5 +1,5 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" project-jdk-name="azul-17" project-jdk-type="JavaSDK" />
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="azul-17" project-jdk-type="JavaSDK" />
|
||||
</project>
|
||||
26
.idea/runConfigurations/UIApplicationKt.xml
generated
26
.idea/runConfigurations/UIApplicationKt.xml
generated
@ -1,26 +0,0 @@
|
||||
<component name="ProjectRunConfigurationManager">
|
||||
<configuration default="false" name="UIApplicationKt" type="JetRunConfigurationType" nameIsGenerated="true">
|
||||
<envs>
|
||||
<env name="DATABASE_ADDRESS" value="192.168.2.250" />
|
||||
<env name="DATABASE_NAME_E" value="eventsV4" />
|
||||
<env name="DATABASE_NAME_S" value="streamitv3" />
|
||||
<env name="DATABASE_PASSWORD" value="shFZ27eL2x2NoxyEDBMfDWkvFO" />
|
||||
<env name="DATABASE_PORT" value="3306" />
|
||||
<env name="DATABASE_USERNAME" value="root" />
|
||||
<env name="DIRECTORY_CONTENT_INCOMING" value="G:\MediaProcessingPlayground\input" />
|
||||
<env name="DIRECTORY_CONTENT_OUTGOING" value="G:\MediaProcessingPlayground\output" />
|
||||
<env name="DISABLE_COMPLETE" value="true" />
|
||||
<env name="DISABLE_PRODUCE" value="true" />
|
||||
<env name="EncoderWs" value="ws://192.168.2.250:6081/ws" />
|
||||
<env name="METADATA_TIMEOUT" value="0" />
|
||||
<env name="SUPPORTING_EXECUTABLE_FFMPEG" value="G:\MediaProcessingPlayground\ffmpeg.exe" />
|
||||
<env name="SUPPORTING_EXECUTABLE_FFPROBE" value="G:\MediaProcessingPlayground\ffprobe.exe" />
|
||||
</envs>
|
||||
<option name="MAIN_CLASS_NAME" value="no.iktdev.mediaprocessing.ui.UIApplicationKt" />
|
||||
<module name="MediaProcessing.apps.ui.main" />
|
||||
<shortenClasspath name="NONE" />
|
||||
<method v="2">
|
||||
<option name="Make" enabled="true" />
|
||||
</method>
|
||||
</configuration>
|
||||
</component>
|
||||
1312
.idea/workspace.xml
generated
1312
.idea/workspace.xml
generated
File diff suppressed because it is too large
Load Diff
@ -20,5 +20,5 @@ tasks.test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
kotlin {
|
||||
jvmToolchain(21)
|
||||
jvmToolchain(17)
|
||||
}
|
||||
@ -1,8 +1,9 @@
|
||||
plugins {
|
||||
id("java")
|
||||
kotlin("jvm")
|
||||
id("org.springframework.boot")
|
||||
id("io.spring.dependency-management")
|
||||
kotlin("plugin.spring") version "1.5.31"
|
||||
id("org.springframework.boot") version "2.5.5"
|
||||
id("io.spring.dependency-management") version "1.0.11.RELEASE"
|
||||
}
|
||||
|
||||
group = "no.iktdev.mediaprocessing.apps"
|
||||
@ -26,41 +27,40 @@ repositories {
|
||||
}
|
||||
}
|
||||
|
||||
val exposedVersion = "0.44.0"
|
||||
dependencies {
|
||||
|
||||
/*Spring boot*/
|
||||
implementation("org.springframework.boot:spring-boot-starter-web")
|
||||
implementation("org.springframework.boot:spring-boot-starter-actuator")
|
||||
implementation("org.springframework.boot:spring-boot-starter-websocket")
|
||||
implementation("org.springframework:spring-tx")
|
||||
implementation("org.springframework.boot:spring-boot-starter:2.7.0")
|
||||
// implementation("org.springframework.kafka:spring-kafka:3.0.1")
|
||||
implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3")
|
||||
implementation("org.springframework.kafka:spring-kafka:2.8.5")
|
||||
|
||||
|
||||
implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
|
||||
implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
|
||||
implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
|
||||
implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion")
|
||||
implementation ("mysql:mysql-connector-java:8.0.29")
|
||||
|
||||
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
|
||||
implementation("com.google.code.gson:gson:2.8.9")
|
||||
implementation("org.json:json:20210307")
|
||||
|
||||
implementation(libs.exfl)
|
||||
implementation("no.iktdev:exfl:0.0.16-SNAPSHOT")
|
||||
implementation("no.iktdev.library:subtitle:1.8.1-SNAPSHOT")
|
||||
implementation(libs.eventi)
|
||||
|
||||
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
|
||||
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
|
||||
implementation("com.github.pgreze:kotlin-process:1.4.1")
|
||||
|
||||
implementation(project(mapOf("path" to ":shared:eventi")))
|
||||
implementation(project(mapOf("path" to ":shared:common")))
|
||||
implementation(project(mapOf("path" to ":shared:database")))
|
||||
|
||||
|
||||
implementation(kotlin("stdlib-jdk8"))
|
||||
|
||||
testImplementation("io.mockk:mockk:1.12.0")
|
||||
testImplementation("org.springframework.boot:spring-boot-starter-test")
|
||||
testImplementation(project(":shared:common", configuration = "testArtifacts"))
|
||||
testImplementation(project(":shared:database", configuration = "testArtifacts"))
|
||||
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
|
||||
|
||||
val exposedVersion = "0.61.0"
|
||||
testImplementation("org.jetbrains.exposed:exposed-core:${exposedVersion}")
|
||||
}
|
||||
|
||||
tasks.test {
|
||||
@ -78,5 +78,5 @@ tasks.jar {
|
||||
}
|
||||
|
||||
kotlin {
|
||||
jvmToolchain(21)
|
||||
jvmToolchain(17)
|
||||
}
|
||||
@ -1,28 +1,49 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.events.EventTypeRegistry
|
||||
import no.iktdev.eventi.tasks.TaskTypeRegistry
|
||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||
import no.iktdev.exfl.observable.Observables
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry
|
||||
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||
import no.iktdev.eventi.database.MySqlDataSource
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.runners
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.tasks
|
||||
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
||||
import no.iktdev.mediaprocessing.shared.database.DatabaseApplication
|
||||
import no.iktdev.mediaprocessing.shared.database.DatabasebasedMediaProcessingApp
|
||||
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||
import org.springframework.boot.runApplication
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import javax.annotation.PreDestroy
|
||||
|
||||
@DatabasebasedMediaProcessingApp
|
||||
open class ConverterApplication: DatabaseApplication() {
|
||||
@SpringBootApplication
|
||||
class ConvertApplication {
|
||||
@PreDestroy
|
||||
fun onShutdown() {
|
||||
doTransactionalCleanup()
|
||||
}
|
||||
|
||||
@Transactional
|
||||
fun doTransactionalCleanup() {
|
||||
runnerManager.unlist()
|
||||
}
|
||||
}
|
||||
|
||||
val ioCoroutine = CoroutinesIO()
|
||||
val defaultCoroutine = CoroutinesDefault()
|
||||
|
||||
lateinit var taskManager: TasksManager
|
||||
lateinit var runnerManager: RunnerManager
|
||||
|
||||
|
||||
private lateinit var eventsDatabase: MySqlDataSource
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
fun getEventsDatabase(): MySqlDataSource {
|
||||
return eventsDatabase
|
||||
}
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
@ -36,19 +57,15 @@ fun main(args: Array<String>) {
|
||||
})
|
||||
|
||||
|
||||
runApplication<ConverterApplication>(*args)
|
||||
eventsDatabase = DatabaseEnvConfig.toEventsDatabase()
|
||||
eventsDatabase.createDatabase()
|
||||
eventsDatabase.createTables(tasks, runners)
|
||||
taskManager = TasksManager(eventsDatabase)
|
||||
|
||||
runnerManager = RunnerManager(dataSource = getEventsDatabase(), applicationName = ConvertApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
|
||||
runApplication<ConvertApplication>(*args)
|
||||
log.info { "App Version: ${getAppVersion()}" }
|
||||
}
|
||||
//private val logger = KotlinLogging.logger {}
|
||||
|
||||
@Configuration
|
||||
open class ApplicationConfiguration() {
|
||||
init {
|
||||
EventRegistry.getEvents().let {
|
||||
EventTypeRegistry.register(it)
|
||||
}
|
||||
TaskRegistry.getTasks().let {
|
||||
TaskTypeRegistry.register(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt
Executable file → Normal file
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt
Executable file → Normal file
@ -1,10 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import java.io.File
|
||||
|
||||
interface ConverterEnvironment {
|
||||
fun canRead(file: File): Boolean
|
||||
fun getReader(file: File): BaseReader?
|
||||
fun createExporter(input: File, outputDir: File, name: String): Exporter
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import no.iktdev.library.subtitle.classes.Dialog
|
||||
import no.iktdev.library.subtitle.export.Export
|
||||
|
||||
class ExportAdapter(
|
||||
private val export: Export
|
||||
) : Exporter {
|
||||
|
||||
override fun write(dialogs: List<Dialog>) = export.write(dialogs)
|
||||
override fun writeSrt(dialogs: List<Dialog>) = export.writeSrt(dialogs)
|
||||
override fun writeSmi(dialogs: List<Dialog>) = export.writeSmi(dialogs)
|
||||
override fun writeVtt(dialogs: List<Dialog>) = export.writeVtt(dialogs)
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import no.iktdev.library.subtitle.classes.Dialog
|
||||
import java.io.File
|
||||
|
||||
interface Exporter {
|
||||
fun write(dialogs: List<Dialog>): MutableList<File>
|
||||
fun writeSrt(dialogs: List<Dialog>): File
|
||||
fun writeSmi(dialogs: List<Dialog>): File
|
||||
fun writeVtt(dialogs: List<Dialog>): File
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.context.annotation.Import
|
||||
|
||||
@Configuration
|
||||
class SocketLocalInit: SocketImplementation()
|
||||
@ -0,0 +1,92 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.*
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.ActiveMode
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import org.springframework.scheduling.annotation.EnableScheduling
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
@Service
|
||||
@EnableScheduling
|
||||
class TaskCoordinator(): TaskCoordinatorBase() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
override fun onProduceEvent(event: Event) {
|
||||
taskManager.produceEvent(event)
|
||||
}
|
||||
|
||||
override fun onCoordinatorReady() {
|
||||
super.onCoordinatorReady()
|
||||
runnerManager = RunnerManager(dataSource = getEventsDatabase(), applicationName = ConvertApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
}
|
||||
|
||||
|
||||
override val taskAvailabilityEventListener: MutableMap<TaskType, MutableList<TaskQueueListener>> = mutableMapOf(
|
||||
TaskType.Convert to mutableListOf(),
|
||||
)
|
||||
|
||||
private val taskListeners: MutableSet<TaskEvents> = mutableSetOf()
|
||||
fun getTaskListeners(): List<TaskEvents> {
|
||||
return taskListeners.toList()
|
||||
}
|
||||
fun addTaskEventListener(listener: TaskEvents) {
|
||||
taskListeners.add(listener)
|
||||
}
|
||||
|
||||
fun addConvertTaskListener(listener: TaskQueueListener) {
|
||||
addTaskListener(TaskType.Convert, listener)
|
||||
}
|
||||
|
||||
override fun addTaskListener(type: TaskType, listener: TaskQueueListener) {
|
||||
super.addTaskListener(type, listener)
|
||||
pullForAvailableTasks()
|
||||
}
|
||||
|
||||
|
||||
override fun pullForAvailableTasks() {
|
||||
if (runnerManager.iAmSuperseded()) {
|
||||
// This will let the application complete but not consume new
|
||||
val prevState = taskMode
|
||||
taskMode = ActiveMode.Passive
|
||||
if (taskMode != prevState && taskMode == ActiveMode.Passive) {
|
||||
log.warn { "A newer version has been detected. Changing mode to $taskMode, no new tasks will be processed" }
|
||||
}
|
||||
return
|
||||
}
|
||||
val available = taskManager.getClaimableTasks().asClaimable()
|
||||
available.forEach { (type, list) ->
|
||||
taskAvailabilityEventListener[type]?.forEach { listener ->
|
||||
list.foreachOrUntilClaimed {
|
||||
listener.onTaskAvailable(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun clearExpiredClaims() {
|
||||
val expiredClaims = taskManager.getTasksWithExpiredClaim().filter { it.task in listOf(TaskType.Convert) }
|
||||
expiredClaims.forEach {
|
||||
log.info { "Found event with expired claim: ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
}
|
||||
expiredClaims.forEach {
|
||||
val result = taskManager.deleteTaskClaim(referenceId = it.referenceId, eventId = it.eventId)
|
||||
if (result) {
|
||||
log.info { "Released claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
} else {
|
||||
log.error { "Failed to release claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun getEnabledState(): Boolean {
|
||||
return runnerManager.amIEnabled()
|
||||
}
|
||||
|
||||
interface TaskEvents {
|
||||
fun onCancelOrStopProcess(eventId: String)
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskPollerImplementation
|
||||
import no.iktdev.eventi.tasks.TaskReporter
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
import org.springframework.boot.ApplicationArguments
|
||||
import org.springframework.boot.ApplicationRunner
|
||||
import org.springframework.stereotype.Component
|
||||
import org.springframework.stereotype.Service
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class PollerAdministrator(
|
||||
private val taskPoller: TaskPoller,
|
||||
): ApplicationRunner {
|
||||
override fun run(args: ApplicationArguments?) {
|
||||
CoroutineScope(Dispatchers.Default).launch {
|
||||
taskPoller.start()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Service
|
||||
class TaskPoller(
|
||||
private val reporter: TaskReporter,
|
||||
) : TaskPollerImplementation(
|
||||
taskStore = TaskStore,
|
||||
reporterFactory = { reporter } // én reporter brukes for alle tasks
|
||||
) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Component
|
||||
class DefaultTaskReporter() : TaskReporter {
|
||||
override fun markClaimed(taskId: UUID, workerId: String) {
|
||||
TaskStore.claim(taskId, workerId)
|
||||
}
|
||||
|
||||
override fun updateLastSeen(taskId: UUID) {
|
||||
TaskStore.heartbeat(taskId)
|
||||
}
|
||||
|
||||
override fun markCompleted(taskId: UUID) {
|
||||
TaskStore.markConsumed(taskId, TaskStatus.Completed)
|
||||
|
||||
}
|
||||
|
||||
override fun markFailed(referenceId: UUID, taskId: UUID) {
|
||||
TaskStore.markConsumed(taskId, TaskStatus.Failed)
|
||||
}
|
||||
|
||||
override fun markCancelled(referenceId: UUID, taskId: UUID) {
|
||||
TaskStore.markConsumed(taskId, TaskStatus.Cancelled)
|
||||
}
|
||||
|
||||
|
||||
override fun updateProgress(taskId: UUID, progress: Int) {
|
||||
// Not to be implemented for this application
|
||||
}
|
||||
|
||||
override fun log(taskId: UUID, message: String) {
|
||||
// Not to be implemented for this application
|
||||
}
|
||||
|
||||
override fun publishEvent(event: Event) {
|
||||
EventStore.persist(event)
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.controller
|
||||
|
||||
import org.springframework.boot.actuate.health.HealthEndpoint
|
||||
import org.springframework.boot.actuate.health.Status
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/system")
|
||||
class ReadinessController(
|
||||
private val healthEndpoint: HealthEndpoint
|
||||
) {
|
||||
|
||||
@GetMapping("/ready")
|
||||
fun ready(): ResponseEntity<String> {
|
||||
val health = healthEndpoint.health()
|
||||
|
||||
return if (health.status == Status.UP) {
|
||||
ResponseEntity.ok("READY")
|
||||
} else {
|
||||
ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
|
||||
.body("NOT_READY: ${health.status}")
|
||||
}
|
||||
}
|
||||
}
|
||||
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt
Executable file → Normal file
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt
Executable file → Normal file
@ -1,18 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.convert
|
||||
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
||||
import java.io.File
|
||||
|
||||
abstract class Converter(val env: ConverterEnvironment, val listener: ConvertListener) {
|
||||
var writtenUris: List<String>? = null
|
||||
|
||||
abstract fun getSubtitleReader(useFile: File): BaseReader?
|
||||
abstract suspend fun convert(data: ConvertTask.Data)
|
||||
|
||||
class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException()
|
||||
class FileUnavailableException(override val message: String): RuntimeException()
|
||||
|
||||
abstract fun getResult(): List<String>
|
||||
}
|
||||
@ -4,70 +4,75 @@ import no.iktdev.library.subtitle.Configuration
|
||||
import no.iktdev.library.subtitle.Syncro
|
||||
import no.iktdev.library.subtitle.classes.Dialog
|
||||
import no.iktdev.library.subtitle.classes.DialogType
|
||||
import no.iktdev.library.subtitle.export.Export
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import no.iktdev.library.subtitle.reader.Reader
|
||||
import no.iktdev.mediaprocessing.converter.ConverterEnv
|
||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.SubtitleFormats
|
||||
import java.io.File
|
||||
import kotlin.jvm.Throws
|
||||
|
||||
class Converter2(
|
||||
env: ConverterEnvironment,
|
||||
listener: ConvertListener
|
||||
): Converter(env = env, listener = listener) {
|
||||
class Converter2(val data: ConvertData,
|
||||
private val listener: ConvertListener) {
|
||||
|
||||
@Throws(FileUnavailableException::class)
|
||||
override fun getSubtitleReader(useFile: File): BaseReader? {
|
||||
if (!env.canRead(useFile)) {
|
||||
private fun getReader(): BaseReader? {
|
||||
val file = File(data.inputFile)
|
||||
if (!file.canRead())
|
||||
throw FileUnavailableException("Can't open file for reading..")
|
||||
}
|
||||
return env.getReader(useFile)
|
||||
return Reader(file).getSubtitleReader()
|
||||
}
|
||||
|
||||
private fun syncDialogs(input: List<Dialog>): List<Dialog> {
|
||||
return if (ConverterEnv.syncDialogs) Syncro().sync(input) else input
|
||||
}
|
||||
|
||||
fun canRead(): Boolean {
|
||||
try {
|
||||
val reader = getReader()
|
||||
return reader != null
|
||||
} catch (e: FileUnavailableException) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(FileUnavailableException::class, FileIsNullOrEmpty::class)
|
||||
override suspend fun convert(data: ConvertTask.Data) {
|
||||
fun execute() {
|
||||
val file = File(data.inputFile)
|
||||
listener.onStarted(file.absolutePath)
|
||||
try {
|
||||
Configuration.exportJson = true
|
||||
val read = getSubtitleReader(file)?.read() ?: throw FileIsNullOrEmpty()
|
||||
val read = getReader()?.read() ?: throw FileIsNullOrEmpty()
|
||||
if (read.isEmpty())
|
||||
throw FileIsNullOrEmpty()
|
||||
val filtered = read.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) }
|
||||
val syncOrNotSync = syncDialogs(filtered)
|
||||
|
||||
val exporter = env.createExporter(file, File(data.outputDirectory), data.outputFileName)
|
||||
val exporter = Export(file, File(data.outputDirectory), data.outputFileName)
|
||||
|
||||
val outFiles = if (data.formats.isEmpty()) {
|
||||
exporter.write(syncOrNotSync)
|
||||
} else {
|
||||
val exported = mutableListOf<File>()
|
||||
if (data.formats.contains(SubtitleFormat.SRT)) {
|
||||
if (data.formats.contains(SubtitleFormats.SRT)) {
|
||||
exported.add(exporter.writeSrt(syncOrNotSync))
|
||||
}
|
||||
if (data.formats.contains(SubtitleFormat.SMI)) {
|
||||
if (data.formats.contains(SubtitleFormats.SMI)) {
|
||||
exported.add(exporter.writeSmi(syncOrNotSync))
|
||||
}
|
||||
if (data.formats.contains(SubtitleFormat.VTT)) {
|
||||
if (data.formats.contains(SubtitleFormats.VTT)) {
|
||||
exported.add(exporter.writeVtt(syncOrNotSync))
|
||||
}
|
||||
exported
|
||||
}
|
||||
writtenUris = outFiles.map { it.absolutePath }
|
||||
listener.onCompleted(file.absolutePath, writtenUris!!)
|
||||
listener.onCompleted(file.absolutePath, outFiles.map { it.absolutePath })
|
||||
} catch (e: Exception) {
|
||||
listener.onError(file.absolutePath, e.message ?: e.localizedMessage)
|
||||
}
|
||||
}
|
||||
|
||||
override fun getResult(): List<String> {
|
||||
if (writtenUris == null) {
|
||||
throw IllegalStateException("Execute must be called before getting result")
|
||||
}
|
||||
return writtenUris!!
|
||||
}
|
||||
|
||||
class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException()
|
||||
class FileUnavailableException(override val message: String): RuntimeException()
|
||||
}
|
||||
@ -1,117 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.listeners
|
||||
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.Task
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskListener
|
||||
import no.iktdev.eventi.tasks.TaskType
|
||||
import no.iktdev.library.subtitle.export.Export
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import no.iktdev.library.subtitle.reader.Reader
|
||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
||||
import no.iktdev.mediaprocessing.converter.ExportAdapter
|
||||
import no.iktdev.mediaprocessing.converter.Exporter
|
||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
||||
import no.iktdev.mediaprocessing.converter.convert.Converter
|
||||
import no.iktdev.mediaprocessing.converter.convert.Converter2
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
open class ConvertTaskListener(): TaskListener(TaskType.CPU_INTENSIVE) {
|
||||
|
||||
override fun getWorkerId(): String {
|
||||
return "${this::class.java.simpleName}-${TaskType.CPU_INTENSIVE}-${UUID.randomUUID()}"
|
||||
}
|
||||
|
||||
override fun supports(task: Task): Boolean {
|
||||
return task is ConvertTask
|
||||
}
|
||||
|
||||
|
||||
override suspend fun onTask(task: Task): Event? {
|
||||
if (task !is ConvertTask) {
|
||||
throw IllegalArgumentException("Invalid task type: ${task::class.java.name}")
|
||||
}
|
||||
val converter = getConverter()
|
||||
|
||||
withHeartbeatRunner {
|
||||
reporter?.updateLastSeen(task.taskId)
|
||||
}
|
||||
|
||||
withContext(Dispatchers.Unconfined) {
|
||||
converter.convert(task.data)
|
||||
}
|
||||
|
||||
return try {
|
||||
val result = converter.getResult()
|
||||
val newEvent = ConvertTaskResultEvent(
|
||||
data = ConvertTaskResultEvent.ConvertedData(
|
||||
language = task.data.language,
|
||||
outputFiles = result,
|
||||
baseName = task.data.outputFileName
|
||||
),
|
||||
status = TaskStatus.Completed
|
||||
).producedFrom(task)
|
||||
newEvent
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
val newEvent = ConvertTaskResultEvent(
|
||||
data = null,
|
||||
status = TaskStatus.Failed
|
||||
).producedFrom(task)
|
||||
newEvent
|
||||
}
|
||||
}
|
||||
|
||||
override fun createIncompleteStateTaskEvent(
|
||||
task: Task,
|
||||
status: TaskStatus,
|
||||
exception: Exception?
|
||||
): Event {
|
||||
val message = when (status) {
|
||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
||||
TaskStatus.Cancelled -> "Canceled"
|
||||
else -> ""
|
||||
}
|
||||
return ConvertTaskResultEvent(null, status, error = message)
|
||||
}
|
||||
|
||||
open fun getConverter(): Converter {
|
||||
return Converter2(getConverterEnvironment(), getListener())
|
||||
}
|
||||
|
||||
open fun getListener(): ConvertListener {
|
||||
return DefaultConvertListener()
|
||||
}
|
||||
|
||||
class DefaultConvertListener: ConvertListener {
|
||||
override fun onStarted(inputFile: String) {
|
||||
}
|
||||
|
||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
||||
}
|
||||
}
|
||||
|
||||
open fun getConverterEnvironment(): ConverterEnvironment {
|
||||
return DefaultConverterEnvironment()
|
||||
}
|
||||
|
||||
class DefaultConverterEnvironment : ConverterEnvironment {
|
||||
override fun canRead(file: File) = file.canRead()
|
||||
|
||||
override fun getReader(file: File): BaseReader? =
|
||||
Reader(file).getSubtitleReader()
|
||||
|
||||
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
|
||||
return ExportAdapter(Export(input, outputDir, name))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,136 @@
|
||||
package no.iktdev.mediaprocessing.converter.tasks
|
||||
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.mediaprocessing.converter.*
|
||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
||||
import no.iktdev.mediaprocessing.converter.convert.Converter2
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.Status
|
||||
import no.iktdev.mediaprocessing.shared.common.services.TaskService
|
||||
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertWorkPerformed
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertedData
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Service
|
||||
|
||||
|
||||
@Service
|
||||
class ConvertService(
|
||||
@Autowired var tasks: TaskCoordinator,
|
||||
) : TaskService(), ConvertListener, TaskCoordinator.TaskEvents {
|
||||
|
||||
fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
override val log = KotlinLogging.logger {}
|
||||
override val logDir = ConverterEnv.logDirectory
|
||||
|
||||
override fun getServiceId(serviceName: String): String {
|
||||
return super.getServiceId(this::class.java.simpleName)
|
||||
}
|
||||
|
||||
|
||||
var worker: Converter2? = null
|
||||
|
||||
override fun onAttachListener() {
|
||||
tasks.addConvertTaskListener(this)
|
||||
tasks.addTaskEventListener(this)
|
||||
}
|
||||
|
||||
|
||||
override fun isReadyToConsume(): Boolean {
|
||||
return worker == null
|
||||
}
|
||||
|
||||
override fun isTaskClaimable(task: Task): Boolean {
|
||||
return !taskManager.isTaskClaimed(referenceId = task.referenceId, eventId = task.eventId)
|
||||
}
|
||||
|
||||
|
||||
override fun onTaskAssigned(task: Task) {
|
||||
startConvert(task)
|
||||
}
|
||||
|
||||
fun startConvert(task: Task) {
|
||||
val convert = task.data as ConvertData
|
||||
worker = Converter2(convert, this)
|
||||
worker?.execute()
|
||||
}
|
||||
|
||||
override fun onStarted(inputFile: String) {
|
||||
val task = assignedTask ?: return
|
||||
taskManager.markTaskAsClaimed(task.referenceId, task.eventId, serviceId)
|
||||
log.info { "Convert started for ${task.referenceId}" }
|
||||
|
||||
}
|
||||
|
||||
|
||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
||||
val task = assignedTask ?: return
|
||||
val taskData: ConvertData = task.data as ConvertData
|
||||
log.info { "Convert completed for ${task.referenceId}" }
|
||||
val claimSuccessful = taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||
|
||||
runBlocking {
|
||||
delay(1000)
|
||||
if (!claimSuccessful) {
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||
delay(1000)
|
||||
}
|
||||
var readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||
while (!readbackIsSuccess) {
|
||||
delay(1000)
|
||||
readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||
}
|
||||
|
||||
tasks.onProduceEvent(ConvertWorkPerformed(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = ConvertedData(
|
||||
language = taskData.language,
|
||||
outputFiles = outputFiles,
|
||||
baseName = taskData.storeFileName
|
||||
)
|
||||
))
|
||||
onClearTask()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onError(inputFile: String, message: String) {
|
||||
val task = assignedTask ?: return
|
||||
super.onError(inputFile, message)
|
||||
log.info { "Convert error for ${task.referenceId}\nmessage: $message" }
|
||||
|
||||
taskManager.markTaskAsCompleted(task.referenceId, task.eventId, Status.ERROR)
|
||||
|
||||
tasks.onProduceEvent(ConvertWorkPerformed(
|
||||
metadata = EventMetadata(
|
||||
referenceId = task.referenceId,
|
||||
derivedFromEventId = task.eventId,
|
||||
status = EventStatus.Failed,
|
||||
source = getProducerName()
|
||||
)
|
||||
))
|
||||
onClearTask()
|
||||
}
|
||||
|
||||
|
||||
override fun onClearTask() {
|
||||
super.onClearTask()
|
||||
worker = null
|
||||
}
|
||||
|
||||
override fun onCancelOrStopProcess(eventId: String) {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
}
|
||||
5
apps/converter/src/main/resources/application.properties
Normal file
5
apps/converter/src/main/resources/application.properties
Normal file
@ -0,0 +1,5 @@
|
||||
spring.output.ansi.enabled=always
|
||||
logging.level.org.apache.kafka=INFO
|
||||
logging.level.root=INFO
|
||||
logging.level.Exposed=OFF
|
||||
logging.level.org.springframework.web.socket.config.WebSocketMessageBrokerStats = WARN
|
||||
@ -1,33 +0,0 @@
|
||||
spring:
|
||||
output:
|
||||
ansi:
|
||||
enabled: always
|
||||
flyway:
|
||||
enabled: true
|
||||
locations: classpath:flyway
|
||||
baseline-on-migrate: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health
|
||||
endpoint:
|
||||
health:
|
||||
show-details: always
|
||||
|
||||
logging:
|
||||
level:
|
||||
root: INFO
|
||||
org.apache.kafka: INFO
|
||||
Exposed: OFF
|
||||
org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN
|
||||
|
||||
|
||||
media:
|
||||
cache: /src/cache
|
||||
outgoing: /src/output
|
||||
incoming: /src/input
|
||||
|
||||
streamit:
|
||||
address: http://streamit.service
|
||||
@ -1,82 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import io.mockk.every
|
||||
import io.mockk.mockk
|
||||
import io.mockk.mockkObject
|
||||
import io.mockk.verify
|
||||
import no.iktdev.eventi.models.Task
|
||||
import no.iktdev.mediaprocessing.shared.common.TestBase
|
||||
import no.iktdev.mediaprocessing.shared.database.config.DatasourceConfiguration
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
|
||||
import org.jetbrains.exposed.sql.Database
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.api.extension.ExtendWith
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder
|
||||
import org.springframework.boot.test.context.SpringBootTest
|
||||
import org.springframework.test.context.TestPropertySource
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension
|
||||
import javax.sql.DataSource
|
||||
|
||||
@SpringBootTest(
|
||||
classes = [ConverterApplication::class,
|
||||
DatasourceConfiguration::class],
|
||||
webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT
|
||||
)
|
||||
@TestPropertySource(properties = ["spring.flyway.enabled=true"])
|
||||
@ExtendWith(SpringExtension::class)
|
||||
class ConverterApplicationTest : TestBase() {
|
||||
|
||||
data class TestTask(
|
||||
val success: Boolean
|
||||
) : Task()
|
||||
|
||||
|
||||
@Test
|
||||
fun `context loads and common configuration is available`() {
|
||||
// Hvis du har beans du vil verifisere, kan du autowire dem her
|
||||
// @Autowired lateinit var database: Database
|
||||
|
||||
// Dummy assertion for å verifisere at konteksten starter
|
||||
assertNotNull(Unit)
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
fun `Verify that we can access TaskStore`() {
|
||||
val tasks = TaskStore.getPendingTasks()
|
||||
assertNotNull(tasks)
|
||||
assert(tasks.isEmpty())
|
||||
|
||||
TaskStore.persist(TestTask(success = true).newReferenceId())
|
||||
|
||||
val tasksAfter = TaskStore.getPendingTasks()
|
||||
assertNotNull(tasksAfter)
|
||||
assert(tasksAfter.isNotEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `ExposedInitializer should connect to database`() {
|
||||
mockkObject(Database)
|
||||
|
||||
every {
|
||||
Database.connect(
|
||||
any<DataSource>(),
|
||||
any(),
|
||||
any(),
|
||||
any(),
|
||||
any()
|
||||
)
|
||||
} returns mockk()
|
||||
|
||||
|
||||
val context = SpringApplicationBuilder(ConverterApplication::class.java)
|
||||
.properties("spring.main.web-application-type=none")
|
||||
.run()
|
||||
|
||||
verify(exactly = 1) { Database.connect(any<DataSource>(), any(), any(), any(), any()) }
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import kotlinx.coroutines.delay
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
||||
import no.iktdev.mediaprocessing.converter.convert.Converter
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
||||
import java.io.File
|
||||
|
||||
class MockConverter(
|
||||
val delayMillis: Long = 0,
|
||||
private val simulatedResult: List<String>? = null,
|
||||
private val taskResultStatus: TaskStatus = TaskStatus.Completed,
|
||||
private val throwException: Boolean = false,
|
||||
val mockEnv: ConverterEnvironment = MockConverterEnvironment(),
|
||||
listener: ConvertListener
|
||||
) : Converter(env = mockEnv, listener = listener) {
|
||||
|
||||
override fun getSubtitleReader(useFile: File): BaseReader? {
|
||||
TODO("Not yet implemented")
|
||||
}
|
||||
|
||||
override suspend fun convert(data: ConvertTask.Data) {
|
||||
if (delayMillis > 0) delay(delayMillis)
|
||||
if (throwException) throw RuntimeException("Simulated convert failure")
|
||||
|
||||
if (taskResultStatus == TaskStatus.Failed) {
|
||||
listener.onError(data.inputFile, "Failed state desired")
|
||||
} else {
|
||||
listener.onCompleted(data.inputFile, simulatedResult!!)
|
||||
}
|
||||
}
|
||||
|
||||
override fun getResult(): List<String> {
|
||||
return simulatedResult!!
|
||||
}
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter
|
||||
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import java.io.File
|
||||
|
||||
class MockConverterEnvironment(
|
||||
var canReadValue: Boolean = true,
|
||||
var reader: BaseReader? = null,
|
||||
var exporter: Exporter? = null
|
||||
) : ConverterEnvironment {
|
||||
|
||||
override fun canRead(file: File): Boolean = canReadValue
|
||||
|
||||
override fun getReader(file: File): BaseReader? = reader
|
||||
|
||||
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
|
||||
return exporter ?: error("FakeEnv.exporter must be set before calling createExporter")
|
||||
}
|
||||
}
|
||||
@ -1,249 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.convert
|
||||
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import no.iktdev.library.subtitle.classes.Dialog
|
||||
import no.iktdev.library.subtitle.classes.DialogType
|
||||
import no.iktdev.library.subtitle.classes.Time
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
||||
import no.iktdev.mediaprocessing.converter.Exporter
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.DisplayName
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.io.File
|
||||
|
||||
class Converter2Test {
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Fake implementations
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
class FakeReader(
|
||||
private val dialogs: List<Dialog>,
|
||||
private val shouldThrow: Boolean = false
|
||||
) : BaseReader() {
|
||||
override fun read(): List<Dialog> {
|
||||
if (shouldThrow) throw RuntimeException("reader failed")
|
||||
return dialogs
|
||||
}
|
||||
}
|
||||
|
||||
class FakeExporter(
|
||||
private val srtFile: File? = null,
|
||||
private val smiFile: File? = null,
|
||||
private val vttFile: File? = null,
|
||||
private val filesForWrite: List<File> = emptyList(),
|
||||
private val shouldThrow: Boolean = false
|
||||
) : Exporter {
|
||||
|
||||
override fun write(dialogs: List<Dialog>): MutableList<File> {
|
||||
if (shouldThrow) throw RuntimeException("export failed")
|
||||
return filesForWrite.toMutableList()
|
||||
}
|
||||
|
||||
override fun writeSrt(dialogs: List<Dialog>): File =
|
||||
srtFile ?: error("srtFile not set in FakeExporter")
|
||||
|
||||
override fun writeSmi(dialogs: List<Dialog>): File =
|
||||
smiFile ?: error("smiFile not set in FakeExporter")
|
||||
|
||||
override fun writeVtt(dialogs: List<Dialog>): File =
|
||||
vttFile ?: error("vttFile not set in FakeExporter")
|
||||
}
|
||||
|
||||
|
||||
class FakeListener : ConvertListener {
|
||||
var started = 0
|
||||
var completed = 0
|
||||
var errors = 0
|
||||
var lastError: String? = null
|
||||
|
||||
override fun onStarted(inputFile: String) {
|
||||
started++
|
||||
}
|
||||
|
||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
||||
completed++
|
||||
}
|
||||
|
||||
override fun onError(inputFile: String, message: String) {
|
||||
errors++
|
||||
lastError = message
|
||||
}
|
||||
}
|
||||
|
||||
class FakeEnv(
|
||||
var canReadValue: Boolean = true,
|
||||
var reader: BaseReader? = null,
|
||||
var exporter: Exporter? = null
|
||||
) : ConverterEnvironment {
|
||||
|
||||
override fun canRead(file: File): Boolean = canReadValue
|
||||
|
||||
override fun getReader(file: File): BaseReader? = reader
|
||||
|
||||
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
|
||||
return exporter ?: error("FakeEnv.exporter must be set before calling createExporter")
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private fun makeTaskData(
|
||||
input: String = "input.srt",
|
||||
language: String = "no",
|
||||
outDir: String = "out",
|
||||
outName: String = "name",
|
||||
formats: List<SubtitleFormat> = emptyList(),
|
||||
allowOverwrite: Boolean = true
|
||||
) = ConvertTask.Data(
|
||||
inputFile = input,
|
||||
language = language,
|
||||
outputDirectory = outDir,
|
||||
outputFileName = outName,
|
||||
formats = formats,
|
||||
allowOverwrite = allowOverwrite
|
||||
)
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Tests
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"""
|
||||
Når execute() kjøres
|
||||
Hvis reader returnerer tom liste
|
||||
Så kalles onError
|
||||
"""
|
||||
)
|
||||
fun execute_emptyFile() = runTest {
|
||||
val env = FakeEnv(
|
||||
canReadValue = true,
|
||||
reader = FakeReader(emptyList())
|
||||
)
|
||||
val listener = FakeListener()
|
||||
|
||||
val converter = Converter2(
|
||||
env = env,
|
||||
listener = listener
|
||||
)
|
||||
|
||||
converter.convert(makeTaskData())
|
||||
|
||||
assertEquals(1, listener.errors)
|
||||
assertEquals(0, listener.completed)
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"""
|
||||
Når execute() kjøres
|
||||
Hvis reader returnerer dialoger
|
||||
Og exporter.write lykkes
|
||||
Så kalles onCompleted
|
||||
"""
|
||||
)
|
||||
fun execute_success() = runTest {
|
||||
val dialogs = listOf(
|
||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
||||
)
|
||||
|
||||
val env = FakeEnv(
|
||||
canReadValue = true,
|
||||
reader = FakeReader(dialogs),
|
||||
exporter = FakeExporter(srtFile = File("/fake/out.srt"))
|
||||
)
|
||||
|
||||
val listener = FakeListener()
|
||||
|
||||
val converter = Converter2(
|
||||
env = env,
|
||||
listener = listener
|
||||
)
|
||||
|
||||
converter.convert(
|
||||
makeTaskData(
|
||||
formats = listOf(SubtitleFormat.SRT)
|
||||
)
|
||||
)
|
||||
|
||||
assertEquals(1, listener.completed)
|
||||
assertEquals(listOf("/fake/out.srt"), converter.getResult())
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"""
|
||||
Når execute() kjøres
|
||||
Hvis exporter.write kaster exception
|
||||
Så kalles onError
|
||||
"""
|
||||
)
|
||||
fun execute_exportFails() = runTest {
|
||||
val dialogs = listOf(
|
||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
||||
)
|
||||
|
||||
val env = FakeEnv(
|
||||
canReadValue = true,
|
||||
reader = FakeReader(dialogs),
|
||||
exporter = FakeExporter(srtFile = File("/fake/out.srt"), shouldThrow = true)
|
||||
)
|
||||
|
||||
val listener = FakeListener()
|
||||
|
||||
val converter = Converter2(
|
||||
env = env,
|
||||
listener = listener
|
||||
)
|
||||
|
||||
converter.convert(makeTaskData())
|
||||
|
||||
assertEquals(1, listener.errors)
|
||||
assertEquals(0, listener.completed)
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"""
|
||||
Når execute() kjøres
|
||||
Hvis formats inneholder SRT og VTT
|
||||
Så brukes writeSrt og writeVtt
|
||||
"""
|
||||
)
|
||||
fun execute_multipleFormats() = runTest {
|
||||
val dialogs = listOf(
|
||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
||||
)
|
||||
|
||||
|
||||
val env = FakeEnv(
|
||||
canReadValue = true,
|
||||
reader = FakeReader(dialogs),
|
||||
exporter = FakeExporter(srtFile = File("/fake/out.srt"), vttFile = File("/fake/out.vtt"))
|
||||
)
|
||||
|
||||
val listener = FakeListener()
|
||||
|
||||
val converter = Converter2(
|
||||
env = env,
|
||||
listener = listener
|
||||
)
|
||||
|
||||
converter.convert(
|
||||
makeTaskData(
|
||||
formats = listOf(SubtitleFormat.SRT, SubtitleFormat.VTT)
|
||||
)
|
||||
)
|
||||
|
||||
assertEquals(1, listener.completed)
|
||||
assertEquals(listOf("/fake/out.srt", "/fake/out.vtt"), converter.getResult())
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,247 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.converter.listeners
|
||||
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.Task
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskReporter
|
||||
import no.iktdev.library.subtitle.classes.Dialog
|
||||
import no.iktdev.library.subtitle.classes.DialogType
|
||||
import no.iktdev.library.subtitle.classes.Time
|
||||
import no.iktdev.library.subtitle.reader.BaseReader
|
||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
||||
import no.iktdev.mediaprocessing.converter.Exporter
|
||||
import no.iktdev.mediaprocessing.converter.MockConverter
|
||||
import no.iktdev.mediaprocessing.converter.MockConverterEnvironment
|
||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
||||
import no.iktdev.mediaprocessing.converter.convert.Converter
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
|
||||
import org.junit.jupiter.api.Assertions.*
|
||||
import org.junit.jupiter.api.DisplayName
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
import kotlin.system.measureTimeMillis
|
||||
|
||||
class ConvertTaskListenerTest {
|
||||
|
||||
class ConvertTaskListenerTestImplementation : ConvertTaskListener() {
|
||||
fun getJob() = currentJob
|
||||
|
||||
|
||||
var overrideEnv: ConverterEnvironment = DefaultConverterEnvironment()
|
||||
override fun getConverterEnvironment(): ConverterEnvironment {
|
||||
return overrideEnv
|
||||
}
|
||||
|
||||
var overrideListener: ConvertListener? = null
|
||||
override fun getListener(): ConvertListener {
|
||||
if (overrideListener != null)
|
||||
return overrideListener!!
|
||||
else
|
||||
return super.getListener()
|
||||
}
|
||||
|
||||
var overrideConverter: Converter? = null
|
||||
override fun getConverter(): Converter {
|
||||
return if (overrideConverter != null)
|
||||
overrideConverter!!
|
||||
else
|
||||
super.getConverter()
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
val listener = ConvertTaskListenerTestImplementation()
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Fake environment + fake converter
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
class FakeListener : ConvertListener {
|
||||
var started = 0
|
||||
var completed = 0
|
||||
var errors = 0
|
||||
|
||||
override fun onStarted(inputFile: String) {
|
||||
started++
|
||||
}
|
||||
|
||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
||||
completed++
|
||||
}
|
||||
|
||||
override fun onError(inputFile: String, message: String) {
|
||||
errors++
|
||||
}
|
||||
}
|
||||
|
||||
class FakeExporter(
|
||||
private val files: List<File>,
|
||||
private val shouldThrow: Boolean = false
|
||||
) : Exporter {
|
||||
|
||||
override fun write(dialogs: List<Dialog>): MutableList<File> {
|
||||
if (shouldThrow) throw RuntimeException("export failed")
|
||||
return files.toMutableList()
|
||||
}
|
||||
|
||||
override fun writeSrt(dialogs: List<Dialog>) = files[0]
|
||||
override fun writeSmi(dialogs: List<Dialog>) = files[0]
|
||||
override fun writeVtt(dialogs: List<Dialog>) = files[0]
|
||||
}
|
||||
|
||||
class FakeReader(
|
||||
private val dialogs: List<Dialog>,
|
||||
private val shouldThrow: Boolean = false
|
||||
) : BaseReader() {
|
||||
override fun read(): List<Dialog> {
|
||||
if (shouldThrow) throw RuntimeException("reader failed")
|
||||
return dialogs
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
val overrideReporter = object : TaskReporter {
|
||||
override fun markClaimed(taskId: UUID, workerId: String) {}
|
||||
override fun updateLastSeen(taskId: UUID) {}
|
||||
override fun markCompleted(taskId: UUID) {}
|
||||
override fun markFailed(referenceId: UUID, taskId: UUID) {}
|
||||
override fun markCancelled(referenceId: UUID, taskId: UUID) {}
|
||||
override fun updateProgress(taskId: UUID, progress: Int) {}
|
||||
override fun log(taskId: UUID, message: String) {}
|
||||
override fun publishEvent(event: Event) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
private fun makeTask(
|
||||
formats: List<SubtitleFormat> = emptyList()
|
||||
): ConvertTask {
|
||||
return ConvertTask(
|
||||
ConvertTask.Data(
|
||||
inputFile = "input.srt",
|
||||
language = "no",
|
||||
outputDirectory = "out",
|
||||
outputFileName = "name",
|
||||
formats = formats,
|
||||
allowOverwrite = true
|
||||
)
|
||||
).apply { newReferenceId() }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
@Test
|
||||
@DisplayName("""
|
||||
Når onTask kjøres og converterer
|
||||
Hvis den bruker lengre tid
|
||||
Så:
|
||||
Skal koden vente til den er ferdig
|
||||
""")
|
||||
fun onTask_validate_delay() = runTest {
|
||||
val delay = 1000L
|
||||
val converter = MockConverter(
|
||||
delay,
|
||||
listOf("file:///potato.srt"),
|
||||
listener = FakeListener()
|
||||
)
|
||||
listener.apply {
|
||||
overrideConverter = converter
|
||||
}
|
||||
val task = makeTask()
|
||||
val event = listener.onTask(task)
|
||||
|
||||
val time = measureTimeMillis {
|
||||
val accepted = listener.accept(task, overrideReporter)
|
||||
assertTrue(accepted, "Task listener did not accept the task.")
|
||||
listener.getJob()?.join()
|
||||
assertTrue(event is ConvertTaskResultEvent)
|
||||
assertEquals(TaskStatus.Completed, (event as ConvertTaskResultEvent).status)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"""
|
||||
Når onTask kjøres
|
||||
Hvis converter lykkes
|
||||
Så returneres Completed-event
|
||||
"""
|
||||
)
|
||||
fun onTask_success() = runTest {
|
||||
val dialogs = listOf(
|
||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
||||
)
|
||||
|
||||
val env = MockConverterEnvironment(
|
||||
canReadValue = true,
|
||||
reader = FakeReader(dialogs),
|
||||
exporter = FakeExporter(listOf(File("/fake/out.srt")))
|
||||
)
|
||||
|
||||
listener.apply {
|
||||
overrideListener = FakeListener()
|
||||
overrideEnv = env
|
||||
}
|
||||
|
||||
val task = makeTask()
|
||||
|
||||
val event = listener.onTask(task) as ConvertTaskResultEvent
|
||||
|
||||
assertEquals(TaskStatus.Completed, event.status)
|
||||
assertEquals(listOf("/fake/out.srt"), event.data!!.outputFiles)
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"""
|
||||
Når onTask kjøres
|
||||
Hvis converter feiler
|
||||
Så returneres Failed-event
|
||||
"""
|
||||
)
|
||||
fun onTask_failure() = runTest {
|
||||
val env = MockConverterEnvironment(
|
||||
canReadValue = true,
|
||||
reader = FakeReader(emptyList()) // triggers FileIsNullOrEmpty
|
||||
)
|
||||
|
||||
listener.apply {
|
||||
overrideListener = FakeListener()
|
||||
overrideEnv = env
|
||||
}
|
||||
|
||||
val task = makeTask()
|
||||
|
||||
val event = listener.onTask(task) as ConvertTaskResultEvent
|
||||
|
||||
assertEquals(TaskStatus.Failed, event.status)
|
||||
assertNull(event.data)
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName(
|
||||
"""
|
||||
Når supports() kalles
|
||||
Så returnerer den true for ConvertTask og false for andre typer
|
||||
"""
|
||||
)
|
||||
fun supports_test() {
|
||||
val listener = ConvertTaskListener()
|
||||
|
||||
assertTrue(listener.supports(makeTask()))
|
||||
assertFalse(listener.supports(DummyTask()))
|
||||
}
|
||||
|
||||
class DummyTask : Task()
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
spring:
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
flyway:
|
||||
enabled: false
|
||||
locations: classpath:flyway
|
||||
autoconfigure:
|
||||
exclude:
|
||||
- org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
|
||||
|
||||
output:
|
||||
ansi:
|
||||
enabled: always
|
||||
|
||||
springdoc:
|
||||
swagger-ui:
|
||||
path: /open/swagger-ui
|
||||
|
||||
logging:
|
||||
level:
|
||||
org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN
|
||||
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping: DEBUG
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include:
|
||||
- mappings
|
||||
- health
|
||||
endpoint:
|
||||
health:
|
||||
show-details: always
|
||||
|
||||
media:
|
||||
cache: /src/cache
|
||||
outgoing: /src/output
|
||||
incoming: /src/input
|
||||
|
||||
streamit:
|
||||
address: http://streamit.service
|
||||
@ -22,7 +22,7 @@ Only one instance is supported, while multiple processer's can be run at any tim
|
||||
- Extracts info from filename
|
||||
- Extracts info from file media streams
|
||||
- Produces title and sanitized
|
||||
- py-metadata:
|
||||
- pyMetadata:
|
||||
- Picks up event
|
||||
- Searches with sources using title and sanitized
|
||||
- Produces result
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
plugins {
|
||||
id("java")
|
||||
kotlin("jvm")
|
||||
kotlin("plugin.spring")
|
||||
id("org.springframework.boot")
|
||||
id("io.spring.dependency-management")
|
||||
id("org.jetbrains.kotlin.plugin.serialization")
|
||||
kotlin("plugin.spring") version "1.5.31"
|
||||
id("org.springframework.boot") version "2.5.5"
|
||||
id("io.spring.dependency-management") version "1.0.11.RELEASE"
|
||||
id("org.jetbrains.kotlin.plugin.serialization") version "1.5.0" // Legg til Kotlin Serialization-plugin
|
||||
}
|
||||
|
||||
group = "no.iktdev.mediaprocessing"
|
||||
@ -22,42 +22,43 @@ repositories {
|
||||
}
|
||||
|
||||
|
||||
val exposedVersion = "0.44.0"
|
||||
dependencies {
|
||||
|
||||
/*Spring boot*/
|
||||
implementation("org.springframework.boot:spring-boot-starter")
|
||||
implementation("org.springframework.boot:spring-boot-starter-web")
|
||||
implementation("org.springframework.boot:spring-boot-starter-webflux")
|
||||
implementation("org.springframework.boot:spring-boot-starter-actuator")
|
||||
implementation("org.springframework.boot:spring-boot-starter-websocket")
|
||||
implementation("org.springframework:spring-tx")
|
||||
implementation("org.springframework.boot:spring-boot-starter-validation")
|
||||
|
||||
implementation("org.springframework.boot:spring-boot-starter:3.2.0")
|
||||
// implementation("org.springframework.kafka:spring-kafka:3.0.1")
|
||||
implementation("org.springframework.kafka:spring-kafka:2.8.5")
|
||||
implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3")
|
||||
|
||||
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
|
||||
implementation("com.google.code.gson:gson:2.8.9")
|
||||
implementation("org.json:json:20210307")
|
||||
|
||||
implementation(libs.exfl)
|
||||
implementation("no.iktdev:exfl:0.0.16-SNAPSHOT")
|
||||
implementation("no.iktdev.streamit.library:streamit-library-db:1.0.0-alpha14")
|
||||
implementation(libs.eventi)
|
||||
|
||||
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
|
||||
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
|
||||
|
||||
//implementation(project(mapOf("path" to ":shared")))
|
||||
|
||||
implementation(project(mapOf("path" to ":shared:ffmpeg")))
|
||||
implementation(project(mapOf("path" to ":shared:eventi")))
|
||||
implementation(project(mapOf("path" to ":shared:common")))
|
||||
implementation(project(mapOf("path" to ":shared:database")))
|
||||
|
||||
implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
|
||||
implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
|
||||
implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
|
||||
implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion")
|
||||
implementation ("mysql:mysql-connector-java:8.0.29")
|
||||
|
||||
|
||||
implementation("org.jetbrains.kotlin:kotlin-stdlib")
|
||||
implementation(kotlin("stdlib-jdk8"))
|
||||
testImplementation("org.assertj:assertj-core:3.21.0")
|
||||
|
||||
|
||||
testImplementation("junit:junit:4.12")
|
||||
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2")
|
||||
|
||||
@ -70,35 +71,30 @@ dependencies {
|
||||
testImplementation("org.mockito:mockito-core:3.+")
|
||||
testImplementation("org.assertj:assertj-core:3.4.1")
|
||||
|
||||
/*testImplementation("org.junit.vintage:junit-vintage-engine")
|
||||
testImplementation("org.junit.jupiter:junit-jupiter:5.10.1")
|
||||
testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1")
|
||||
testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.1")
|
||||
testRuntimeOnly ("org.junit.jupiter:junit-jupiter-engine:5.10.1")
|
||||
testImplementation("org.mockito:mockito-core:5.8.0") // Oppdater versjonen hvis det er nyere tilgjengelig
|
||||
testImplementation("org.mockito:mockito-junit-jupiter:5.8.0")
|
||||
testImplementation(platform("org.junit:junit-bom:5.10.1"))
|
||||
testImplementation("org.junit.platform:junit-platform-runner:1.10.1")*/
|
||||
|
||||
testImplementation(platform("org.junit:junit-bom:5.9.1"))
|
||||
testImplementation("org.junit.jupiter:junit-jupiter")
|
||||
testImplementation("org.junit.jupiter:junit-jupiter-params")
|
||||
testImplementation("junit:junit:4.13.2")
|
||||
testImplementation("org.mockito:mockito-core:3.+")
|
||||
testImplementation("org.assertj:assertj-core:3.4.1")
|
||||
testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0")
|
||||
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
|
||||
testImplementation("io.mockk:mockk:1.13.9")
|
||||
testImplementation("org.mockito:mockito-inline:5.2.0")
|
||||
testImplementation("org.mockito.kotlin:mockito-kotlin:5.2.1")
|
||||
testImplementation("org.mockito:mockito-junit-jupiter:5.11.0")
|
||||
testImplementation(project(":shared:common", configuration = "testArtifacts"))
|
||||
testImplementation(project(":shared:database", configuration = "testArtifacts"))
|
||||
testImplementation("org.springframework.boot:spring-boot-starter-test")
|
||||
val exposedVersion = "0.61.0"
|
||||
testImplementation("org.jetbrains.exposed:exposed-core:${exposedVersion}")
|
||||
|
||||
|
||||
}
|
||||
|
||||
tasks.withType<Test> {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
|
||||
|
||||
|
||||
kotlin {
|
||||
jvmToolchain(21)
|
||||
jvmToolchain(17)
|
||||
}
|
||||
|
||||
tasks.bootJar {
|
||||
|
||||
@ -1,33 +1,82 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.events.EventTypeRegistry
|
||||
import no.iktdev.eventi.tasks.TaskTypeRegistry
|
||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||
import no.iktdev.exfl.observable.Observables
|
||||
import no.iktdev.mediaprocessing.coordinator.config.ExecutablesConfig
|
||||
import no.iktdev.mediaprocessing.coordinator.config.ProcesserClientProperties
|
||||
import no.iktdev.mediaprocessing.shared.common.configs.StreamItConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry
|
||||
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
||||
import no.iktdev.mediaprocessing.shared.database.DatabaseApplication
|
||||
import no.iktdev.mediaprocessing.shared.database.DatabasebasedMediaProcessingApp
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties
|
||||
import no.iktdev.mediaprocessing.shared.common.*
|
||||
import no.iktdev.eventi.database.MySqlDataSource
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||
import no.iktdev.streamit.library.db.tables.*
|
||||
import no.iktdev.streamit.library.db.tables.helper.cast_errors
|
||||
import no.iktdev.streamit.library.db.tables.helper.data_audio
|
||||
import no.iktdev.streamit.library.db.tables.helper.data_video
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||
import org.springframework.boot.runApplication
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import javax.annotation.PreDestroy
|
||||
|
||||
val log = KotlinLogging.logger {}
|
||||
lateinit var eventDatabase: EventsDatabase
|
||||
private lateinit var eventsManager: EventsManager
|
||||
lateinit var runnerManager: RunnerManager
|
||||
|
||||
|
||||
@SpringBootApplication
|
||||
class CoordinatorApplication {
|
||||
|
||||
@Bean
|
||||
fun eventManager(): EventsManager {
|
||||
return eventsManager
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
fun onShutdown() {
|
||||
doTransactionalCleanup()
|
||||
}
|
||||
|
||||
@Transactional
|
||||
fun doTransactionalCleanup() {
|
||||
runnerManager.unlist()
|
||||
}
|
||||
|
||||
@DatabasebasedMediaProcessingApp
|
||||
class CoordinatorApplication: DatabaseApplication() {
|
||||
}
|
||||
|
||||
val ioCoroutine = CoroutinesIO()
|
||||
val defaultCoroutine = CoroutinesDefault()
|
||||
private lateinit var storeDatabase: MySqlDataSource
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
val ioCoroutine = CoroutinesIO().
|
||||
also {
|
||||
it.addListener(object : Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
log.error { "IO Coroutine" + value.printStackTrace() }
|
||||
}
|
||||
})
|
||||
}
|
||||
val defaultCoroutine = CoroutinesDefault().
|
||||
also {
|
||||
it.addListener(object : Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
log.error { "Default Coroutine" + value.printStackTrace() }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
fun getStoreDatabase(): MySqlDataSource {
|
||||
return storeDatabase
|
||||
}
|
||||
|
||||
lateinit var taskManager: TasksManager
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
|
||||
|
||||
printSharedConfig()
|
||||
|
||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||
override fun onUpdated(value: Throwable) {
|
||||
value.printStackTrace()
|
||||
@ -38,31 +87,49 @@ fun main(args: Array<String>) {
|
||||
value.printStackTrace()
|
||||
}
|
||||
})
|
||||
eventDatabase = EventsDatabase().also {
|
||||
eventsManager = EventsManager(it.database)
|
||||
}
|
||||
|
||||
|
||||
|
||||
storeDatabase = DatabaseEnvConfig.toStoredDatabase()
|
||||
storeDatabase.createDatabase()
|
||||
|
||||
|
||||
taskManager = TasksManager(eventDatabase.database)
|
||||
|
||||
|
||||
val tables = arrayOf(
|
||||
catalog,
|
||||
genre,
|
||||
movie,
|
||||
serie,
|
||||
subtitle,
|
||||
summary,
|
||||
users,
|
||||
progress,
|
||||
data_audio,
|
||||
data_video,
|
||||
cast_errors,
|
||||
titles
|
||||
)
|
||||
storeDatabase.createTables(*tables)
|
||||
|
||||
runnerManager = RunnerManager(dataSource = eventDatabase.database, applicationName = CoordinatorApplication::class.java.simpleName)
|
||||
runnerManager.assignRunner()
|
||||
|
||||
runApplication<CoordinatorApplication>(*args)
|
||||
log.info { "App Version: ${getAppVersion()}" }
|
||||
}
|
||||
//private val logger = KotlinLogging.logger {}
|
||||
|
||||
@Configuration
|
||||
open class ApplicationConfiguration() {
|
||||
init {
|
||||
EventRegistry.getEvents().let {
|
||||
EventTypeRegistry.register(it)
|
||||
}
|
||||
TaskRegistry.getTasks().let {
|
||||
TaskTypeRegistry.register(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
fun printSharedConfig() {
|
||||
log.info { "File Input: ${SharedConfig.incomingContent}" }
|
||||
log.info { "File Output: ${SharedConfig.outgoingContent}" }
|
||||
log.info { "Ffprobe: ${SharedConfig.ffprobe}" }
|
||||
log.info { "Ffmpeg: ${SharedConfig.ffmpeg}" }
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(
|
||||
value = [
|
||||
ExecutablesConfig::class,
|
||||
StreamItConfig::class,
|
||||
ProcesserClientProperties::class
|
||||
]
|
||||
)
|
||||
class CoordinatorConfig
|
||||
/*log.info { "Database: ${DatabaseConfig.database} @ ${DatabaseConfig.address}:${DatabaseConfig.port}" }
|
||||
log.info { "Username: ${DatabaseConfig.username}" }
|
||||
log.info { "Password: ${if (DatabaseConfig.password.isNullOrBlank()) "Is not set" else "Is set"}" }*/
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.config.ExecutablesConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.configs.MediaPaths
|
||||
import no.iktdev.mediaprocessing.shared.common.configs.StreamItConfig
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
|
||||
@Service
|
||||
class CoordinatorEnv(
|
||||
val streamIt: StreamItConfig,
|
||||
val exec: ExecutablesConfig,
|
||||
val media: MediaPaths
|
||||
) {
|
||||
val streamitAddress = streamIt.address
|
||||
val ffprobe = exec.ffprobe
|
||||
|
||||
val cachedContent = File(media.cache)
|
||||
val outgoingContent = File(media.outgoing)
|
||||
val incomingContent = File(media.incoming)
|
||||
val preference: File = File("/data/config/preference.json")
|
||||
|
||||
}
|
||||
@ -0,0 +1,107 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.eventi.data.EventMetadata
|
||||
import no.iktdev.eventi.data.EventStatus
|
||||
import no.iktdev.eventi.data.eventId
|
||||
import no.iktdev.eventi.implementations.ActiveMode
|
||||
import no.iktdev.eventi.implementations.EventCoordinator
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaProcessStartEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.PermitWorkCreationEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.data.StartEventData
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent
|
||||
import org.springframework.context.ApplicationContext
|
||||
import org.springframework.context.event.EventListener
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class Coordinator(
|
||||
@Autowired
|
||||
override var applicationContext: ApplicationContext,
|
||||
@Autowired
|
||||
override var eventManager: EventsManager
|
||||
|
||||
) : EventCoordinator<Event, EventsManager>() {
|
||||
|
||||
@EventListener(ApplicationReadyEvent::class)
|
||||
fun onApplicationReady() {
|
||||
onReady()
|
||||
}
|
||||
|
||||
fun getProducerName(): String {
|
||||
return this::class.java.simpleName
|
||||
}
|
||||
|
||||
public fun startProcess(file: File, type: ProcessType) {
|
||||
val operations: List<OperationEvents> = listOf(
|
||||
OperationEvents.ENCODE,
|
||||
OperationEvents.EXTRACT,
|
||||
OperationEvents.CONVERT
|
||||
)
|
||||
startProcess(file, type, operations)
|
||||
}
|
||||
|
||||
fun startProcess(file: File, type: ProcessType, operations: List<OperationEvents>): UUID {
|
||||
val referenceId: UUID = UUID.randomUUID()
|
||||
val event = MediaProcessStartEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = referenceId.toString(),
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = StartEventData(
|
||||
file = file.absolutePath,
|
||||
type = type,
|
||||
operations = operations
|
||||
)
|
||||
)
|
||||
|
||||
produceNewEvent(event)
|
||||
return referenceId
|
||||
}
|
||||
|
||||
fun permitWorkToProceedOn(referenceId: String, events: List<Event>, message: String) {
|
||||
val defaultRequiredBy = listOf(Events.EncodeParameterCreated, Events.ExtractParameterCreated)
|
||||
val eventToAttachTo = if (events.any { it.eventType in defaultRequiredBy }) {
|
||||
events.findLast { it.eventType in defaultRequiredBy }
|
||||
} else events.find { it.eventType == Events.ProcessStarted }
|
||||
if (eventToAttachTo == null) {
|
||||
log.error { "No event to attach permit to" }
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
produceNewEvent(
|
||||
PermitWorkCreationEvent(
|
||||
metadata = EventMetadata(
|
||||
referenceId = referenceId,
|
||||
derivedFromEventId = eventToAttachTo.eventId(),
|
||||
status = EventStatus.Success,
|
||||
source = getProducerName()
|
||||
),
|
||||
data = message
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override fun getActiveTaskMode(): ActiveMode {
|
||||
if (runnerManager.iAmSuperseded()) {
|
||||
// This will let the application complete but not consume new
|
||||
taskMode = ActiveMode.Passive
|
||||
}
|
||||
return taskMode
|
||||
}
|
||||
|
||||
override fun updateEnabledState(): Boolean {
|
||||
isEnabled = runnerManager.amIEnabled()
|
||||
return isEnabled
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.EventsListenerContract
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
|
||||
abstract class CoordinatorEventListener(): EventsListenerContract<EventsManager, Coordinator>() {
|
||||
abstract override val produceEvent: Events
|
||||
abstract override val listensForEvents: List<Events>
|
||||
abstract override var coordinator: Coordinator?
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.model.ProgressUpdate
|
||||
import org.springframework.stereotype.Service
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
@Service
|
||||
class CoordinatorService {
|
||||
|
||||
private val progressMap = ConcurrentHashMap<String, ProgressUpdate>()
|
||||
|
||||
fun updateProgress(update: ProgressUpdate) {
|
||||
progressMap[update.taskId] = update
|
||||
}
|
||||
|
||||
fun getProgress(taskId: String): ProgressUpdate? =
|
||||
progressMap[taskId]
|
||||
|
||||
fun getProgress(): List<ProgressUpdate> =
|
||||
progressMap.values.toList()
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import dev.vishna.watchservice.KWatchChannel
|
||||
import dev.vishna.watchservice.asWatchChannel
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import java.io.File
|
||||
|
||||
data class FileWatcher(val file: File, val watcher: KWatchChannel)
|
||||
|
||||
suspend fun File.asWatcher(block: suspend (KWatchChannel) -> Unit): FileWatcher {
|
||||
val channel = this.asWatchChannel()
|
||||
block(channel)
|
||||
return FileWatcher(this, channel)
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.launch
|
||||
import no.iktdev.eventi.events.EventDispatcher
|
||||
import no.iktdev.eventi.events.EventPollerImplementation
|
||||
import no.iktdev.eventi.events.SequenceDispatchQueue
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
||||
import org.springframework.context.SmartLifecycle
|
||||
import org.springframework.context.annotation.DependsOn
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
|
||||
@Component
|
||||
class EventPollerAdministrator(
|
||||
private val eventPoller: EventPoller
|
||||
) : SmartLifecycle {
|
||||
|
||||
private var running = false
|
||||
|
||||
var job: Job? = null
|
||||
override fun start() {
|
||||
job = CoroutineScope(Dispatchers.Default).launch {
|
||||
eventPoller.start()
|
||||
}
|
||||
running = true
|
||||
}
|
||||
|
||||
override fun stop() {
|
||||
job?.cancel()
|
||||
}
|
||||
|
||||
override fun isRunning() = running
|
||||
}
|
||||
|
||||
|
||||
val sequenceDispatcher = SequenceDispatchQueue(8)
|
||||
val dispatcher = EventDispatcher(eventStore = EventStore)
|
||||
|
||||
@Component
|
||||
@DependsOn("ExposedInit")
|
||||
class EventPoller: EventPollerImplementation(eventStore = EventStore, dispatchQueue = sequenceDispatcher, dispatcher = dispatcher) {
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||
import no.iktdev.mediaprocessing.shared.common.database.tables.*
|
||||
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||
|
||||
class EventsDatabase() {
|
||||
val database = DatabaseEnvConfig.toEventsDatabase()
|
||||
val tables = listOf(
|
||||
events, // For kafka
|
||||
allEvents,
|
||||
tasks,
|
||||
runners,
|
||||
filesProcessed,
|
||||
files
|
||||
)
|
||||
|
||||
init {
|
||||
database.createDatabase()
|
||||
database.createTables(*tables.toTypedArray())
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.context.annotation.Import
|
||||
|
||||
@Configuration
|
||||
class SocketLocalInit: SocketImplementation() {
|
||||
}
|
||||
|
||||
|
||||
@ -1,190 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.JsonObject
|
||||
import com.google.gson.JsonParser
|
||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.AudioCodec
|
||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoCodec
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// MODELLER
|
||||
// ------------------------------------------------------------
|
||||
|
||||
data class PeferenceConfig(
|
||||
val processer: ProcesserPreference,
|
||||
val language: LanguagePreference
|
||||
)
|
||||
|
||||
data class LanguagePreference(
|
||||
val preferredAudio: List<String>,
|
||||
val preferredSubtitles: List<String>,
|
||||
|
||||
val preferOriginal: Boolean = true,
|
||||
val avoidDub: Boolean = true,
|
||||
|
||||
// NEW: Prioritet for hvilket subtitle-format som skal brukes som master
|
||||
val subtitleFormatPriority: List<String> = listOf("ass", "srt", "vtt", "smi"),
|
||||
|
||||
// NEW: Hvordan subtitles skal velges
|
||||
val subtitleSelectionMode: SubtitleSelectionMode = SubtitleSelectionMode.DialogueOnly
|
||||
) {
|
||||
companion object {
|
||||
fun default() = LanguagePreference(
|
||||
preferredAudio = listOf("eng", "nor", "jpn"),
|
||||
preferredSubtitles = listOf("eng", "nor"),
|
||||
preferOriginal = true,
|
||||
avoidDub = true,
|
||||
subtitleFormatPriority = listOf("ass", "srt", "vtt", "smi"),
|
||||
subtitleSelectionMode = SubtitleSelectionMode.DialogueOnly
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
enum class SubtitleSelectionMode {
|
||||
DialogueOnly, // Kun dialog
|
||||
DialogueAndForced, // Dialog + forced
|
||||
All // Alle typer
|
||||
}
|
||||
|
||||
|
||||
data class ProcesserPreference(
|
||||
val videoPreference: VideoPreference? = null,
|
||||
val audioPreference: AudioPreference? = null
|
||||
) {
|
||||
companion object {
|
||||
fun default(): ProcesserPreference {
|
||||
return ProcesserPreference(
|
||||
videoPreference = VideoPreference(VideoCodec.Hevc(), false),
|
||||
audioPreference = AudioPreference(AudioCodec.Aac())
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data class VideoPreference(
|
||||
val codec: VideoCodec,
|
||||
val enforceMkv: Boolean = false
|
||||
)
|
||||
|
||||
data class AudioPreference(
|
||||
val codec: AudioCodec
|
||||
)
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// PREFERENCE COMPONENT
|
||||
// ------------------------------------------------------------
|
||||
|
||||
@Component
|
||||
class Preference(private val coordinatorEnv: CoordinatorEnv) {
|
||||
|
||||
private val gson = Gson()
|
||||
private val lock = Any()
|
||||
|
||||
/**
|
||||
* Leser hele configen, men bevarer ukjente felter.
|
||||
*/
|
||||
fun getFullConfig(): PeferenceConfig {
|
||||
val file = coordinatorEnv.preference
|
||||
|
||||
if (!file.exists()) {
|
||||
val default = PeferenceConfig(
|
||||
processer = ProcesserPreference.default(),
|
||||
language = LanguagePreference.default()
|
||||
)
|
||||
writeJsonObject(JsonObject().apply {
|
||||
add("processer", gson.toJsonTree(default.processer))
|
||||
add("language", gson.toJsonTree(default.language))
|
||||
}, file)
|
||||
return default
|
||||
}
|
||||
|
||||
val root = try {
|
||||
JsonParser.parseString(file.readText()).asJsonObject
|
||||
} catch (e: Exception) {
|
||||
return PeferenceConfig(
|
||||
processer = ProcesserPreference.default(),
|
||||
language = LanguagePreference.default()
|
||||
)
|
||||
}
|
||||
|
||||
val processer = try {
|
||||
gson.fromJson(root.get("processer"), ProcesserPreference::class.java)
|
||||
?: ProcesserPreference.default()
|
||||
} catch (_: Exception) {
|
||||
ProcesserPreference.default()
|
||||
}
|
||||
|
||||
val language = try {
|
||||
gson.fromJson(root.get("language"), LanguagePreference::class.java)
|
||||
?: LanguagePreference.default()
|
||||
} catch (_: Exception) {
|
||||
LanguagePreference.default()
|
||||
}
|
||||
|
||||
return PeferenceConfig(processer, language)
|
||||
}
|
||||
|
||||
fun getProcesserPreference(): ProcesserPreference =
|
||||
getFullConfig().processer
|
||||
|
||||
fun getLanguagePreference(): LanguagePreference =
|
||||
getFullConfig().language
|
||||
|
||||
/**
|
||||
* Oppdaterer kun processer-delen og bevarer resten av JSON.
|
||||
*/
|
||||
fun saveProcesserPreference(pref: ProcesserPreference) {
|
||||
val file = coordinatorEnv.preference
|
||||
synchronized(lock) {
|
||||
val root = readOrEmpty(file)
|
||||
root.add("processer", gson.toJsonTree(pref))
|
||||
writeJsonObject(root, file)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Oppdaterer kun language-delen og bevarer resten av JSON.
|
||||
*/
|
||||
fun saveLanguagePreference(pref: LanguagePreference) {
|
||||
val file = coordinatorEnv.preference
|
||||
synchronized(lock) {
|
||||
val root = readOrEmpty(file)
|
||||
root.add("language", gson.toJsonTree(pref))
|
||||
writeJsonObject(root, file)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Overskriver hele configen (brukes hvis FE sender alt).
|
||||
*/
|
||||
fun saveFullConfig(cfg: PeferenceConfig) {
|
||||
val file = coordinatorEnv.preference
|
||||
synchronized(lock) {
|
||||
val root = JsonObject()
|
||||
root.add("processer", gson.toJsonTree(cfg.processer))
|
||||
root.add("language", gson.toJsonTree(cfg.language))
|
||||
writeJsonObject(root, file)
|
||||
}
|
||||
}
|
||||
|
||||
private fun readOrEmpty(file: File): JsonObject =
|
||||
if (file.exists()) {
|
||||
try {
|
||||
JsonParser.parseString(file.readText()).asJsonObject
|
||||
} catch (_: Exception) {
|
||||
JsonObject()
|
||||
}
|
||||
} else JsonObject()
|
||||
|
||||
private fun writeJsonObject(obj: JsonObject, file: File) {
|
||||
try {
|
||||
file.parentFile?.mkdirs()
|
||||
file.writeText(gson.toJson(obj))
|
||||
} catch (e: IOException) {
|
||||
throw RuntimeException("Failed to write preference file", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import org.springframework.stereotype.Component
|
||||
import org.springframework.web.reactive.function.client.WebClient
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
@Component
|
||||
class ProcesserClient(
|
||||
private val processerWebClient: WebClient
|
||||
) {
|
||||
|
||||
fun fetchLog(path: String): Mono<String> =
|
||||
processerWebClient.get()
|
||||
.uri { it.path("/state/log").queryParam("path", path).build() }
|
||||
.retrieve()
|
||||
.bodyToMono(String::class.java)
|
||||
|
||||
fun ping(): Mono<String> =
|
||||
processerWebClient.get()
|
||||
.uri("/actuator/health")
|
||||
.retrieve()
|
||||
.bodyToMono(String::class.java)
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import org.springframework.boot.web.client.RestTemplateBuilder
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.web.client.RestTemplate
|
||||
|
||||
@Configuration
|
||||
class RestTemplateConfig {
|
||||
|
||||
@Configuration
|
||||
class RestTemplateConfig(
|
||||
private val coordinatorEnv: CoordinatorEnv
|
||||
) {
|
||||
|
||||
@Bean
|
||||
fun streamitRestTemplate(): RestTemplate {
|
||||
return RestTemplateBuilder()
|
||||
.rootUri(coordinatorEnv.streamitAddress)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,74 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskPollerImplementation
|
||||
import no.iktdev.eventi.tasks.TaskReporter
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
import org.springframework.boot.ApplicationArguments
|
||||
import org.springframework.boot.ApplicationRunner
|
||||
import org.springframework.stereotype.Component
|
||||
import org.springframework.stereotype.Service
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class TaskPollerAdministrator(
|
||||
private val taskPoller: TaskPoller,
|
||||
): ApplicationRunner {
|
||||
override fun run(args: ApplicationArguments?) {
|
||||
CoroutineScope(Dispatchers.Default).launch {
|
||||
taskPoller.start()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Service
|
||||
class TaskPoller(
|
||||
private val reporter: TaskReporter,
|
||||
) : TaskPollerImplementation(
|
||||
taskStore = TaskStore,
|
||||
reporterFactory = { reporter } // én reporter brukes for alle tasks
|
||||
) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Component
|
||||
class DefaultTaskReporter() : TaskReporter {
|
||||
override fun markClaimed(taskId: UUID, workerId: String) {
|
||||
TaskStore.claim(taskId, workerId)
|
||||
}
|
||||
|
||||
override fun updateLastSeen(taskId: UUID) {
|
||||
TaskStore.heartbeat(taskId)
|
||||
}
|
||||
|
||||
override fun markCompleted(taskId: UUID) {
|
||||
TaskStore.markConsumed(taskId, TaskStatus.Completed)
|
||||
}
|
||||
|
||||
override fun markFailed(referenceId: UUID, taskId: UUID) {
|
||||
TaskStore.markConsumed(taskId, TaskStatus.Failed)
|
||||
}
|
||||
|
||||
override fun markCancelled(referenceId: UUID, taskId: UUID) {
|
||||
TaskStore.markConsumed(taskId, TaskStatus.Cancelled)
|
||||
}
|
||||
|
||||
override fun updateProgress(taskId: UUID, progress: Int) {
|
||||
// Not to be implemented for this application
|
||||
}
|
||||
|
||||
override fun log(taskId: UUID, message: String) {
|
||||
// Not to be implemented for this application
|
||||
}
|
||||
|
||||
override fun publishEvent(event: Event) {
|
||||
EventStore.persist(event)
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.config
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties
|
||||
|
||||
@ConfigurationProperties(prefix = "executables")
|
||||
data class ExecutablesConfig(
|
||||
val ffprobe: String
|
||||
)
|
||||
@ -1,12 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.config
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.web.reactive.function.client.WebClient
|
||||
|
||||
@ConfigurationProperties(prefix = "processer")
|
||||
data class ProcesserClientProperties(
|
||||
val baseUrl: String
|
||||
)
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.config
|
||||
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
import org.springframework.web.reactive.function.client.WebClient
|
||||
|
||||
@Configuration
|
||||
class WebClients(
|
||||
private val processerClientProperties: ProcesserClientProperties
|
||||
|
||||
) {
|
||||
@Bean
|
||||
fun webClient(): WebClient.Builder =
|
||||
WebClient
|
||||
.builder()
|
||||
.codecs { it.defaultCodecs().maxInMemorySize(10 * 1024 * 1024) }
|
||||
@Bean
|
||||
fun processerWebClient(builder: WebClient.Builder): WebClient {
|
||||
return builder.baseUrl(processerClientProperties.baseUrl).build()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import com.google.gson.Gson
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.RequestWorkProceed
|
||||
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.bind.annotation.PostMapping
|
||||
import org.springframework.web.bind.annotation.RequestBody
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
|
||||
@Controller
|
||||
@RequestMapping(path = ["/action"])
|
||||
class ActionEventController(@Autowired var coordinator: Coordinator, @Autowired var eventsManager: EventsManager) {
|
||||
|
||||
|
||||
@PostMapping("/flow/proceed")
|
||||
fun permitRunOnSequence(@RequestBody data: RequestWorkProceed): ResponseEntity<String> {
|
||||
|
||||
val set = eventsManager.getEventsWith(data.referenceId)
|
||||
if (set.isEmpty()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(data))
|
||||
}
|
||||
coordinator.permitWorkToProceedOn(data.referenceId, set, "Requested by ${data.source}")
|
||||
|
||||
//EVENT_MEDIA_WORK_PROCEED_PERMITTED("event:media-work-proceed:permitted")
|
||||
return ResponseEntity.ok(null)
|
||||
}
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.eventi.models.store.PersistedEvent
|
||||
import no.iktdev.mediaprocessing.coordinator.services.EventService
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.EventQuery
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.SequenceEvent
|
||||
import org.springframework.web.bind.annotation.*
|
||||
import java.util.*
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/events")
|
||||
class EventsController(
|
||||
private val paging: EventService
|
||||
) {
|
||||
|
||||
@GetMapping()
|
||||
fun getEvents(query: EventQuery): Paginated<PersistedEvent> {
|
||||
return paging.getEvents(query)
|
||||
}
|
||||
|
||||
@GetMapping("/sequence/{referenceId}")
|
||||
fun getEventSequence(
|
||||
@PathVariable referenceId: UUID,
|
||||
@RequestParam(required = false) beforeEventId: UUID?,
|
||||
@RequestParam(required = false) afterEventId: UUID?,
|
||||
@RequestParam(defaultValue = "50") limit: Int
|
||||
): List<SequenceEvent> {
|
||||
return paging.getPagedEvents(
|
||||
referenceId = referenceId,
|
||||
beforeEventId = beforeEventId,
|
||||
afterEventId = afterEventId,
|
||||
limit = limit
|
||||
)
|
||||
}
|
||||
|
||||
@GetMapping("/history/{referenceId}/effective")
|
||||
fun getEffectiveHistory(
|
||||
@PathVariable referenceId: UUID,
|
||||
): List<PersistedEvent> {
|
||||
return paging.getEffectiveHistory(referenceId)
|
||||
}
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.FileTableItem
|
||||
import no.iktdev.mediaprocessing.shared.database.queries.FilesTableQueries
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/files")
|
||||
class FilesController {
|
||||
|
||||
@GetMapping()
|
||||
fun getFilesInDatabase(): ResponseEntity<List<FileTableItem>?>? {
|
||||
val files = FilesTableQueries().getFiles()
|
||||
return ResponseEntity.ok(files)
|
||||
}
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
|
||||
import no.iktdev.mediaprocessing.coordinator.services.CoordinatorHealthService
|
||||
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/health")
|
||||
class HealthController(
|
||||
private val healthService: CoordinatorHealthService
|
||||
) {
|
||||
|
||||
@GetMapping
|
||||
fun getHealth(): CoordinatorHealth {
|
||||
return healthService.getHealth()
|
||||
}
|
||||
|
||||
@GetMapping("/events")
|
||||
fun getEventRate(): EventRate = healthService.getEventRate()
|
||||
|
||||
@GetMapping("/storage")
|
||||
fun getDiskStatus(): List<DiskInfo> = healthService.getDiskHealth()
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorService
|
||||
import no.iktdev.mediaprocessing.coordinator.services.SseHub
|
||||
import no.iktdev.mediaprocessing.shared.common.model.ProgressUpdate
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.*
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/internal")
|
||||
class InternalProcesserController(
|
||||
private val coordinator: CoordinatorService,
|
||||
private val hub: SseHub
|
||||
) {
|
||||
|
||||
@PostMapping("/progress")
|
||||
fun receiveProgress(@RequestBody update: ProgressUpdate): ResponseEntity<Void> {
|
||||
coordinator.updateProgress(update)
|
||||
|
||||
hub.broadcast("progress", update)
|
||||
return ResponseEntity.ok().build()
|
||||
}
|
||||
|
||||
@GetMapping("/progress")
|
||||
fun getAllProgress(): List<ProgressUpdate> {
|
||||
return coordinator.getProgress()
|
||||
}
|
||||
|
||||
@GetMapping("/sse")
|
||||
fun stream(): SseEmitter {
|
||||
return hub.createEmitter()
|
||||
}
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.ProcesserClient
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.RequestParam
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
import reactor.core.publisher.Mono
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/log")
|
||||
class LogController(
|
||||
private val processerClient: ProcesserClient
|
||||
) {
|
||||
|
||||
@GetMapping
|
||||
fun getLog(@RequestParam path: String): Mono<String> {
|
||||
return processerClient.fetchLog(path)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.services.CommandService
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.requests.StartProcessRequest
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.PostMapping
|
||||
import org.springframework.web.bind.annotation.RequestBody
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/operations")
|
||||
class OperationsController(
|
||||
private val commandService: CommandService
|
||||
) {
|
||||
|
||||
@PostMapping("/start")
|
||||
fun startProcess(@RequestBody req: StartProcessRequest): ResponseEntity<Map<String, String>> {
|
||||
return when (val result = commandService.startProcess(req)) {
|
||||
is CommandService.StartResult.Accepted -> ResponseEntity
|
||||
.accepted()
|
||||
.body(
|
||||
mapOf(
|
||||
"referenceId" to result.referenceId.toString(),
|
||||
"status" to "accepted",
|
||||
"message" to "Process accepted and StartedEvent created"
|
||||
)
|
||||
)
|
||||
|
||||
is CommandService.StartResult.Rejected -> ResponseEntity
|
||||
.badRequest()
|
||||
.body(
|
||||
mapOf(
|
||||
"status" to "rejected",
|
||||
"message" to result.reason
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
|
||||
@Controller
|
||||
@RequestMapping(path = ["/polls"])
|
||||
class PollController(@Autowired var coordinator: Coordinator) {
|
||||
|
||||
@GetMapping()
|
||||
fun polls(): String {
|
||||
val stat = coordinator.getActivePolls()
|
||||
return "Active Polls ${stat.active}/${stat.total}"
|
||||
}
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.PeferenceConfig
|
||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
||||
import no.iktdev.mediaprocessing.coordinator.ProcesserPreference
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.*
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/preference")
|
||||
class PreferenceController(
|
||||
private val preference: Preference
|
||||
) {
|
||||
|
||||
@GetMapping
|
||||
fun getFull(): ResponseEntity<PeferenceConfig> =
|
||||
ResponseEntity.ok(preference.getFullConfig())
|
||||
|
||||
@PutMapping
|
||||
fun putFull(@RequestBody body: PeferenceConfig): ResponseEntity<PeferenceConfig> {
|
||||
preference.saveFullConfig(body)
|
||||
return ResponseEntity.ok(preference.getFullConfig())
|
||||
}
|
||||
|
||||
@GetMapping("/processer")
|
||||
fun getProcesser(): ResponseEntity<ProcesserPreference> =
|
||||
ResponseEntity.ok(preference.getProcesserPreference())
|
||||
|
||||
@PutMapping("/processer")
|
||||
fun putProcesser(@RequestBody body: ProcesserPreference): ResponseEntity<ProcesserPreference> {
|
||||
preference.saveProcesserPreference(body)
|
||||
return ResponseEntity.ok(preference.getProcesserPreference())
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import org.springframework.boot.actuate.health.HealthEndpoint
|
||||
import org.springframework.boot.actuate.health.Status
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/system")
|
||||
class ReadinessController(
|
||||
private val healthEndpoint: HealthEndpoint
|
||||
) {
|
||||
|
||||
@GetMapping("/ready")
|
||||
fun ready(): ResponseEntity<String> {
|
||||
val health = healthEndpoint.health()
|
||||
|
||||
return if (health.status == Status.UP) {
|
||||
ResponseEntity.ok("READY")
|
||||
} else {
|
||||
ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
|
||||
.body("NOT_READY: ${health.status}")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import com.google.gson.Gson
|
||||
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventRequest
|
||||
import no.iktdev.mediaprocessing.shared.common.contract.dto.OperationEvents
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.stereotype.Controller
|
||||
import org.springframework.web.bind.annotation.PostMapping
|
||||
import org.springframework.web.bind.annotation.RequestBody
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.ResponseStatus
|
||||
import java.io.File
|
||||
|
||||
@Controller
|
||||
@RequestMapping(path = ["/request"])
|
||||
class RequestEventController(@Autowired var coordinator: Coordinator) {
|
||||
|
||||
@PostMapping("/convert")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestConvert(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload.file)
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, payload.mode, listOf(OperationEvents.CONVERT)).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
|
||||
@PostMapping("/extract")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestExtract(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(payload))
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, payload.mode, listOf(OperationEvents.EXTRACT)).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
|
||||
@PostMapping("/encode")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestEncode(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(payload))
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, payload.mode, listOf(OperationEvents.ENCODE)).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
|
||||
@PostMapping("/all")
|
||||
@ResponseStatus(HttpStatus.OK)
|
||||
fun requestAll(@RequestBody payload: EventRequest): ResponseEntity<String> {
|
||||
var referenceId: String?
|
||||
try {
|
||||
val file = File(payload.file)
|
||||
if (!file.exists()) {
|
||||
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload.file)
|
||||
}
|
||||
referenceId = coordinator.startProcess(file, type = payload.mode).toString()
|
||||
|
||||
} catch (e: Exception) {
|
||||
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(payload))
|
||||
}
|
||||
return ResponseEntity.ok(referenceId)
|
||||
}
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.translate.ApiResponse
|
||||
import no.iktdev.mediaprocessing.coordinator.services.SequenceAggregatorService
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.SequenceSummary
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.*
|
||||
import java.util.*
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/sequences")
|
||||
class SequenceController(
|
||||
private val aggregator: SequenceAggregatorService
|
||||
) {
|
||||
|
||||
@GetMapping("/active")
|
||||
fun getActive(): List<SequenceSummary> {
|
||||
return aggregator.getActiveSequences()
|
||||
}
|
||||
|
||||
@GetMapping("/recent")
|
||||
fun getRecent(
|
||||
@RequestParam(defaultValue = "15") limit: Int
|
||||
): List<SequenceSummary> {
|
||||
return aggregator.getRecentSequences(limit)
|
||||
}
|
||||
|
||||
@PostMapping("/{referenceId}/continue")
|
||||
fun continueSequence(
|
||||
@PathVariable referenceId: UUID
|
||||
): ResponseEntity<ApiResponse> {
|
||||
return try {
|
||||
|
||||
val id = EventStore.createManuallyContinueEvent(referenceId)
|
||||
|
||||
ResponseEntity.ok(
|
||||
ApiResponse(
|
||||
ok = true,
|
||||
message = "Sequence continued, event $id created!"
|
||||
)
|
||||
)
|
||||
|
||||
} catch (ex: Exception) {
|
||||
ResponseEntity
|
||||
.status(HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
.body(
|
||||
ApiResponse(
|
||||
ok = false,
|
||||
message = ex.message ?: "Unknown error"
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@PostMapping("/{referenceId}/delete")
|
||||
fun deleteSequences(
|
||||
@PathVariable referenceId: UUID
|
||||
): ResponseEntity<ApiResponse> {
|
||||
return try {
|
||||
|
||||
val id = EventStore.deleteSequence(referenceId)
|
||||
|
||||
ResponseEntity.ok(
|
||||
ApiResponse(
|
||||
ok = true,
|
||||
message = "Sequence deleted, Event id for deletion marking is $id"
|
||||
)
|
||||
)
|
||||
|
||||
} catch (ex: Exception) {
|
||||
ResponseEntity
|
||||
.status(HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
.body(
|
||||
ApiResponse(
|
||||
ok = false,
|
||||
message = ex.message ?: "Unknown error"
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.controller
|
||||
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.translate.CoordinatorTaskTransferDto
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.translate.toCoordinatorTransferDto
|
||||
import no.iktdev.mediaprocessing.coordinator.services.EventService
|
||||
import no.iktdev.mediaprocessing.coordinator.services.TaskService
|
||||
import no.iktdev.mediaprocessing.ffmpeg.util.UtcNow
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.ResetTaskResponse
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.TaskQuery
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.map
|
||||
import org.springframework.http.HttpStatus
|
||||
import org.springframework.http.ResponseEntity
|
||||
import org.springframework.web.bind.annotation.GetMapping
|
||||
import org.springframework.web.bind.annotation.PathVariable
|
||||
import org.springframework.web.bind.annotation.RequestMapping
|
||||
import org.springframework.web.bind.annotation.RestController
|
||||
import java.util.*
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/tasks")
|
||||
class TaskController(
|
||||
private val taskService: TaskService,
|
||||
private val eventService: EventService
|
||||
) {
|
||||
|
||||
@GetMapping("/active")
|
||||
fun getActiveTasks(): List<CoordinatorTaskTransferDto> {
|
||||
val tasks = taskService.getActiveTasks()
|
||||
val logEvents = eventService.getTaskEventResultsWithLogs(tasks.map { it.referenceId }.toSet())
|
||||
return tasks.map { it.toCoordinatorTransferDto(logEvents) }
|
||||
}
|
||||
|
||||
@GetMapping
|
||||
fun getPagedTasks(query: TaskQuery): Paginated<CoordinatorTaskTransferDto> {
|
||||
val paginatedTasks = taskService.getPagedTasks(query)
|
||||
val logEvents = eventService.getTaskEventResultsWithLogs(paginatedTasks.items.map { it.referenceId }.toSet())
|
||||
|
||||
return paginatedTasks.map { it.toCoordinatorTransferDto(logEvents) }
|
||||
}
|
||||
|
||||
|
||||
|
||||
@GetMapping("/{id}")
|
||||
fun getTask(@PathVariable id: UUID): CoordinatorTaskTransferDto? {
|
||||
val tasks = taskService.getTaskById(id) ?: return null
|
||||
val logEvents = eventService.getTaskEventResultsWithLogs(setOf(tasks.referenceId))
|
||||
return tasks.toCoordinatorTransferDto(logEvents)
|
||||
}
|
||||
|
||||
|
||||
@GetMapping("/{taskId}/reset")
|
||||
fun resetTask(@PathVariable taskId: UUID, forced: Boolean = false): ResponseEntity<ResetTaskResponse> {
|
||||
val task = taskService.getTaskById(taskId)
|
||||
?: return ResponseEntity.notFound().build()
|
||||
|
||||
val referenceId = task.referenceId
|
||||
if (eventService.isSequenceDeleted(referenceId)) {
|
||||
return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build()
|
||||
}
|
||||
|
||||
// 1. Opprett DeleteEvent
|
||||
val deletedId = eventService.deleteTaskFailureForReset(referenceId, taskId)
|
||||
if (deletedId == null) {
|
||||
if (forced) {
|
||||
eventService.createForcedTaskResetAuditEvent(referenceId, taskId)
|
||||
} else {
|
||||
return ResponseEntity.status(HttpStatus.CONFLICT).build()
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Reset task
|
||||
val success = taskService.resetFailedTask(taskId)
|
||||
|
||||
return ResponseEntity.ok(
|
||||
ResetTaskResponse(
|
||||
taskId = taskId,
|
||||
referenceId = referenceId,
|
||||
reset = success,
|
||||
deletedEventId = deletedId,
|
||||
resetAt = UtcNow()
|
||||
)
|
||||
)
|
||||
}
|
||||
@GetMapping("/{taskId}/reset/force")
|
||||
fun resetTaskForce(@PathVariable taskId: UUID): ResponseEntity<ResetTaskResponse> {
|
||||
return resetTask(taskId, true)
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.coordination
|
||||
|
||||
/**
|
||||
* Class to handle messages from websockets, produced by Processer instances.
|
||||
* This is due to keep a overview of progress by processer
|
||||
*/
|
||||
class ProcesserSocketMessageListener {
|
||||
|
||||
|
||||
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.dto
|
||||
|
||||
import java.util.*
|
||||
|
||||
data class LogAssociatedIds(
|
||||
val referenceId: UUID,
|
||||
val ids: Set<UUID>,
|
||||
val logFile: String
|
||||
)
|
||||
@ -1,26 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.dto.health
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
data class CoordinatorHealth(
|
||||
val status: CoordinatorHealthStatus,
|
||||
val abandonedTasks: Int,
|
||||
val stalledTasks: Int,
|
||||
val activeTasks: Int,
|
||||
val queuedTasks: Int,
|
||||
val failedTasks: Int,
|
||||
val sequencesOnHold: Int,
|
||||
val lastActivity: Instant?,
|
||||
|
||||
// IDs for UI linking
|
||||
val abandonedTaskIds: List<String>,
|
||||
val stalledTaskIds: List<String>,
|
||||
val sequencesOnHoldIds: List<String>,
|
||||
val overdueSequenceIds: List<String>,
|
||||
|
||||
|
||||
// Detailed sequence info
|
||||
val overdueSequences: List<SequenceHealth>,
|
||||
|
||||
val details: Map<String, Any?>
|
||||
)
|
||||
@ -1,7 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.dto.health
|
||||
|
||||
enum class CoordinatorHealthStatus {
|
||||
HEALTHY,
|
||||
DEGRADED,
|
||||
UNHEALTHY
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.dto.health
|
||||
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
|
||||
data class SequenceHealth(
|
||||
val referenceId: String,
|
||||
|
||||
val age: Duration,
|
||||
val expected: Duration,
|
||||
val lastEventAt: Instant,
|
||||
val eventCount: Int,
|
||||
|
||||
// nye felter
|
||||
val startTime: Instant,
|
||||
val expectedFinishTime: Instant,
|
||||
val overdueDuration: Duration,
|
||||
val isOverdue: Boolean
|
||||
)
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.dto.rate
|
||||
|
||||
data class EventRate(
|
||||
val lastMinute: Long,
|
||||
val lastFiveMinutes: Long
|
||||
)
|
||||
@ -1,6 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.dto.translate
|
||||
|
||||
data class ApiResponse(
|
||||
val ok: Boolean,
|
||||
val message: String
|
||||
)
|
||||
@ -1,46 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.dto.translate
|
||||
|
||||
import no.iktdev.eventi.models.store.PersistedTask
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.LogAssociatedIds
|
||||
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
||||
import java.time.Instant
|
||||
import java.util.*
|
||||
|
||||
data class CoordinatorTaskTransferDto(
|
||||
val id: Long,
|
||||
val referenceId: UUID,
|
||||
val status: String,
|
||||
val taskId: UUID,
|
||||
val task: String,
|
||||
val data: String,
|
||||
val claimed: Boolean,
|
||||
val claimedBy: String?,
|
||||
val consumed: Boolean,
|
||||
val lastCheckIn: Instant?,
|
||||
val persistedAt: Instant,
|
||||
val logs: List<String> = emptyList(),
|
||||
val abandoned: Boolean,
|
||||
) {
|
||||
}
|
||||
|
||||
fun PersistedTask.toCoordinatorTransferDto(logs: List<LogAssociatedIds>): CoordinatorTaskTransferDto {
|
||||
val matchingLogs = logs
|
||||
.filter { log -> log.ids.contains(taskId) }
|
||||
.map { it.logFile }
|
||||
|
||||
return CoordinatorTaskTransferDto(
|
||||
id = id,
|
||||
referenceId = referenceId,
|
||||
status = status.name,
|
||||
taskId = taskId,
|
||||
task = task,
|
||||
data = data,
|
||||
claimed = claimed,
|
||||
claimedBy = claimedBy,
|
||||
consumed = consumed,
|
||||
lastCheckIn = lastCheckIn,
|
||||
persistedAt = persistedAt,
|
||||
logs = matchingLogs,
|
||||
abandoned = TaskLifecycleRules.isAbandoned(consumed, persistedAt, lastCheckIn)
|
||||
)
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.listeners.SummaryEventListener
|
||||
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class CollectEventsListener(eventStore: no.iktdev.eventi.stores.EventStore = EventStore) : SummaryEventListener(eventStore) {
|
||||
|
||||
private val log = KotlinLogging.logger {}
|
||||
override fun shouldSummarize(fullHistory: List<Event>): Boolean {
|
||||
val projection = CollectProjection(fullHistory)
|
||||
if (projection.startedWith == null) return false
|
||||
if (!projection.isWorkflowComplete()) return false
|
||||
return true
|
||||
}
|
||||
|
||||
override fun produceSummary(fullHistory: List<Event>): Event {
|
||||
// Must have all relevant tasks completed
|
||||
val eventIds = fullHistory.map { it.eventId }.toSet()
|
||||
|
||||
return CollectedEvent(eventIds).derivedOf(fullHistory.last())
|
||||
}
|
||||
|
||||
override fun summaryAlreadyExists(fullHistory: List<Event>): Boolean {
|
||||
return fullHistory.any { it is CollectedEvent }
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CompletedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.getName
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class CompletedListener: EventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
if (event !is StoreContentAndMetadataTaskResultEvent)
|
||||
return null
|
||||
val useEvent = event as StoreContentAndMetadataTaskResultEvent
|
||||
if (useEvent.status != TaskStatus.Completed) {
|
||||
log.info { "${useEvent.referenceId} - ${MigrateContentToStoreTaskResultEvent::class.getName()} is failed, thus no task will be created" }
|
||||
return null
|
||||
}
|
||||
return CompletedEvent().derivedOf(event)
|
||||
}
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskCreatedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
|
||||
@Component
|
||||
class MediaCreateConvertTaskListener: EventListener() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
fun allowOverwrite(): Boolean {
|
||||
return true
|
||||
}
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
|
||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
||||
if (startedEvent.data.operation.isNotEmpty()) {
|
||||
if (!startedEvent.data.operation.contains(OperationType.ConvertSubtitles))
|
||||
return null
|
||||
}
|
||||
val selectedEvent = event as? ProcesserExtractResultEvent ?: return null
|
||||
if (selectedEvent.status != TaskStatus.Completed)
|
||||
return null
|
||||
|
||||
|
||||
val result = selectedEvent.data ?: return null
|
||||
if (!Files.exists(Path.of(result.cachedOutputFile)))
|
||||
return null
|
||||
val useFile = File(result.cachedOutputFile)
|
||||
|
||||
val convertTask = ConvertTask(
|
||||
data = ConvertTask.Data(
|
||||
inputFile = result.cachedOutputFile,
|
||||
language = result.language,
|
||||
allowOverwrite = allowOverwrite(),
|
||||
outputDirectory = useFile.parentFile.absolutePath,
|
||||
outputFileName = useFile.nameWithoutExtension,
|
||||
)
|
||||
).derivedOf(event)
|
||||
TaskStore.persist(convertTask)
|
||||
|
||||
return ConvertTaskCreatedEvent(convertTask.taskId).derivedOf(event)
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadTaskCreatedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class MediaCreateCoverDownloadTaskListener: EventListener() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
val useEvent = event as? MetadataSearchResultEvent ?: return null
|
||||
if (useEvent.status != TaskStatus.Completed) {
|
||||
log.warn("MetadataResult on ${event.referenceId} did not complete successfully")
|
||||
return null
|
||||
}
|
||||
|
||||
val downloadData = useEvent.results.map {
|
||||
val data = it.metadata
|
||||
val outputFileName = "${data.title}-${data.source}"
|
||||
CoverDownloadTask.CoverDownloadData(
|
||||
url = it.metadata.cover,
|
||||
source = it.metadata.source,
|
||||
outputFileName = outputFileName
|
||||
)
|
||||
}
|
||||
|
||||
val downloadTasks = downloadData.map {
|
||||
CoverDownloadTask(it)
|
||||
.derivedOf(useEvent)
|
||||
}
|
||||
|
||||
downloadTasks.forEach { TaskStore.persist(it) }
|
||||
|
||||
return CoverDownloadTaskCreatedEvent(
|
||||
downloadTasks.map { it.taskId }
|
||||
).derivedOf(event)
|
||||
}
|
||||
}
|
||||
@ -1,73 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.*
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeData
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
|
||||
@Component
|
||||
class MediaCreateEncodeTaskListener(
|
||||
private val preference: Preference
|
||||
) : EventListener() {
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
val preference = preference.getProcesserPreference()
|
||||
|
||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
||||
if (startedEvent.data.operation.isNotEmpty()) {
|
||||
if (!startedEvent.data.operation.contains(OperationType.Encode))
|
||||
return null
|
||||
}
|
||||
val selectedEvent = event as? MediaTracksEncodeSelectedEvent ?: return null
|
||||
val streams = history.filterIsInstance<MediaStreamParsedEvent>().firstOrNull()?.data ?: return null
|
||||
|
||||
val videoPreference = preference.videoPreference?.codec ?: VideoCodec.Hevc()
|
||||
val audioPreference = preference.audioPreference?.codec ?: AudioCodec.Aac(channels = 2)
|
||||
|
||||
val audioTargets = mutableListOf<AudioTarget>(
|
||||
AudioTarget(
|
||||
index = selectedEvent.selectedAudioTrack,
|
||||
codec = audioPreference
|
||||
)
|
||||
)
|
||||
selectedEvent.selectedAudioExtendedTrack?.let {
|
||||
audioTargets.add(AudioTarget(
|
||||
index = it,
|
||||
codec = audioPreference
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
val plan = MediaPlan(
|
||||
videoTrack = VideoTarget(index = selectedEvent.selectedVideoTrack, codec = videoPreference),
|
||||
audioTracks = audioTargets
|
||||
)
|
||||
val args = plan.toFfmpegArgs(streams.videoStream, streams.audioStream)
|
||||
val filename = startedEvent.data.fileUri.let { File(it) }.nameWithoutExtension
|
||||
val extension = plan.toContainer()
|
||||
|
||||
val task = EncodeTask(
|
||||
data = EncodeData(
|
||||
arguments = args,
|
||||
outputFileName = "$filename.$extension",
|
||||
inputFile = startedEvent.data.fileUri
|
||||
)
|
||||
).derivedOf(event)
|
||||
|
||||
|
||||
TaskStore.persist(task)
|
||||
return ProcesserEncodeTaskCreatedEvent(
|
||||
taskCreated = task.taskId
|
||||
).derivedOf(event)
|
||||
}
|
||||
}
|
||||
@ -1,79 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.SubtitleCodec
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleData
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleTask
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class MediaCreateExtractTaskListener(): EventListener() {
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
|
||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
||||
if (startedEvent.data.operation.isNotEmpty()) {
|
||||
if (!startedEvent.data.operation.contains(OperationType.ExtractSubtitles))
|
||||
return null
|
||||
}
|
||||
|
||||
val selectedEvent = event as? MediaTracksExtractSelectedEvent ?: return null
|
||||
val streams = history.filterIsInstance<MediaStreamParsedEvent>().firstOrNull()?.data ?: return null
|
||||
|
||||
val selectedStreams: Map<Int, SubtitleStream> =
|
||||
selectedEvent.selectedSubtitleTracks.mapNotNull { streamIndex ->
|
||||
val stream = streams.subtitleStream.firstOrNull { it.index == streamIndex }
|
||||
stream?.let { streams.subtitleStream.indexOf(it) to it }
|
||||
}.toMap()
|
||||
|
||||
|
||||
val entries = selectedStreams.mapNotNull { (idx, stream )->
|
||||
toSubtitleArgumentData(idx, startedEvent.data.fileUri.let { File(it) }, stream)
|
||||
}
|
||||
|
||||
val createdTaskIds: MutableList<UUID> = mutableListOf()
|
||||
entries.forEach { entry ->
|
||||
ExtractSubtitleTask(data = entry).derivedOf(event).also {
|
||||
TaskStore.persist(it)
|
||||
createdTaskIds.add(it.taskId)
|
||||
}
|
||||
}
|
||||
|
||||
return ProcesserExtractTaskCreatedEvent(
|
||||
tasksCreated = createdTaskIds
|
||||
).derivedOf(event)
|
||||
}
|
||||
|
||||
fun toSubtitleArgumentData(index: Int, inputFile: File, stream: SubtitleStream): ExtractSubtitleData? {
|
||||
val codec = SubtitleCodec.getCodec(stream.codec_name) ?: return null
|
||||
val extension = codec.getExtension()
|
||||
|
||||
// ffmpeg-args for å mappe og copy akkurat dette subtitle-sporet
|
||||
val args = mutableListOf<String>()
|
||||
args += listOf("-map", "0:s:$index")
|
||||
args += codec.buildFfmpegArgs(stream)
|
||||
|
||||
val language = stream.tags.language?: return null
|
||||
|
||||
// outputfilnavn basert på index og extension
|
||||
val outputFileName = "${inputFile.nameWithoutExtension}-${language}.${extension}"
|
||||
|
||||
return ExtractSubtitleData(
|
||||
inputFile = inputFile.absolutePath,
|
||||
arguments = args,
|
||||
outputFileName = outputFileName,
|
||||
language = language
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.ListenerOrder
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MetadataSearchTask
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
import org.jetbrains.annotations.VisibleForTesting
|
||||
import org.springframework.stereotype.Component
|
||||
import java.util.*
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.ScheduledFuture
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@Component
|
||||
@ListenerOrder(5)
|
||||
class MediaCreateMetadataSearchTaskListener: EventListener() {
|
||||
|
||||
@VisibleForTesting
|
||||
internal val scheduledExpiries = ConcurrentHashMap<UUID, ScheduledFuture<*>>()
|
||||
private val scheduler = Executors.newScheduledThreadPool(1)
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
|
||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
||||
if (startedEvent.data.operation.isNotEmpty()) {
|
||||
if (!startedEvent.data.operation.contains(OperationType.MetadataSearch))
|
||||
return null
|
||||
}
|
||||
|
||||
// For replay
|
||||
if (event is MetadataSearchTaskCreatedEvent) {
|
||||
val hasResult = history.filter { it is MetadataSearchResultEvent }
|
||||
.any { it.metadata.derivedFromId?.contains(event.taskId) == true }
|
||||
|
||||
if (!hasResult) {
|
||||
scheduleTaskExpiry(event.taskId, event.eventId, event.referenceId)
|
||||
}
|
||||
} else if (event is MetadataSearchResultEvent) {
|
||||
val cancelKeys = event.metadata.derivedFromId ?: emptySet()
|
||||
scheduledExpiries.filter { it -> it.key in cancelKeys }.keys.forEach { key ->
|
||||
scheduledExpiries.remove(key)?.cancel(true)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
val useEvent = event as? MediaParsedInfoEvent ?: return null
|
||||
|
||||
val task = MetadataSearchTask(
|
||||
MetadataSearchTask.SearchData(
|
||||
searchTitles = useEvent.data.parsedSearchTitles,
|
||||
collection = useEvent.data.parsedCollection
|
||||
)
|
||||
).derivedOf(useEvent)
|
||||
TaskStore.persist(task)
|
||||
val finalResult = MetadataSearchTaskCreatedEvent(task.taskId).derivedOf(useEvent)
|
||||
scheduleTaskExpiry(task.taskId, finalResult.eventId, task.referenceId)
|
||||
return finalResult
|
||||
}
|
||||
|
||||
private fun scheduleTaskExpiry(taskId: UUID, eventId: UUID, referenceId: UUID) {
|
||||
if (scheduledExpiries.containsKey(taskId)) return
|
||||
|
||||
val future = scheduler.schedule({
|
||||
// Hvis tasken fortsatt ikke har result/failed → marker som failed
|
||||
TaskStore.claim(taskId, "Coordinator-MetadataSearchTaskListener-TimeoutScheduler")
|
||||
TaskStore.markConsumed(taskId, TaskStatus.Failed)
|
||||
val failureEvent = MetadataSearchResultEvent(
|
||||
status = TaskStatus.Failed,
|
||||
).apply { setFailed(listOf(taskId)) }
|
||||
//publishEvent(MetadataSearchFailedEvent(taskId, "Timeout").derivedOf(referenceId))
|
||||
scheduledExpiries.remove(taskId)
|
||||
}, 10, TimeUnit.MINUTES)
|
||||
|
||||
scheduledExpiries[taskId] = future
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleItem
|
||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleType
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class MediaDetermineSubtitleTrackTypeListener: EventListener() {
|
||||
fun ignoreSHD(): Boolean = true
|
||||
fun ignoreCC(): Boolean = true
|
||||
fun ignoreSongs(): Boolean = true
|
||||
fun ignoreCommentary(): Boolean = true
|
||||
|
||||
val supportedCodecs = setOf(
|
||||
"ass", "subrip", "webvtt", "vtt", "smi"
|
||||
)
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
val useEvent = event as? MediaStreamParsedEvent ?: return null
|
||||
|
||||
val collected = useEvent.data.subtitleStream
|
||||
.mapToType()
|
||||
.excludeTypes()
|
||||
.onlySupportedCodecs()
|
||||
|
||||
|
||||
return MediaTracksDetermineSubtitleTypeEvent(
|
||||
subtitleTrackItems = collected
|
||||
).derivedOf(event)
|
||||
}
|
||||
|
||||
|
||||
fun getCommentaryFilters(): Set<String> = setOf("commentary", "comentary", "kommentar", "kommentaar")
|
||||
fun getSongFilters(): Set<String> = setOf("song", "sign", "lyrics")
|
||||
fun getClosedCaptionFilters(): Set<String> = setOf("closed caption", "cc", "close caption", "closed-caption", "cc.")
|
||||
fun getSHDFilters(): Set<String> = setOf("shd", "hh", "hard of hearing", "hard-of-hearing")
|
||||
|
||||
private fun List<SubtitleItem>.excludeTypes(): List<SubtitleItem> {
|
||||
return this.filter {
|
||||
when (it.type) {
|
||||
SubtitleType.Song -> !ignoreSongs()
|
||||
SubtitleType.Commentary -> !ignoreCommentary()
|
||||
SubtitleType.ClosedCaption -> !ignoreCC()
|
||||
SubtitleType.SHD -> !ignoreSHD()
|
||||
SubtitleType.Dialogue -> true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun List<SubtitleStream>.mapToType(): List<SubtitleItem> {
|
||||
return this.map {
|
||||
val title = it.tags.title?.lowercase() ?: ""
|
||||
val type = when {
|
||||
getCommentaryFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.Commentary
|
||||
getSongFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.Song
|
||||
getClosedCaptionFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.ClosedCaption
|
||||
getSHDFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.SHD
|
||||
else -> SubtitleType.Dialogue
|
||||
}
|
||||
SubtitleItem(it, type)
|
||||
}
|
||||
}
|
||||
|
||||
private fun List<SubtitleItem>.onlySupportedCodecs(): List<SubtitleItem> {
|
||||
return this.filter { it.stream.codec_name in supportedCodecs }
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,82 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import com.google.gson.Gson
|
||||
import com.google.gson.JsonObject
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.ListenerOrder
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@ListenerOrder(4)
|
||||
@Component
|
||||
class MediaParseStreamsListener: EventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
if (event !is CoordinatorReadStreamsResultEvent) return null
|
||||
if (event.status != TaskStatus.Completed)
|
||||
return null
|
||||
if (event.data == null) {
|
||||
log.error { "No data to parse in CoordinatorReadStreamsResultEvent" }
|
||||
return null
|
||||
}
|
||||
|
||||
val streams = parseStreams(event.data)
|
||||
return MediaStreamParsedEvent(
|
||||
data = streams
|
||||
).derivedOf(event)
|
||||
}
|
||||
|
||||
|
||||
fun parseStreams(data: JsonObject?): ParsedMediaStreams {
|
||||
val ignoreCodecs = listOf("png", "mjpeg")
|
||||
val gson = Gson()
|
||||
return try {
|
||||
val jStreams = data!!.getAsJsonArray("streams")
|
||||
|
||||
val videoStreams = mutableListOf<VideoStream>()
|
||||
val audioStreams = mutableListOf<AudioStream>()
|
||||
val subtitleStreams = mutableListOf<SubtitleStream>()
|
||||
|
||||
for (streamJson in jStreams) {
|
||||
val streamObject = streamJson.asJsonObject
|
||||
if (!streamObject.has("codec_name")) continue
|
||||
val codecName = streamObject.get("codec_name").asString
|
||||
val codecType = streamObject.get("codec_type").asString
|
||||
|
||||
if (codecName in ignoreCodecs) continue
|
||||
|
||||
when (codecType) {
|
||||
"video" -> videoStreams.add(gson.fromJson(streamObject, VideoStream::class.java))
|
||||
"audio" -> audioStreams.add(gson.fromJson(streamObject, AudioStream::class.java))
|
||||
"subtitle" -> subtitleStreams.add(gson.fromJson(streamObject, SubtitleStream::class.java))
|
||||
}
|
||||
}
|
||||
|
||||
val parsedStreams = ParsedMediaStreams(
|
||||
videoStream = videoStreams,
|
||||
audioStream = audioStreams,
|
||||
subtitleStream = subtitleStreams
|
||||
)
|
||||
parsedStreams
|
||||
|
||||
} catch (e: Exception) {
|
||||
"Failed to parse data, its either not a valid json structure or expected and required fields are not present.".also {
|
||||
log.error { it }
|
||||
}
|
||||
throw e
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,271 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.ListenerOrder
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.model.MediaType
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
|
||||
@ListenerOrder(2)
|
||||
@Component
|
||||
class MediaParsedInfoListener : EventListener() {
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
val started = event as? StartProcessingEvent ?: return null
|
||||
val file = File(started.data.fileUri)
|
||||
|
||||
val filename = file.guessDesiredFileName()
|
||||
val collection = file.getDesiredCollection()
|
||||
val searchTitles = file.guessSearchableTitle()
|
||||
val mediaType = file.guessMovieOrSeries()
|
||||
|
||||
val episodeInfo = if (mediaType == MediaType.Serie) {
|
||||
val serieInfo = file.guessSerieInfo()
|
||||
MediaParsedInfoEvent.ParsedData.EpisodeInfo(
|
||||
episodeNumber = serieInfo.episodeNumber,
|
||||
seasonNumber = serieInfo.seasonNumber,
|
||||
episodeTitle = serieInfo.episodeTitle,
|
||||
)
|
||||
} else null
|
||||
|
||||
return MediaParsedInfoEvent(
|
||||
MediaParsedInfoEvent.ParsedData(
|
||||
parsedFileName = filename,
|
||||
parsedCollection = collection,
|
||||
parsedSearchTitles = searchTitles,
|
||||
mediaType = mediaType,
|
||||
episodeInfo = episodeInfo,
|
||||
)
|
||||
).derivedOf(event)
|
||||
}
|
||||
|
||||
fun String.getCleanedTitle(): String {
|
||||
return this
|
||||
.noBrackets()
|
||||
.noParens()
|
||||
.noResolutionAndAfter()
|
||||
.noSourceTags()
|
||||
.noYear()
|
||||
.noDots()
|
||||
.noTrailingOrLeading()
|
||||
.noUnderscores()
|
||||
.noExtraSpaces()
|
||||
.trim()
|
||||
}
|
||||
|
||||
|
||||
fun String.noBrackets() = Regex("\\[.*?]").replace(this, " ")
|
||||
fun String.noParens() = Regex("\\(.*?\\)").replace(this, " ")
|
||||
fun String.noTrailingOrLeading() = Regex("^[^a-zA-Z0-9!,]+|[^a-zA-Z0-9!~,]+\$").replace(this, " ")
|
||||
fun String.noResolutionAndAfter() = Regex("[0-9]+[pk].*", RegexOption.IGNORE_CASE).replace(this, "")
|
||||
fun String.noSourceTags() =
|
||||
Regex("(?i)(bluray|laserdisc|dvd|web|uhd|hd|htds|imax).*", RegexOption.IGNORE_CASE).replace(this, " ")
|
||||
|
||||
fun String.noUnderscores() = this.replace("_", " ")
|
||||
fun String.noYear() = Regex("\\b\\d{4}\\b").replace(this.takeIf { !it.matches(Regex("^\\d{4}")) } ?: this, "")
|
||||
fun String.noDots() = Regex("(?<!\\b(?:Dr|Mr|Ms|Mrs|Lt|Capt|Prof|St|Ave))\\.").replace(this, " ")
|
||||
fun String.noExtraSpaces() = Regex("\\s{2,}").replace(this, " ")
|
||||
fun String.fullTrim() = this.trim('.', ',', ' ', '_', '-')
|
||||
|
||||
|
||||
fun File.guessMovieOrSeries(): MediaType {
|
||||
val name = this.nameWithoutExtension.lowercase()
|
||||
|
||||
// Serie-mønstre: dekker alle vanlige shorthand og varianter
|
||||
val seriesPatterns = listOf(
|
||||
Regex("s\\d{1,2}e\\d{1,2}"), // S01E03, s1e5
|
||||
Regex("\\d{1,2}x\\d{1,2}"), // 1x03, 2x10
|
||||
Regex("season\\s*\\d+"), // Season 2
|
||||
Regex("episode\\s*\\d+"), // Episode 5
|
||||
Regex("ep\\s*\\d+"), // Ep05, Ep 5
|
||||
Regex("s\\d{1,2}\\s*[- ]\\s*e\\d{1,2}"), // S1 - E5, S01 - E05
|
||||
Regex("s\\d{1,2}\\s*ep\\s*\\d{1,2}"), // S1 Ep05
|
||||
Regex("series\\s*\\d+"), // Series 2 (britisk stil)
|
||||
Regex("s\\d{1,2}[. ]e\\d{1,2}") // S01.E02 eller S01 E02
|
||||
)
|
||||
|
||||
if (seriesPatterns.any { it.containsMatchIn(name) }) {
|
||||
return MediaType.Serie
|
||||
}
|
||||
|
||||
// Film-mønstre: årstall (1900–2099) etter tittel
|
||||
val moviePattern = Regex("\\b(19|20)\\d{2}\\b")
|
||||
if (moviePattern.containsMatchIn(name)) {
|
||||
return MediaType.Movie
|
||||
}
|
||||
|
||||
// Fallback: hvis ingen mønstre passer, anta film
|
||||
return MediaType.Movie
|
||||
}
|
||||
|
||||
|
||||
fun File.guessDesiredFileName(): String {
|
||||
val type = this.guessMovieOrSeries()
|
||||
return when (type) {
|
||||
MediaType.Movie -> this.guessDesiredMovieTitle()
|
||||
MediaType.Serie -> this.guessDesiredSerieTitle()
|
||||
MediaType.Subtitle -> {
|
||||
this.nameWithoutExtension
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun File.getDesiredCollection(): String {
|
||||
val collection = when (this.guessMovieOrSeries()) {
|
||||
MediaType.Movie -> this.guessDesiredMovieTitle()
|
||||
MediaType.Serie -> this.guessDesiredSerieTitle()
|
||||
MediaType.Subtitle -> {
|
||||
this.parentFile.parentFile.nameWithoutExtension
|
||||
}
|
||||
}
|
||||
return collection.noParens().noYear().split(" - ").first().trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A fully cleaned title suitable to use for collection
|
||||
*/
|
||||
fun File.guessDesiredMovieTitle(): String {
|
||||
val cleaned = this.nameWithoutExtension.getCleanedTitle()
|
||||
val yearRegex = Regex("\\b(19|20)\\d{2}\\b")
|
||||
val yearMatch = yearRegex.find(cleaned)
|
||||
|
||||
return if (yearMatch != null) {
|
||||
val title = cleaned.replace(yearRegex, "").trim()
|
||||
"${title} (${yearMatch.value})"
|
||||
} else {
|
||||
cleaned
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return A fully cleaned title including season and episode with possible episode title
|
||||
*/
|
||||
fun File.guessDesiredSerieTitle(): String {
|
||||
val parsedSerieInfo = this.guessSerieInfo()
|
||||
|
||||
val tag = buildString {
|
||||
append("S${(parsedSerieInfo.seasonNumber ?: 1).toString().padStart(2, '0')}")
|
||||
append("E${(parsedSerieInfo.episodeNumber ?: 1).toString().padStart(2, '0')}")
|
||||
if (parsedSerieInfo.revision != null) append(" (v$parsedSerieInfo.revision)")
|
||||
}
|
||||
|
||||
return buildString {
|
||||
append(parsedSerieInfo.serieTitle)
|
||||
append(" - ")
|
||||
append(tag)
|
||||
if (parsedSerieInfo.episodeTitle.isNotEmpty()) {
|
||||
append(" - ")
|
||||
append(parsedSerieInfo.episodeTitle)
|
||||
}
|
||||
}.trim()
|
||||
}
|
||||
|
||||
fun File.guessSerieInfo(): ParsedSerieInfo {
|
||||
val raw = this.nameWithoutExtension
|
||||
|
||||
val seasonRegex = Regex("""(?i)(?:S|Season|Series)\s*(\d{1,2})""")
|
||||
val episodeRegex = Regex("""(?i)(?:E|Episode|Ep)\s*(\d{1,3})""")
|
||||
val revisionRegex = Regex("""(?i)\bv(\d+)\b""")
|
||||
val seasonEpisodeRegex = Regex("""(?i)(\d{1,2})x(\d{1,2})(?:[vV](\d+))?""")
|
||||
|
||||
var season: Int? = null
|
||||
var episode: Int? = null
|
||||
var revision: Int? = null
|
||||
var baseTitle = raw.getCleanedTitle()
|
||||
var episodeTitle = ""
|
||||
|
||||
val seMatch = seasonEpisodeRegex.find(raw)
|
||||
if (seMatch != null) {
|
||||
season = seMatch.groupValues[1].toIntOrNull()
|
||||
episode = seMatch.groupValues[2].toIntOrNull()
|
||||
revision = seMatch.groupValues.getOrNull(3)?.toIntOrNull()
|
||||
baseTitle = raw.substring(0, seMatch.range.first).getCleanedTitle()
|
||||
episodeTitle = raw.substring(seMatch.range.last + 1).getCleanedTitle()
|
||||
} else {
|
||||
val seasonMatch = seasonRegex.find(raw)
|
||||
val episodeMatch = episodeRegex.find(raw)
|
||||
val revisionMatch = revisionRegex.find(raw)
|
||||
|
||||
season = seasonMatch?.groupValues?.get(1)?.toIntOrNull()
|
||||
episode = episodeMatch?.groupValues?.get(1)?.toIntOrNull()
|
||||
revision = revisionMatch?.groupValues?.get(1)?.toIntOrNull()
|
||||
|
||||
baseTitle = if (seasonMatch != null) {
|
||||
raw.substring(0, seasonMatch.range.first).getCleanedTitle()
|
||||
} else raw.getCleanedTitle()
|
||||
|
||||
episodeTitle = if (episodeMatch != null) {
|
||||
raw.substring(episodeMatch.range.last + 1).getCleanedTitle()
|
||||
} else ""
|
||||
}
|
||||
|
||||
// Fallback: hvis baseTitle er tom eller bare inneholder S/E, bruk parent‑mappe
|
||||
if (baseTitle.isBlank() || baseTitle.matches(Regex("""(?i)^s?\d+e?\d+$"""))) {
|
||||
baseTitle = this.parentFile?.name?.getCleanedTitle() ?: "Dumb ways to die"
|
||||
}
|
||||
|
||||
return ParsedSerieInfo(
|
||||
serieTitle = baseTitle,
|
||||
episodeNumber = episode ?: 1,
|
||||
seasonNumber = season ?: 1,
|
||||
revision = revision,
|
||||
episodeTitle
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
fun File.guessSearchableTitle(): List<String> {
|
||||
val cleaned = this.guessDesiredFileName()
|
||||
.noResolutionAndAfter()
|
||||
.noSourceTags()
|
||||
.noDots()
|
||||
.noExtraSpaces()
|
||||
.fullTrim()
|
||||
|
||||
val titles = mutableListOf<String>()
|
||||
|
||||
val yearRegex = Regex("""\b(19|20)\d{2}\b""")
|
||||
val hasYear = yearRegex.containsMatchIn(cleaned)
|
||||
|
||||
// 1. Hvis årstall finnes, legg hele cleaned først
|
||||
if (hasYear) {
|
||||
titles.add(cleaned)
|
||||
}
|
||||
|
||||
// 2. Første del før bindestrek
|
||||
val firstPart = cleaned.split(" - ").firstOrNull()?.trim() ?: cleaned
|
||||
titles.add(firstPart)
|
||||
|
||||
// 3. Hele cleaned (hvis ikke allerede lagt inn først)
|
||||
if (!hasYear) {
|
||||
titles.add(cleaned)
|
||||
}
|
||||
|
||||
// 4. Variant uten årstall
|
||||
val noYear = yearRegex.replace(cleaned, "")
|
||||
.noParens().trim()
|
||||
if (noYear.isNotEmpty() && noYear != cleaned) {
|
||||
titles.add(noYear)
|
||||
}
|
||||
|
||||
return titles.distinct()
|
||||
}
|
||||
|
||||
data class ParsedSerieInfo(
|
||||
val serieTitle: String,
|
||||
val episodeNumber: Int,
|
||||
val seasonNumber: Int,
|
||||
val revision: Int? = null,
|
||||
val episodeTitle: String,
|
||||
)
|
||||
|
||||
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.ListenerOrder
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsTaskCreatedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@ListenerOrder(3)
|
||||
@Component
|
||||
class MediaReadStreamsTaskCreatedListener: EventListener() {
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
if (event !is MediaParsedInfoEvent) return null
|
||||
val startEvent = history.lastOrNull { it is StartProcessingEvent } as? StartProcessingEvent
|
||||
?: return null
|
||||
|
||||
|
||||
val readTask = MediaReadTask(
|
||||
fileUri = startEvent.data.fileUri
|
||||
).derivedOf(event)
|
||||
|
||||
TaskStore.persist(readTask)
|
||||
return CoordinatorReadStreamsTaskCreatedEvent(readTask.taskId).derivedOf(event) // Create task instead of event
|
||||
}
|
||||
}
|
||||
@ -1,158 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksEncodeSelectedEvent
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class MediaSelectEncodeTracksListener(
|
||||
private val preference: Preference
|
||||
) : EventListener() {
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
val useEvent = event as? MediaStreamParsedEvent ?: return null
|
||||
|
||||
val videoTrackIndex = getVideoTrackToUse(useEvent.data.videoStream)
|
||||
val audioDefaultTrack = getAudioDefaultTrackToUse(useEvent.data.audioStream)
|
||||
val audioExtendedTrack = getAudioExtendedTrackToUse(
|
||||
useEvent.data.audioStream,
|
||||
selectedDefaultTrack = audioDefaultTrack
|
||||
)
|
||||
|
||||
return MediaTracksEncodeSelectedEvent(
|
||||
selectedVideoTrack = videoTrackIndex,
|
||||
selectedAudioTrack = audioDefaultTrack,
|
||||
selectedAudioExtendedTrack = audioExtendedTrack
|
||||
).derivedOf(event)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// AUDIO SELECTION
|
||||
// ------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
private fun getAudioDefaultTrackToUse(audioStreams: List<AudioStream>): Int {
|
||||
val pref = preference.getLanguagePreference()
|
||||
|
||||
val selected = selectBestAudioStream(
|
||||
streams = audioStreams,
|
||||
preferredLanguages = pref.preferredAudio,
|
||||
preferOriginal = pref.preferOriginal,
|
||||
avoidDub = pref.avoidDub,
|
||||
mode = AudioSelectMode.DEFAULT
|
||||
) ?: audioStreams.firstOrNull()
|
||||
|
||||
return audioStreams.indexOf(selected)
|
||||
}
|
||||
|
||||
private fun getAudioExtendedTrackToUse(
|
||||
audioStreams: List<AudioStream>,
|
||||
selectedDefaultTrack: Int
|
||||
): Int? {
|
||||
val pref = preference.getLanguagePreference()
|
||||
|
||||
val candidates = audioStreams
|
||||
.filter { it.index != selectedDefaultTrack }
|
||||
.filter { (it.duration_ts ?: 0) > 0 }
|
||||
.filter { it.channels > 2 }
|
||||
|
||||
val selected = selectBestAudioStream(
|
||||
streams = candidates,
|
||||
preferredLanguages = pref.preferredAudio,
|
||||
preferOriginal = pref.preferOriginal,
|
||||
avoidDub = pref.avoidDub,
|
||||
mode = AudioSelectMode.EXTENDED
|
||||
) ?: return null
|
||||
|
||||
return audioStreams.indexOf(selected)
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// CORE AUDIO SELECTION LOGIC
|
||||
// ------------------------------------------------------------
|
||||
|
||||
private enum class AudioSelectMode { DEFAULT, EXTENDED }
|
||||
|
||||
private fun selectBestAudioStream(
|
||||
streams: List<AudioStream>,
|
||||
preferredLanguages: List<String>,
|
||||
preferOriginal: Boolean,
|
||||
avoidDub: Boolean,
|
||||
mode: AudioSelectMode
|
||||
): AudioStream? {
|
||||
if (streams.isEmpty()) return null
|
||||
|
||||
// 1. Originalspråk
|
||||
if (preferOriginal) {
|
||||
val originals = streams.filter { it.disposition.original == 1 }
|
||||
if (originals.isNotEmpty()) {
|
||||
return when (mode) {
|
||||
AudioSelectMode.DEFAULT -> originals.minByOrNull { it.channels }
|
||||
AudioSelectMode.EXTENDED -> originals.maxByOrNull { it.channels }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Filtrer bort dub
|
||||
val filtered = if (avoidDub) {
|
||||
streams.filter { it.disposition.dub != 1 }
|
||||
} else streams
|
||||
|
||||
// 3. Foretrukne språk
|
||||
for (lang in preferredLanguages) {
|
||||
val match = filtered.filter {
|
||||
it.tags.language?.equals(lang, ignoreCase = true) == true
|
||||
}
|
||||
if (match.isNotEmpty()) {
|
||||
return when (mode) {
|
||||
AudioSelectMode.DEFAULT -> match.minByOrNull { it.channels }
|
||||
AudioSelectMode.EXTENDED -> match.maxByOrNull { it.channels }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Default-flagget
|
||||
val default = filtered.firstOrNull { it.disposition.default == 1 }
|
||||
if (default != null) return default
|
||||
|
||||
// 5. Fallback
|
||||
return filtered.firstOrNull()
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// QUALITY SCORE (no bitrate)
|
||||
// ------------------------------------------------------------
|
||||
|
||||
private fun qualityScore(s: AudioStream): Int {
|
||||
val channelsScore = s.channels * 10
|
||||
val bitsScore = s.bits_per_sample
|
||||
val sampleRateScore = s.sample_rate.toIntOrNull()?.div(1000) ?: 0
|
||||
|
||||
return channelsScore + bitsScore + sampleRateScore
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// VIDEO SELECTION
|
||||
// ------------------------------------------------------------
|
||||
|
||||
private fun getVideoTrackToUse(streams: List<VideoStream>): Int {
|
||||
val selectStream = streams
|
||||
.filter { (it.duration_ts ?: 0) > 0 }
|
||||
.maxByOrNull { it.duration_ts ?: 0 }
|
||||
?: streams.minByOrNull { it.index }
|
||||
?: throw Exception("No video streams found")
|
||||
|
||||
return streams.indexOf(selectStream)
|
||||
}
|
||||
}
|
||||
@ -1,130 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
||||
import no.iktdev.mediaprocessing.coordinator.SubtitleSelectionMode
|
||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksExtractSelectedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleItem
|
||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleType
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class MediaSelectExtractTracksListener(
|
||||
private val preference: Preference
|
||||
) : EventListener() {
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
val useEvent = event as? MediaTracksDetermineSubtitleTypeEvent ?: return null
|
||||
|
||||
val pref = preference.getLanguagePreference()
|
||||
|
||||
// 1. Filter by subtitle type (dialogue, forced, etc.)
|
||||
val filteredByType = filterBySelectionMode(
|
||||
items = useEvent.subtitleTrackItems,
|
||||
mode = pref.subtitleSelectionMode
|
||||
)
|
||||
|
||||
// 2. Extract streams
|
||||
val streams = filteredByType.map { it.stream }
|
||||
|
||||
// 3. Select subtitles based on language + format priority
|
||||
val selected = selectSubtitleStreams(
|
||||
streams = streams,
|
||||
preferredLanguages = pref.preferredSubtitles,
|
||||
preferOriginal = pref.preferOriginal,
|
||||
formatPriority = pref.subtitleFormatPriority
|
||||
)
|
||||
|
||||
return MediaTracksExtractSelectedEvent(
|
||||
selectedSubtitleTracks = selected.map { it.index }
|
||||
).derivedOf(event)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// TYPE FILTERING
|
||||
// ------------------------------------------------------------
|
||||
|
||||
private fun filterBySelectionMode(
|
||||
items: List<SubtitleItem>,
|
||||
mode: SubtitleSelectionMode
|
||||
): List<SubtitleItem> {
|
||||
return when (mode) {
|
||||
SubtitleSelectionMode.DialogueOnly ->
|
||||
items.filter { it.type == SubtitleType.Dialogue }
|
||||
|
||||
SubtitleSelectionMode.DialogueAndForced ->
|
||||
items.filter {
|
||||
it.type == SubtitleType.Dialogue || it.stream.disposition?.forced == 1
|
||||
}
|
||||
|
||||
SubtitleSelectionMode.All ->
|
||||
items
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// SUBTITLE SELECTION LOGIC
|
||||
// ------------------------------------------------------------
|
||||
|
||||
private fun selectSubtitleStreams(
|
||||
streams: List<SubtitleStream>,
|
||||
preferredLanguages: List<String>,
|
||||
preferOriginal: Boolean,
|
||||
formatPriority: List<String>
|
||||
): List<SubtitleStream> {
|
||||
if (streams.isEmpty()) return emptyList()
|
||||
|
||||
// 1. Originalspråk
|
||||
if (preferOriginal) {
|
||||
val originals = streams.filter { it.disposition?.original == 1 }
|
||||
if (originals.isNotEmpty()) {
|
||||
return originals.uniquePerLanguageBestFormat(formatPriority)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Preferred languages
|
||||
for (lang in preferredLanguages) {
|
||||
val match = streams.filter {
|
||||
it.tags.language?.equals(lang, ignoreCase = true) == true ||
|
||||
it.subtitle_tags.language?.equals(lang, ignoreCase = true) == true
|
||||
}
|
||||
if (match.isNotEmpty()) {
|
||||
return match.uniquePerLanguageBestFormat(formatPriority)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Default subtitles
|
||||
val defaults = streams.filter { it.disposition?.default == 1 }
|
||||
if (defaults.isNotEmpty()) {
|
||||
return defaults.uniquePerLanguageBestFormat(formatPriority)
|
||||
}
|
||||
|
||||
// 4. Fallback: all subtitles
|
||||
return streams.uniquePerLanguageBestFormat(formatPriority)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// UNIQUE PER LANGUAGE + FORMAT PRIORITY
|
||||
// ------------------------------------------------------------
|
||||
|
||||
private fun List<SubtitleStream>.uniquePerLanguageBestFormat(
|
||||
formatPriority: List<String>
|
||||
): List<SubtitleStream> {
|
||||
return this
|
||||
.groupBy { it.tags.language ?: it.subtitle_tags.language ?: "unknown" }
|
||||
.mapNotNull { (_, langGroup) ->
|
||||
langGroup
|
||||
.sortedBy { s ->
|
||||
val idx = formatPriority.indexOf(s.codec_name.lowercase())
|
||||
if (idx == -1) Int.MAX_VALUE else idx
|
||||
}
|
||||
.firstOrNull()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,79 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ManualAllowCompletionEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskCreatedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask
|
||||
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
|
||||
import no.iktdev.mediaprocessing.shared.common.projection.MigrateContentProject
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class MigrateCreateStoreTaskListener(
|
||||
private val coordinatorEnv: CoordinatorEnv,
|
||||
): EventListener() {
|
||||
private val log = KotlinLogging.logger {}
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
|
||||
if (event !is CollectedEvent && event !is ManualAllowCompletionEvent)
|
||||
return null
|
||||
|
||||
val useEvent = if (event is ManualAllowCompletionEvent) {
|
||||
history.lastOrNull { it is CollectedEvent } as? CollectedEvent
|
||||
?: return null
|
||||
} else {
|
||||
event as CollectedEvent
|
||||
}
|
||||
|
||||
val useHistory = history.filter { useEvent.eventIds.contains(it.eventId) }
|
||||
|
||||
val collectProjection = CollectProjection(useHistory)
|
||||
log.info { collectProjection.prettyPrint() }
|
||||
|
||||
val statusAcceptable = collectProjection.getTaskStatus().none { it == CollectProjection.TaskStatus.Failed }
|
||||
if (!statusAcceptable) {
|
||||
log.warn { "One or more tasks have failed in ${event.referenceId}" }
|
||||
}
|
||||
|
||||
val migrateContentProjection = MigrateContentProject(useHistory, coordinatorEnv.outgoingContent)
|
||||
|
||||
val collection = migrateContentProjection.useStore?.name ?:
|
||||
throw RuntimeException("No content store configured for migration in ${event.referenceId}")
|
||||
|
||||
val videContent = migrateContentProjection.getVideoStoreFile()?.let { MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath) }
|
||||
val subtitleContent = migrateContentProjection.getSubtitleStoreFiles()?.map {
|
||||
MigrateToContentStoreTask.Data.SingleSubtitle(it.language, it.cts.cachedFile.absolutePath, it.cts.storeFile.absolutePath, )
|
||||
}
|
||||
val coverContent = migrateContentProjection.getCoverStoreFiles()?.map {
|
||||
MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath)
|
||||
}
|
||||
val storeTask = MigrateToContentStoreTask(
|
||||
MigrateToContentStoreTask.Data(
|
||||
collection = collection,
|
||||
videoContent = videContent,
|
||||
subtitleContent = subtitleContent,
|
||||
coverContent = coverContent
|
||||
)
|
||||
).derivedOf(event)
|
||||
|
||||
|
||||
if (!CollectProjection(history).isStorePermitted()) {
|
||||
log.info { "\uD83D\uDED1 Not storing content and metadata automatically for collection: $collection @ ${useEvent.referenceId}" }
|
||||
log.info { "A manual allow completion event is required to proceed." }
|
||||
return null
|
||||
}
|
||||
|
||||
TaskStore.persist(storeTask)
|
||||
|
||||
return MigrateContentToStoreTaskCreatedEvent(storeTask.taskId).derivedOf(useEvent)
|
||||
}
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import no.iktdev.eventi.ListenerOrder
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@ListenerOrder(1)
|
||||
@Component
|
||||
class StartedListener : EventListener() {
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
val useEvent = event as? FileReadyEvent ?: return null
|
||||
|
||||
return StartProcessingEvent(
|
||||
data = StartData(
|
||||
flow = StartFlow.Auto,
|
||||
fileUri = useEvent.data.fileUri,
|
||||
operation = setOf(
|
||||
OperationType.ExtractSubtitles,
|
||||
OperationType.ConvertSubtitles,
|
||||
OperationType.Encode,
|
||||
OperationType.MetadataSearch
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,78 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.events.EventListener
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ManualAllowCompletionEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskCreatedEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask
|
||||
import no.iktdev.mediaprocessing.shared.common.getName
|
||||
import no.iktdev.mediaprocessing.shared.common.model.ContentExport
|
||||
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
|
||||
import no.iktdev.mediaprocessing.shared.common.projection.StoreProjection
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
|
||||
import org.springframework.stereotype.Component
|
||||
|
||||
@Component
|
||||
class StoreContentAndMetadataListener: EventListener() {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun onEvent(
|
||||
event: Event,
|
||||
history: List<Event>
|
||||
): Event? {
|
||||
if (event !is MigrateContentToStoreTaskResultEvent && event !is ManualAllowCompletionEvent)
|
||||
return null
|
||||
|
||||
val useEvent = if (event is ManualAllowCompletionEvent) {
|
||||
history.lastOrNull { it is MigrateContentToStoreTaskResultEvent } as? MigrateContentToStoreTaskResultEvent
|
||||
?: return null
|
||||
} else {
|
||||
event as MigrateContentToStoreTaskResultEvent
|
||||
}
|
||||
|
||||
val collectionEvent = history.lastOrNull { it is CollectedEvent } as? CollectedEvent
|
||||
?: return null
|
||||
|
||||
val useHistory = (history.filter { collectionEvent.eventIds.contains(it.eventId) }) + listOf(useEvent)
|
||||
val projection = StoreProjection(useHistory)
|
||||
|
||||
val collection = projection.getCollection()
|
||||
if (collection.isNullOrBlank()) {
|
||||
log.error { "Collection is null @ ${useEvent.referenceId}" }
|
||||
return null
|
||||
}
|
||||
val metadata = projection.projectMetadata()
|
||||
if (metadata == null) {
|
||||
log.error { "Metadata is null @ ${useEvent.referenceId}"}
|
||||
return null
|
||||
}
|
||||
if (useEvent.status != TaskStatus.Completed) {
|
||||
log.info { "${useEvent.referenceId} - ${MigrateContentToStoreTaskResultEvent::class.getName()} is failed, thus no task will be created" }
|
||||
return null
|
||||
}
|
||||
|
||||
if (!CollectProjection(history).isStorePermitted()) {
|
||||
log.info { "\uD83D\uDED1 Not storing content and metadata automatically for collection: $collection @ ${useEvent.referenceId}" }
|
||||
log.info { "A manual allow completion event is required to proceed." }
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
val exportInfo = ContentExport(
|
||||
collection = collection,
|
||||
media = projection.projectMediaFiles(),
|
||||
episodeInfo = projection.projectEpisodeInfo(),
|
||||
metadata = metadata
|
||||
)
|
||||
|
||||
val task = StoreContentAndMetadataTask(exportInfo).derivedOf(useEvent)
|
||||
TaskStore.persist(task)
|
||||
|
||||
return StoreContentAndMetadataTaskCreatedEvent(task.taskId).derivedOf(useEvent)
|
||||
}
|
||||
}
|
||||
@ -1,90 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.Task
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskListener
|
||||
import no.iktdev.eventi.tasks.TaskType
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
||||
import no.iktdev.mediaprocessing.shared.common.DownloadClient
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask
|
||||
import no.iktdev.mediaprocessing.shared.common.notExist
|
||||
import org.springframework.stereotype.Component
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class DownloadCoverTaskListener(
|
||||
private val coordinatorEnv: CoordinatorEnv,
|
||||
): TaskListener(TaskType.MIXED) {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getWorkerId(): String {
|
||||
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
||||
}
|
||||
|
||||
override fun supports(task: Task): Boolean {
|
||||
return task is CoverDownloadTask
|
||||
}
|
||||
|
||||
override suspend fun onTask(task: Task): Event? {
|
||||
val pickedTask = task as? CoverDownloadTask ?: return null
|
||||
log.info { "Downloading cover from ${pickedTask.data.url}" }
|
||||
val taskData = pickedTask.data
|
||||
|
||||
val downloadClient = getDownloadClient()
|
||||
val downloadResult = try {
|
||||
downloadClient.download(taskData.url, taskData.outputFileName)
|
||||
} catch (e: Exception) {
|
||||
return CoverDownloadResultEvent(status = TaskStatus.Failed)
|
||||
}
|
||||
val downloadedFile = downloadResult.result
|
||||
|
||||
if (downloadResult.success && downloadedFile != null) {
|
||||
log.info { "Downloaded cover to ${downloadedFile.absolutePath}" }
|
||||
return CoverDownloadResultEvent(
|
||||
status = TaskStatus.Completed,
|
||||
data = CoverDownloadResultEvent.CoverDownloadedData(
|
||||
source = taskData.source,
|
||||
outputFile = downloadedFile.absolutePath
|
||||
)
|
||||
).producedFrom(pickedTask)
|
||||
} else {
|
||||
log.error { "Failed to download cover from ${taskData.url}" }
|
||||
return CoverDownloadResultEvent(
|
||||
status = TaskStatus.Failed,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun createIncompleteStateTaskEvent(
|
||||
task: Task,
|
||||
status: TaskStatus,
|
||||
exception: Exception?
|
||||
): Event {
|
||||
val message = when (status) {
|
||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
||||
TaskStatus.Cancelled -> "Canceled"
|
||||
else -> ""
|
||||
}
|
||||
return CoverDownloadResultEvent(null, status, error = message)
|
||||
}
|
||||
|
||||
open fun getDownloadClient(): DownloadClient {
|
||||
return DefaultDownloadClient(coordinatorEnv)
|
||||
}
|
||||
|
||||
class DefaultDownloadClient(private val coordinatorEnv: CoordinatorEnv) : DownloadClient(
|
||||
outDir = coordinatorEnv.cachedContent,
|
||||
connectionFactory = DefaultConnectionFactory(),) {
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
if (outDir.notExist()) {
|
||||
outDir.mkdirs()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
||||
|
||||
import no.iktdev.eventi.tasks.TaskListener
|
||||
import no.iktdev.eventi.tasks.TaskType
|
||||
import no.iktdev.mediaprocessing.ffmpeg.FFprobe
|
||||
|
||||
abstract class FfprobeTaskListener(taskType: TaskType): TaskListener(taskType) {
|
||||
abstract fun getFfprobe(): FFprobe
|
||||
}
|
||||
@ -1,71 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.Task
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskType
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
||||
import no.iktdev.mediaprocessing.ffmpeg.FFprobe
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask
|
||||
import org.springframework.stereotype.Component
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class MediaStreamReadTaskListener(
|
||||
private val coordinatorEnv: CoordinatorEnv
|
||||
): FfprobeTaskListener(TaskType.CPU_INTENSIVE) {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
override fun getWorkerId(): String {
|
||||
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
||||
}
|
||||
|
||||
override fun supports(task: Task): Boolean {
|
||||
return task is MediaReadTask
|
||||
}
|
||||
|
||||
override suspend fun onTask(task: Task): Event? {
|
||||
val pickedTask = task as? MediaReadTask ?: return null
|
||||
try {
|
||||
val probeResult = getFfprobe()
|
||||
.readJsonStreams(pickedTask.fileUri)
|
||||
|
||||
val result =
|
||||
probeResult.data ?: throw RuntimeException("No data returned from ffprobe for ${pickedTask.fileUri}")
|
||||
|
||||
return CoordinatorReadStreamsResultEvent(
|
||||
status = TaskStatus.Completed,
|
||||
data = result
|
||||
).producedFrom(task)
|
||||
|
||||
} catch (e: Exception) {
|
||||
log.error(e) { "Error reading media streams for ${pickedTask.fileUri}" }
|
||||
return CoordinatorReadStreamsResultEvent(
|
||||
status = TaskStatus.Failed,
|
||||
data = null
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun createIncompleteStateTaskEvent(
|
||||
task: Task,
|
||||
status: TaskStatus,
|
||||
exception: Exception?
|
||||
): Event {
|
||||
val message = when (status) {
|
||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
||||
TaskStatus.Cancelled -> "Canceled"
|
||||
else -> ""
|
||||
}
|
||||
return CoordinatorReadStreamsResultEvent(null, status, error = message)
|
||||
}
|
||||
|
||||
override fun getFfprobe(): FFprobe {
|
||||
return JsonFfinfo(coordinatorEnv.ffprobe)
|
||||
}
|
||||
|
||||
class JsonFfinfo(executable: String): FFprobe(executable) {
|
||||
}
|
||||
}
|
||||
@ -1,197 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
||||
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.Task
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskListener
|
||||
import no.iktdev.eventi.tasks.TaskType
|
||||
import no.iktdev.mediaprocessing.coordinator.util.FileServiceException
|
||||
import no.iktdev.mediaprocessing.coordinator.util.FileSystemService
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask
|
||||
import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus
|
||||
import no.iktdev.mediaprocessing.shared.common.silentTry
|
||||
import org.springframework.stereotype.Component
|
||||
import java.io.File
|
||||
import java.nio.file.Files
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class MigrateContentToStoreTaskListener : TaskListener(TaskType.IO_INTENSIVE) {
|
||||
|
||||
override fun getWorkerId(): String =
|
||||
"${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
||||
|
||||
override fun supports(task: Task): Boolean =
|
||||
task is MigrateToContentStoreTask
|
||||
|
||||
override suspend fun onTask(task: Task): Event? {
|
||||
val picked = task as? MigrateToContentStoreTask ?: return null
|
||||
val fs = getFileSystemService()
|
||||
|
||||
val video = migrateVideo(fs, picked.data.videoContent)
|
||||
val subs = migrateSubtitle(fs, picked.data.subtitleContent ?: emptyList())
|
||||
val covers = migrateCover(fs, picked.data.coverContent ?: emptyList())
|
||||
|
||||
deleteCache(fs, picked)
|
||||
|
||||
return MigrateContentToStoreTaskResultEvent(
|
||||
status = TaskStatus.Completed,
|
||||
migrateData = MigrateContentToStoreTaskResultEvent.MigrateData(
|
||||
collection = picked.data.collection,
|
||||
videoMigrate = video,
|
||||
subtitleMigrate = subs,
|
||||
coverMigrate = covers
|
||||
)
|
||||
).producedFrom(task)
|
||||
}
|
||||
|
||||
override fun createIncompleteStateTaskEvent(
|
||||
task: Task,
|
||||
status: TaskStatus,
|
||||
exception: Exception?
|
||||
): Event {
|
||||
val message = when (status) {
|
||||
TaskStatus.Failed -> exception?.message ?: "Unknown error"
|
||||
TaskStatus.Cancelled -> "Canceled"
|
||||
else -> ""
|
||||
}
|
||||
|
||||
return MigrateContentToStoreTaskResultEvent(
|
||||
migrateData = null,
|
||||
status = status,
|
||||
error = message
|
||||
).producedFrom(task)
|
||||
}
|
||||
|
||||
private fun deleteCache(fs: FileSystemService, task: MigrateToContentStoreTask) {
|
||||
task.data.videoContent?.cachedUri?.let { silentTry { fs.delete(File(it)) } }
|
||||
task.data.subtitleContent?.forEach { silentTry { fs.delete(File(it.cachedUri)) } }
|
||||
task.data.coverContent?.forEach { silentTry { fs.delete(File(it.cachedUri)) } }
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// MIGRATION HELPERS
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private fun migrateFile(fs: FileSystemService, source: File, destination: File) {
|
||||
if (destination.exists()) {
|
||||
try {
|
||||
fs.verifyIdentical(source, destination)
|
||||
return
|
||||
} catch (e: FileServiceException.VerificationFailed) {
|
||||
throw FileServiceException.DestinationExistsButDifferent(source, destination)
|
||||
}
|
||||
}
|
||||
|
||||
fs.copy(source, destination)
|
||||
fs.verifyIdentical(source, destination)
|
||||
}
|
||||
|
||||
|
||||
internal fun migrateVideo(
|
||||
fs: FileSystemService,
|
||||
content: MigrateToContentStoreTask.Data.SingleContent?
|
||||
): MigrateContentToStoreTaskResultEvent.FileMigration {
|
||||
|
||||
if (content == null) {
|
||||
return MigrateContentToStoreTaskResultEvent.FileMigration(null, MigrateStatus.NotPresent)
|
||||
}
|
||||
|
||||
val source = File(content.cachedUri)
|
||||
val dest = File(content.storeUri)
|
||||
|
||||
migrateFile(fs, source, dest)
|
||||
|
||||
return MigrateContentToStoreTaskResultEvent.FileMigration(
|
||||
storedUri = dest.absolutePath,
|
||||
status = MigrateStatus.Completed
|
||||
)
|
||||
}
|
||||
|
||||
internal fun migrateSubtitle(
|
||||
fs: FileSystemService,
|
||||
subs: List<MigrateToContentStoreTask.Data.SingleSubtitle>
|
||||
): List<MigrateContentToStoreTaskResultEvent.SubtitleMigration> {
|
||||
|
||||
if (subs.isEmpty()) {
|
||||
return listOf(
|
||||
MigrateContentToStoreTaskResultEvent.SubtitleMigration(
|
||||
language = null,
|
||||
storedUri = null,
|
||||
status = MigrateStatus.NotPresent
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
return subs.map { sub ->
|
||||
val source = File(sub.cachedUri)
|
||||
val dest = File(sub.storeUri)
|
||||
|
||||
migrateFile(fs, source, dest)
|
||||
|
||||
MigrateContentToStoreTaskResultEvent.SubtitleMigration(
|
||||
language = sub.language,
|
||||
storedUri = dest.absolutePath,
|
||||
status = MigrateStatus.Completed
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
internal fun migrateCover(
|
||||
fs: FileSystemService,
|
||||
covers: List<MigrateToContentStoreTask.Data.SingleContent>
|
||||
): List<MigrateContentToStoreTaskResultEvent.FileMigration> {
|
||||
|
||||
if (covers.isEmpty()) {
|
||||
return listOf(
|
||||
MigrateContentToStoreTaskResultEvent.FileMigration(
|
||||
storedUri = null,
|
||||
status = MigrateStatus.NotPresent
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
return covers.map { cover ->
|
||||
val source = File(cover.cachedUri)
|
||||
val dest = File(cover.storeUri)
|
||||
|
||||
migrateFile(fs, source, dest)
|
||||
|
||||
MigrateContentToStoreTaskResultEvent.FileMigration(
|
||||
storedUri = dest.absolutePath,
|
||||
status = MigrateStatus.Completed
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
open fun getFileSystemService(): FileSystemService =
|
||||
DefaultFileSystemService()
|
||||
|
||||
class DefaultFileSystemService : FileSystemService {
|
||||
|
||||
override fun copy(source: File, destination: File) {
|
||||
if (!source.exists()) {
|
||||
throw FileServiceException.SourceMissing(source)
|
||||
}
|
||||
|
||||
try {
|
||||
source.copyTo(destination, overwrite = true)
|
||||
} catch (e: Exception) {
|
||||
throw FileServiceException.CopyFailed(source, destination, e)
|
||||
}
|
||||
}
|
||||
|
||||
override fun verifyIdentical(source: File, destination: File) {
|
||||
val mismatch = Files.mismatch(source.toPath(), destination.toPath())
|
||||
if (mismatch != -1L) {
|
||||
throw FileServiceException.VerificationFailed(source, destination)
|
||||
}
|
||||
}
|
||||
|
||||
override fun delete(file: File) {
|
||||
file.delete()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,73 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
||||
|
||||
import no.iktdev.eventi.models.Event
|
||||
import no.iktdev.eventi.models.Task
|
||||
import no.iktdev.eventi.models.store.TaskStatus
|
||||
import no.iktdev.eventi.tasks.TaskListener
|
||||
import no.iktdev.eventi.tasks.TaskType
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.http.HttpEntity
|
||||
import org.springframework.http.HttpHeaders
|
||||
import org.springframework.http.HttpMethod
|
||||
import org.springframework.http.MediaType
|
||||
import org.springframework.stereotype.Component
|
||||
import org.springframework.web.client.RestTemplate
|
||||
import java.util.*
|
||||
|
||||
@Component
|
||||
class StoreContentAndMetadataTaskListener : TaskListener(TaskType.MIXED) {
|
||||
|
||||
@Autowired
|
||||
lateinit var streamitRestTemplate: RestTemplate
|
||||
|
||||
override fun getWorkerId(): String {
|
||||
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
||||
}
|
||||
|
||||
override fun supports(task: Task): Boolean {
|
||||
return task is StoreContentAndMetadataTask
|
||||
}
|
||||
|
||||
override suspend fun onTask(task: Task): Event? {
|
||||
val pickedTask = task as? StoreContentAndMetadataTask ?: return null
|
||||
|
||||
val headers = HttpHeaders().apply { contentType = MediaType.APPLICATION_JSON }
|
||||
val entity = HttpEntity(pickedTask.data, headers)
|
||||
|
||||
// ❗ Ikke fang exceptions — la TaskListener håndtere dem
|
||||
val response = streamitRestTemplate.exchange(
|
||||
"/api/mediaprocesser/import",
|
||||
HttpMethod.POST,
|
||||
entity,
|
||||
Void::class.java,
|
||||
)
|
||||
|
||||
if (!response.statusCode.is2xxSuccessful) {
|
||||
throw IllegalStateException("StreamIt returned ${response.statusCode}")
|
||||
}
|
||||
|
||||
// Hvis vi kommer hit → alt OK
|
||||
return StoreContentAndMetadataTaskResultEvent(
|
||||
status = TaskStatus.Completed
|
||||
).producedFrom(task)
|
||||
}
|
||||
|
||||
override fun createIncompleteStateTaskEvent(
|
||||
task: Task,
|
||||
status: TaskStatus,
|
||||
exception: Exception?
|
||||
): Event {
|
||||
val message = when (status) {
|
||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
||||
TaskStatus.Cancelled -> "Canceled"
|
||||
else -> ""
|
||||
}
|
||||
|
||||
return StoreContentAndMetadataTaskResultEvent(
|
||||
status = status,
|
||||
error = message
|
||||
).producedFrom(task)
|
||||
}
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.services
|
||||
|
||||
import mu.KotlinLogging
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.requests.StartProcessRequest
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartFlow
|
||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
||||
import no.iktdev.mediaprocessing.shared.common.notExist
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
||||
import org.springframework.stereotype.Service
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
|
||||
@Service
|
||||
class CommandService {
|
||||
val log = KotlinLogging.logger {}
|
||||
|
||||
fun startProcess(request: StartProcessRequest): StartResult {
|
||||
return try {
|
||||
val file = File(request.fileUri)
|
||||
if (file.notExist()) {
|
||||
throw IllegalArgumentException("File does not exists at ${request.fileUri}")
|
||||
}
|
||||
if (!file.canRead()) {
|
||||
throw IllegalStateException("File is not readable ${request.fileUri}")
|
||||
}
|
||||
|
||||
val startProcessingEvent = StartProcessingEvent(
|
||||
data = StartData(
|
||||
fileUri = request.fileUri,
|
||||
operation = request.operationTypes,
|
||||
flow = StartFlow.Manual
|
||||
)
|
||||
).newReferenceId()
|
||||
EventStore.persist(startProcessingEvent)
|
||||
StartResult.Accepted(startProcessingEvent.referenceId)
|
||||
} catch (e: Exception) {
|
||||
log.error { e }
|
||||
StartResult.Rejected("Failed to start process for file ${request.fileUri}, with the following reason: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sealed class StartResult {
|
||||
data class Accepted(val referenceId: UUID) : StartResult()
|
||||
data class Rejected(val reason: String) : StartResult()
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,132 +0,0 @@
|
||||
package no.iktdev.mediaprocessing.coordinator.services
|
||||
|
||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealthStatus
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.health.SequenceHealth
|
||||
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
|
||||
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
|
||||
import no.iktdev.mediaprocessing.coordinator.util.getDiskInfoFor
|
||||
import no.iktdev.mediaprocessing.shared.common.dto.CurrentState
|
||||
import no.iktdev.mediaprocessing.shared.common.rules.EventLifecycleRules
|
||||
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
||||
import org.springframework.stereotype.Service
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
|
||||
@Service
|
||||
class CoordinatorHealthService(
|
||||
private val taskService: TaskService,
|
||||
private val eventService: EventService,
|
||||
private val aggregator: SequenceAggregatorService,
|
||||
private val coordinatorEnv: CoordinatorEnv
|
||||
) {
|
||||
|
||||
fun getHealth(): CoordinatorHealth {
|
||||
val tasks = taskService.getActiveTasks()
|
||||
val incompleteSequences = eventService.getIncompleteSequences()
|
||||
.groupBy { it.referenceId }
|
||||
.values
|
||||
|
||||
// --- TASK HEALTH ---
|
||||
val abandonedTaskIds = tasks
|
||||
.filter { TaskLifecycleRules.isAbandoned(it.consumed, it.persistedAt, it.lastCheckIn) }
|
||||
.map { it.taskId }
|
||||
|
||||
val stalledTaskIds = tasks
|
||||
.filter { TaskLifecycleRules.isStalled(it) }
|
||||
.map { it.taskId }
|
||||
|
||||
val failedTasks = taskService.getFailedTasks()
|
||||
val sequencesOnHold = aggregator.getActiveSequences().filter { it.currentState == CurrentState.OnHold }
|
||||
|
||||
// --- SEQUENCE HEALTH ---
|
||||
val overdueSequences = incompleteSequences
|
||||
.filter { EventLifecycleRules.isOverdue(it) }
|
||||
.map { seq ->
|
||||
val refId = seq.first().referenceId
|
||||
val firstEventAt = seq.minOf { it.persistedAt }
|
||||
val lastEventAt = seq.maxOf { it.persistedAt }
|
||||
|
||||
val expectedWindow = EventLifecycleRules.expectedCompletionTimeWindow(seq)
|
||||
val actualAge = Duration.between(firstEventAt, Instant.now())
|
||||
|
||||
// Operasjonelle verdier frontend trenger
|
||||
val expectedFinish = firstEventAt.plus(expectedWindow)
|
||||
val overdueDuration = actualAge.minus(expectedWindow).coerceAtLeast(Duration.ZERO)
|
||||
val isOverdue = overdueDuration > Duration.ZERO
|
||||
|
||||
SequenceHealth(
|
||||
referenceId = refId.toString(),
|
||||
|
||||
// eksisterende felter
|
||||
age = actualAge,
|
||||
expected = expectedWindow,
|
||||
lastEventAt = lastEventAt,
|
||||
eventCount = seq.size,
|
||||
|
||||
// nye operasjonelle felter
|
||||
startTime = firstEventAt,
|
||||
expectedFinishTime = expectedFinish,
|
||||
overdueDuration = overdueDuration,
|
||||
isOverdue = isOverdue
|
||||
)
|
||||
}
|
||||
|
||||
val overdueSequenceIds = overdueSequences.map { it.referenceId }
|
||||
|
||||
// --- AGGREGATED STATUS ---
|
||||
val status = when {
|
||||
abandonedTaskIds.isNotEmpty() ||
|
||||
stalledTaskIds.isNotEmpty() ||
|
||||
overdueSequenceIds.isNotEmpty() -> CoordinatorHealthStatus.DEGRADED
|
||||
|
||||
else -> CoordinatorHealthStatus.HEALTHY
|
||||
}
|
||||
|
||||
|
||||
val lastActivityCandidates = listOfNotNull(
|
||||
tasks.maxOfOrNull { it.persistedAt },
|
||||
eventService.getLastEventTimestamp()
|
||||
)
|
||||
|
||||
return CoordinatorHealth(
|
||||
status = status,
|
||||
abandonedTasks = abandonedTaskIds.size,
|
||||
stalledTasks = stalledTaskIds.size,
|
||||
activeTasks = tasks.count { !it.consumed },
|
||||
failedTasks = failedTasks.count(),
|
||||
queuedTasks = TaskStore.getPendingTasks().size,
|
||||
lastActivity = lastActivityCandidates.maxOrNull(),
|
||||
|
||||
abandonedTaskIds = abandonedTaskIds.map { it.toString() },
|
||||
stalledTaskIds = stalledTaskIds.map { it.toString() },
|
||||
sequencesOnHold = sequencesOnHold.size,
|
||||
sequencesOnHoldIds = sequencesOnHold.map { it.referenceId },
|
||||
overdueSequenceIds = overdueSequenceIds,
|
||||
overdueSequences = overdueSequences,
|
||||
|
||||
details = mapOf(
|
||||
"oldestActiveTaskAgeMinutes" to tasks.minOfOrNull {
|
||||
Duration.between(it.persistedAt, Instant.now()).toMinutes()
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fun getEventRate(): EventRate {
|
||||
return EventRate(
|
||||
lastMinute = eventService.getEventsLast(1),
|
||||
lastFiveMinutes = eventService.getEventsLast(5)
|
||||
)
|
||||
}
|
||||
|
||||
fun getDiskHealth(): List<DiskInfo> {
|
||||
val paths = listOf(coordinatorEnv.incomingContent,
|
||||
coordinatorEnv.cachedContent,
|
||||
coordinatorEnv.outgoingContent)
|
||||
.map { it -> it.absolutePath }
|
||||
return getDiskInfoFor(paths)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user