Compare commits

..

1 Commits
v5 ... v3

Author SHA1 Message Date
77b086cd82 Updated database version 2024-12-23 23:03:13 +01:00
670 changed files with 41794 additions and 28633 deletions

View File

@ -1,100 +0,0 @@
name: Build Java App
on:
workflow_call:
inputs:
app:
required: true
type: string
dockerTag:
required: true
type: string
enabled:
required: true
type: boolean
shouldBuild:
required: true
type: boolean
jobs:
build-java:
if: ${{ inputs.enabled && inputs.shouldBuild }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Detect frontend
id: detect_web
run: |
if [ -d "apps/${{ inputs.app }}/web" ]; then
echo "has_web=true" >> $GITHUB_OUTPUT
else
echo "has_web=false" >> $GITHUB_OUTPUT
fi
- name: Build React frontend
if: ${{ steps.detect_web.outputs.has_web == 'true' }}
run: |
cd apps/${{ inputs.app }}/web
npm install
export CI=false
npm run build
- name: Copy React build into Spring Boot resources
if: ${{ steps.detect_web.outputs.has_web == 'true' }}
run: |
rm -rf apps/${{ inputs.app }}/src/main/resources/static
mkdir -p apps/${{ inputs.app }}/src/main/resources/static
cp -r apps/${{ inputs.app }}/web/build/* apps/${{ inputs.app }}/src/main/resources/static
- name: Extract version
run: |
VERSION=$(grep '^version' apps/${{ inputs.app }}/build.gradle.kts | sed 's/.*"\(.*\)".*/\1/')
echo "VERSION=$VERSION" >> $GITHUB_ENV
- name: Run unit tests
run: |
chmod +x ./gradlew
./gradlew :apps:${{ inputs.app }}:test --info --stacktrace
- name: Build Java module
run: |
chmod +x ./gradlew
./gradlew :apps:${{ inputs.app }}:bootJar --info --stacktrace
- name: Build Docker image locally
run: |
docker build \
-f ./dockerfiles/DebianJava \
-t local-${{ inputs.app }}:${{ inputs.dockerTag }} \
--build-arg MODULE_NAME=${{ inputs.app }} \
--build-arg PASS_APP_VERSION=${{ env.VERSION }} \
.
- name: Test Docker container
run: |
docker run --rm local-${{ inputs.app }}:${{ inputs.dockerTag }} /bin/sh -c "echo 'Smoke test OK'"
- name: Docker login
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./dockerfiles/DebianJavaFfmpeg
build-args: |
MODULE_NAME=${{ inputs.app }}
PASS_APP_VERSION=${{ env.VERSION }}
push: true
tags: |
bskjon/mediaprocessing-${{ inputs.app }}:v5
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ inputs.dockerTag }}
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ github.sha }}

View File

@ -1,127 +0,0 @@
name: Build Python App
on:
workflow_call:
inputs:
app:
required: true
type: string
dockerTag:
required: true
type: string
enabled:
required: true
type: boolean
shouldBuild:
required: true
type: boolean
jobs:
build-python:
if: ${{ inputs.enabled && inputs.shouldBuild }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
# -----------------------------
# Cache pip per app
# -----------------------------
- name: Cache pip
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ inputs.app }}-${{ hashFiles('apps/${{ inputs.app }}/requirements*.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ inputs.app }}-
# -----------------------------
# Install Python deps
# -----------------------------
- name: Install dependencies
working-directory: apps/${{ inputs.app }}
run: |
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
# -----------------------------
# Run tests
# -----------------------------
- name: Run Python tests
working-directory: apps/${{ inputs.app }}
run: python -m pytest -q
# -----------------------------
# Setup Buildx
# -----------------------------
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# -----------------------------
# Cache Docker layers per app
# -----------------------------
- name: Cache Docker layers
uses: actions/cache@v4
with:
path: /tmp/.buildx-cache-${{ inputs.app }}
key: ${{ runner.os }}-buildx-${{ inputs.app }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-${{ inputs.app }}-
# -----------------------------
# Build image (load locally for smoke test)
# -----------------------------
- name: Build Docker image (local load)
uses: docker/build-push-action@v5
with:
context: .
file: ./dockerfiles/Python
build-args: |
MODULE_NAME=${{ inputs.app }}
load: true
tags: local-${{ inputs.app }}:${{ inputs.dockerTag }}
cache-from: type=local,src=/tmp/.buildx-cache-${{ inputs.app }}
cache-to: type=local,dest=/tmp/.buildx-cache-${{ inputs.app }}-new
# -----------------------------
# Smoke test
# -----------------------------
- name: Smoke test container
run: |
docker run --rm local-${{ inputs.app }}:${{ inputs.dockerTag }} \
/bin/sh -c "echo 'Smoke test OK'"
# -----------------------------
# Docker login
# -----------------------------
- name: Docker login
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
# -----------------------------
# Push final image (no rebuild)
# -----------------------------
- name: Push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./dockerfiles/Python
build-args: |
MODULE_NAME=${{ inputs.app }}
push: true
tags: |
bskjon/mediaprocessing-${{ inputs.app }}:v5
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ inputs.dockerTag }}
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ github.sha }}
cache-from: type=local,src=/tmp/.buildx-cache-${{ inputs.app }}
cache-to: type=local,dest=/tmp/.buildx-cache-${{ inputs.app }}-new
# -----------------------------
# Move Docker cache
# -----------------------------
- name: Move Docker cache
run: |
rm -rf /tmp/.buildx-cache-${{ inputs.app }}
mv /tmp/.buildx-cache-${{ inputs.app }}-new /tmp/.buildx-cache-${{ inputs.app }}

View File

@ -1,26 +0,0 @@
name: Build Shared
on:
workflow_call:
inputs:
dockerTag:
required: true
type: string
jobs:
build-shared:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Cache Gradle
uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
- name: Build Shared module
run: |
chmod +x ./gradlew
./gradlew :shared:build --info --stacktrace

View File

@ -1,115 +0,0 @@
name: Build v5
on:
push:
branches: [ v5 ]
pull_request:
branches: [ v5 ]
workflow_dispatch:
jobs:
pre-check:
runs-on: ubuntu-latest
outputs:
sharedDefinitions: ${{ steps.filter.outputs.sharedDefinitions }}
shared: ${{ steps.filter.outputs.shared }}
processer: ${{ steps.filter.outputs.processer }}
converter: ${{ steps.filter.outputs.converter }}
coordinator: ${{ steps.filter.outputs.coordinator }}
ui: ${{ steps.filter.outputs.ui }}
py-metadata: ${{ steps.filter.outputs.metadata }}
py-watcher: ${{ steps.filter.outputs.watcher }}
dockerTag: ${{ steps.tag.outputs.tag }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
base: ${{ github.event.before }}
- name: Detect changes
id: filter
uses: dorny/paths-filter@v3
with:
filters: |
shared:
- 'shared/**'
sharedDefinitions:
- 'gradle/**'
processer:
- 'apps/processer/**'
converter:
- 'apps/converter/**'
coordinator:
- 'apps/coordinator/**'
ui:
- 'apps/ui/**'
metadata:
- 'apps/py-metadata/**'
watcher:
- 'apps/py-watcher/**'
- name: Generate docker tag
id: tag
run: echo "tag=$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" >> $GITHUB_OUTPUT
build-shared:
needs: pre-check
if: ${{
needs.pre-check.outputs.shared == 'true'
|| needs.pre-check.outputs.sharedDefinitions == 'true'
|| needs.pre-check.outputs.processer == 'true'
|| needs.pre-check.outputs.converter == 'true'
|| needs.pre-check.outputs.coordinator == 'true'
|| needs.pre-check.outputs.ui == 'true'
|| github.event_name == 'workflow_dispatch'
}}
uses: ./.github/workflows/build-shared.yml
with:
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
build-java:
needs: [pre-check, build-shared]
strategy:
matrix:
include:
- app: processer
enabled: true
- app: converter
enabled: true
- app: coordinator
enabled: true
- app: ui
enabled: false
uses: ./.github/workflows/build-java-app.yml
with:
app: ${{ matrix.app }}
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
enabled: ${{ matrix.enabled }}
shouldBuild: ${{ needs.pre-check.outputs[matrix.app] == 'true'
|| needs.pre-check.outputs.shared == 'true'
|| needs.pre-check.outputs.sharedDefinitions == 'true'
|| github.event_name == 'workflow_dispatch' }}
secrets: inherit
build-python:
needs: [pre-check]
strategy:
matrix:
include:
- app: py-metadata
enabled: true
- app: py-watcher
enabled: true
uses: ./.github/workflows/build-python-app.yml
with:
app: ${{ matrix.app }}
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
enabled: ${{ matrix.enabled }}
shouldBuild: ${{ needs.pre-check.outputs[matrix.app] == 'true'
|| github.event_name == 'workflow_dispatch' }}
secrets: inherit

View File

@ -1,386 +0,0 @@
name: Build v4
on:
push:
branches:
- v4
pull_request:
branches:
- v4
workflow_dispatch:
jobs:
pre-check:
runs-on: ubuntu-latest
outputs:
pyMetadata: ${{ steps.checkAppChanges.outputs.metadata }}
sharedLibrary: ${{ steps.checkAppChanges.outputs.shared }}
coordinator: ${{ steps.checkAppChanges.outputs.coordinator }}
processer: ${{ steps.checkAppChanges.outputs.processer }}
converter: ${{ steps.checkAppChanges.outputs.converter }}
ui: ${{ steps.checkAppChanges.outputs.ui }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
name: "Detect app changes"
id: checkAppChanges
with:
base: ''
filters: |
metadata:
- 'apps/pyMetadata/**'
coordinator:
- 'apps/coordinator/**'
processer:
- 'apps/processer/**'
converter:
- 'apps/converter/**'
ui:
- 'apps/ui/**'
shared:
- 'shared/**'
# Step to print the outputs from "pre-check" job
- name: Print Outputs from pre-check job
run: |
echo "Apps\n"
echo "app:pyMetadata: ${{ steps.checkAppChanges.outputs.metadata }}"
echo "app:coordinator: ${{ steps.checkAppChanges.outputs.coordinator }}"
echo "app:processer: ${{ steps.checkAppChanges.outputs.processer }}"
echo "app:converter: ${{ steps.checkAppChanges.outputs.converter }}"
echo "app:ui: ${{ steps.checkAppChanges.outputs.ui }}"
echo "Shared"
echo "shared: ${{ steps.checkAppChanges.outputs.shared }}"
echo "${{ steps.checkAppChanges.outputs }}"
build-shared:
runs-on: ubuntu-latest
needs: pre-check
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Cache Shared code Gradle dependencies
id: cache-gradle
uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
- name: Build Shared code
if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.sharedLibrary == 'true' || github.event_name == 'workflow_dispatch'
run: |
chmod +x ./gradlew
./gradlew :shared:build --stacktrace --info
build-processer:
needs:
- build-shared
- pre-check
if: ${{ needs.pre-check.outputs.processer == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
runs-on: ubuntu-latest
#if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Cache Shared Gradle dependencies
id: cache-gradle
uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
- name: Extract version from build.gradle.kts
id: extract_version
run: |
VERSION=$(cat ./apps/processer/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
echo "VERSION=$VERSION"
echo "VERSION=$VERSION" >> $GITHUB_ENV
- name: Build Processer module
id: build-processer
run: |
chmod +x ./gradlew
./gradlew :apps:processer:bootJar --info --stacktrace
echo "Build completed"
- name: Generate Docker image tag
id: docker-tag
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
- name: Docker login
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v2
with:
context: .
file: ./dockerfiles/DebianJavaFfmpeg
build-args: |
MODULE_NAME=processer
PASS_APP_VERSION=${{ env.VERSION }}
push: true
tags: |
bskjon/mediaprocessing-processer:v4
bskjon/mediaprocessing-processer:v4-${{ github.sha }}
bskjon/mediaprocessing-processer:v4-${{ steps.docker-tag.outputs.tag }}
build-converter:
needs:
- build-shared
- pre-check
if: ${{ needs.pre-check.outputs.converter == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
runs-on: ubuntu-latest
#if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Cache Shared Gradle dependencies
id: cache-gradle
uses: actions/cache@v3
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
- name: Extract version from build.gradle.kts
id: extract_version
run: |
VERSION=$(cat ./apps/converter/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
echo "VERSION=$VERSION"
echo "VERSION=$VERSION" >> $GITHUB_ENV
- name: Build Converter module
id: build-converter
run: |
chmod +x ./gradlew
./gradlew :apps:converter:bootJar --info --debug
echo "Build completed"
- name: Generate Docker image tag
id: docker-tag
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
- name: Docker login
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v2
with:
context: .
file: ./dockerfiles/DebianJava
build-args: |
MODULE_NAME=converter
PASS_APP_VERSION=${{ env.VERSION }}
push: true
tags: |
bskjon/mediaprocessing-converter:v4
bskjon/mediaprocessing-converter:v4-${{ github.sha }}
bskjon/mediaprocessing-converter:v4-${{ steps.docker-tag.outputs.tag }}
build-coordinator:
needs:
- build-shared
- pre-check
if: ${{ needs.pre-check.outputs.coordinator == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
runs-on: ubuntu-latest
#if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Cache Shared Gradle dependencies
id: cache-gradle
uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
- name: Extract version from build.gradle.kts
id: extract_version
run: |
VERSION=$(cat ./apps/coordinator/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
echo "VERSION=$VERSION"
echo "VERSION=$VERSION" >> $GITHUB_ENV
- name: Build Coordinator module
id: build-coordinator
run: |
chmod +x ./gradlew
./gradlew :apps:coordinator:bootJar
echo "Build completed"
- name: Generate Docker image tag
id: docker-tag
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
- name: Docker login
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Debug Check extracted version
run: |
echo "Extracted version: ${{ env.VERSION }}"
- name: Build and push Docker image
uses: docker/build-push-action@v2
with:
context: .
file: ./dockerfiles/DebianJavaFfmpeg
build-args: |
MODULE_NAME=coordinator
PASS_APP_VERSION=${{ env.VERSION }}
push: true
tags: |
bskjon/mediaprocessing-coordinator:v4
bskjon/mediaprocessing-coordinator:v4-${{ github.sha }}
bskjon/mediaprocessing-coordinator:v4-${{ steps.docker-tag.outputs.tag }}
build-pymetadata:
needs:
- pre-check
if: ${{ needs.pre-check.outputs.pyMetadata == 'true' || github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Build pyMetadata module
id: build-pymetadata
run: |
if [[ "${{ needs.pre-check.outputs.pyMetadata }}" == "true" || "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
cd apps/pyMetadata
# Add the necessary build steps for your Python module here
echo "Build completed"
else
echo "pyMetadata has not changed. Skipping pyMetadata module build."
echo "::set-output name=job_skipped::true"
fi
- name: Generate Docker image tag
id: docker-tag
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
- name: Docker login
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v5.1.0
with:
context: .
file: ./dockerfiles/Python
build-args:
MODULE_NAME=pyMetadata
push: true
tags: |
bskjon/mediaprocessing-pymetadata:v4
bskjon/mediaprocessing-pymetadata:v4-${{ github.sha }}
bskjon/mediaprocessing-pymetadata:v4-${{ steps.docker-tag.outputs.tag }}
build-ui:
needs:
- build-shared
- pre-check
if: ${{ needs.pre-check.outputs.ui == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Cache Shared Gradle dependencies
id: cache-gradle
uses: actions/cache@v4
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
- name: Setup Node.js
uses: actions/setup-node@v2
with:
node-version: '14'
- name: Install dependencies
run: npm install
working-directory: ./apps/ui/web
- name: Build React app
run: |
export CI=false
npm run build
working-directory: ./apps/ui/web
- name : Copy build files
run: |
mkdir -p ./apps/ui/src/main/resources/static
cp -r ./apps/ui/web/build/* ./apps/ui/src/main/resources/static
- name: Extract version from build.gradle.kts
id: extract_version
run: |
VERSION=$(cat ./apps/ui/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
echo "VERSION=$VERSION"
echo "VERSION=$VERSION" >> $GITHUB_ENV
- name: Build UI module
id: build-ui
run: |
chmod +x ./gradlew
./gradlew :apps:ui:bootJar
echo "Build completed"
- name: Generate Docker image tag
id: docker-tag
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
- name: Docker login
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Debug Check extracted version
run: |
echo "Extracted version: ${{ env.VERSION }}"
- name: Build and push Docker image
uses: docker/build-push-action@v2
with:
context: .
file: ./dockerfiles/DebianJava
build-args: |
MODULE_NAME=ui
PASS_APP_VERSION=${{ env.VERSION }}
push: true
tags: |
bskjon/mediaprocessing-ui:v4
bskjon/mediaprocessing-ui:v4-${{ github.sha }}
bskjon/mediaprocessing-ui:v4-${{ steps.docker-tag.outputs.tag }}

7
.gitignore vendored
View File

@ -36,12 +36,7 @@ bin/
/.nb-gradle/ /.nb-gradle/
### VS Code ### ### VS Code ###
.vscode/
### Mac OS ### ### Mac OS ###
.DS_Store .DS_Store
.idea/runConfigurations
/apps/py-metadata/venv/
/apps/py-watcher/venv/

View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="AgentMigrationStateService">
<option name="migrationStatus" value="COMPLETED" />
</component>
</project>

View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="AskMigrationStateService">
<option name="migrationStatus" value="COMPLETED" />
</component>
</project>

View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Ask2AgentMigrationStateService">
<option name="migrationStatus" value="COMPLETED" />
</component>
</project>

View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="EditMigrationStateService">
<option name="migrationStatus" value="COMPLETED" />
</component>
</project>

4
.idea/gradle.xml generated
View File

@ -5,7 +5,6 @@
<option name="linkedExternalProjectsSettings"> <option name="linkedExternalProjectsSettings">
<GradleProjectSettings> <GradleProjectSettings>
<option name="externalProjectPath" value="$PROJECT_DIR$" /> <option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="gradleJvm" value="17" />
<option name="modules"> <option name="modules">
<set> <set>
<option value="$PROJECT_DIR$" /> <option value="$PROJECT_DIR$" />
@ -16,8 +15,7 @@
<option value="$PROJECT_DIR$/apps/ui" /> <option value="$PROJECT_DIR$/apps/ui" />
<option value="$PROJECT_DIR$/shared" /> <option value="$PROJECT_DIR$/shared" />
<option value="$PROJECT_DIR$/shared/common" /> <option value="$PROJECT_DIR$/shared/common" />
<option value="$PROJECT_DIR$/shared/event-task-contract" /> <option value="$PROJECT_DIR$/shared/eventi" />
<option value="$PROJECT_DIR$/shared/ffmpeg" />
</set> </set>
</option> </option>
</GradleProjectSettings> </GradleProjectSettings>

2
.idea/kotlinc.xml generated
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="KotlinJpsPluginSettings"> <component name="KotlinJpsPluginSettings">
<option name="version" value="2.1.0" /> <option name="version" value="1.9.20" />
</component> </component>
</project> </project>

2
.idea/misc.xml generated
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" /> <component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" project-jdk-name="azul-17" project-jdk-type="JavaSDK" /> <component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="azul-17" project-jdk-type="JavaSDK" />
</project> </project>

View File

@ -1,26 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="UIApplicationKt" type="JetRunConfigurationType" nameIsGenerated="true">
<envs>
<env name="DATABASE_ADDRESS" value="192.168.2.250" />
<env name="DATABASE_NAME_E" value="eventsV4" />
<env name="DATABASE_NAME_S" value="streamitv3" />
<env name="DATABASE_PASSWORD" value="shFZ27eL2x2NoxyEDBMfDWkvFO" />
<env name="DATABASE_PORT" value="3306" />
<env name="DATABASE_USERNAME" value="root" />
<env name="DIRECTORY_CONTENT_INCOMING" value="G:\MediaProcessingPlayground\input" />
<env name="DIRECTORY_CONTENT_OUTGOING" value="G:\MediaProcessingPlayground\output" />
<env name="DISABLE_COMPLETE" value="true" />
<env name="DISABLE_PRODUCE" value="true" />
<env name="EncoderWs" value="ws://192.168.2.250:6081/ws" />
<env name="METADATA_TIMEOUT" value="0" />
<env name="SUPPORTING_EXECUTABLE_FFMPEG" value="G:\MediaProcessingPlayground\ffmpeg.exe" />
<env name="SUPPORTING_EXECUTABLE_FFPROBE" value="G:\MediaProcessingPlayground\ffprobe.exe" />
</envs>
<option name="MAIN_CLASS_NAME" value="no.iktdev.mediaprocessing.ui.UIApplicationKt" />
<module name="MediaProcessing.apps.ui.main" />
<shortenClasspath name="NONE" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

1182
.idea/workspace.xml generated

File diff suppressed because it is too large Load Diff

View File

@ -20,5 +20,5 @@ tasks.test {
useJUnitPlatform() useJUnitPlatform()
} }
kotlin { kotlin {
jvmToolchain(21) jvmToolchain(17)
} }

View File

@ -1,8 +1,9 @@
plugins { plugins {
id("java") id("java")
kotlin("jvm") kotlin("jvm")
id("org.springframework.boot") kotlin("plugin.spring") version "1.5.31"
id("io.spring.dependency-management") id("org.springframework.boot") version "2.5.5"
id("io.spring.dependency-management") version "1.0.11.RELEASE"
} }
group = "no.iktdev.mediaprocessing.apps" group = "no.iktdev.mediaprocessing.apps"
@ -26,41 +27,40 @@ repositories {
} }
} }
val exposedVersion = "0.44.0"
dependencies { dependencies {
/*Spring boot*/ /*Spring boot*/
implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.boot:spring-boot-starter-actuator") implementation("org.springframework.boot:spring-boot-starter:2.7.0")
implementation("org.springframework.boot:spring-boot-starter-websocket") // implementation("org.springframework.kafka:spring-kafka:3.0.1")
implementation("org.springframework:spring-tx") implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3")
implementation("org.springframework.kafka:spring-kafka:2.8.5")
implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion")
implementation ("mysql:mysql-connector-java:8.0.29")
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
implementation("com.google.code.gson:gson:2.8.9") implementation("com.google.code.gson:gson:2.8.9")
implementation("org.json:json:20210307") implementation("org.json:json:20210307")
implementation(libs.exfl) implementation("no.iktdev:exfl:0.0.16-SNAPSHOT")
implementation("no.iktdev.library:subtitle:1.8.1-SNAPSHOT") implementation("no.iktdev.library:subtitle:1.7.9-SNAPSHOT")
implementation(libs.eventi)
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
implementation("com.github.pgreze:kotlin-process:1.4.1") implementation("com.github.pgreze:kotlin-process:1.4.1")
implementation(project(mapOf("path" to ":shared:eventi")))
implementation(project(mapOf("path" to ":shared:common"))) implementation(project(mapOf("path" to ":shared:common")))
implementation(project(mapOf("path" to ":shared:database")))
implementation(kotlin("stdlib-jdk8")) implementation(kotlin("stdlib-jdk8"))
testImplementation("io.mockk:mockk:1.12.0")
testImplementation("org.springframework.boot:spring-boot-starter-test")
testImplementation(project(":shared:common", configuration = "testArtifacts"))
testImplementation(project(":shared:database", configuration = "testArtifacts"))
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
val exposedVersion = "0.61.0"
testImplementation("org.jetbrains.exposed:exposed-core:${exposedVersion}")
} }
tasks.test { tasks.test {
@ -78,5 +78,5 @@ tasks.jar {
} }
kotlin { kotlin {
jvmToolchain(21) jvmToolchain(17)
} }

View File

@ -1,28 +1,37 @@
package no.iktdev.mediaprocessing.converter package no.iktdev.mediaprocessing.converter
import mu.KotlinLogging import mu.KotlinLogging
import no.iktdev.eventi.events.EventTypeRegistry
import no.iktdev.eventi.tasks.TaskTypeRegistry
import no.iktdev.exfl.coroutines.CoroutinesDefault import no.iktdev.exfl.coroutines.CoroutinesDefault
import no.iktdev.exfl.coroutines.CoroutinesIO import no.iktdev.exfl.coroutines.CoroutinesIO
import no.iktdev.exfl.observable.Observables import no.iktdev.exfl.observable.Observables
import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry import no.iktdev.eventi.database.MySqlDataSource
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
import no.iktdev.mediaprocessing.shared.common.database.tables.runners
import no.iktdev.mediaprocessing.shared.common.database.tables.tasks
import no.iktdev.mediaprocessing.shared.common.getAppVersion import no.iktdev.mediaprocessing.shared.common.getAppVersion
import no.iktdev.mediaprocessing.shared.database.DatabaseApplication import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
import no.iktdev.mediaprocessing.shared.database.DatabasebasedMediaProcessingApp import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.boot.runApplication import org.springframework.boot.runApplication
import org.springframework.context.annotation.Configuration
@DatabasebasedMediaProcessingApp @SpringBootApplication
open class ConverterApplication: DatabaseApplication() { class ConvertApplication
}
val ioCoroutine = CoroutinesIO() val ioCoroutine = CoroutinesIO()
val defaultCoroutine = CoroutinesDefault() val defaultCoroutine = CoroutinesDefault()
lateinit var taskManager: TasksManager
lateinit var runnerManager: RunnerManager
private lateinit var eventsDatabase: MySqlDataSource
private val log = KotlinLogging.logger {} private val log = KotlinLogging.logger {}
fun getEventsDatabase(): MySqlDataSource {
return eventsDatabase
}
fun main(args: Array<String>) { fun main(args: Array<String>) {
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> { ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
override fun onUpdated(value: Throwable) { override fun onUpdated(value: Throwable) {
@ -36,19 +45,15 @@ fun main(args: Array<String>) {
}) })
runApplication<ConverterApplication>(*args) eventsDatabase = DatabaseEnvConfig.toEventsDatabase()
eventsDatabase.createDatabase()
eventsDatabase.createTables(tasks, runners)
taskManager = TasksManager(eventsDatabase)
runnerManager = RunnerManager(dataSource = getEventsDatabase(), name = ConvertApplication::class.java.simpleName)
runnerManager.assignRunner()
runApplication<ConvertApplication>(*args)
log.info { "App Version: ${getAppVersion()}" } log.info { "App Version: ${getAppVersion()}" }
} }
//private val logger = KotlinLogging.logger {} //private val logger = KotlinLogging.logger {}
@Configuration
open class ApplicationConfiguration() {
init {
EventRegistry.getEvents().let {
EventTypeRegistry.register(it)
}
TaskRegistry.getTasks().let {
TaskTypeRegistry.register(it)
}
}
}

View File

@ -1,10 +0,0 @@
package no.iktdev.mediaprocessing.converter
import no.iktdev.library.subtitle.reader.BaseReader
import java.io.File
interface ConverterEnvironment {
fun canRead(file: File): Boolean
fun getReader(file: File): BaseReader?
fun createExporter(input: File, outputDir: File, name: String): Exporter
}

View File

@ -1,14 +0,0 @@
package no.iktdev.mediaprocessing.converter
import no.iktdev.library.subtitle.classes.Dialog
import no.iktdev.library.subtitle.export.Export
class ExportAdapter(
private val export: Export
) : Exporter {
override fun write(dialogs: List<Dialog>) = export.write(dialogs)
override fun writeSrt(dialogs: List<Dialog>) = export.writeSrt(dialogs)
override fun writeSmi(dialogs: List<Dialog>) = export.writeSmi(dialogs)
override fun writeVtt(dialogs: List<Dialog>) = export.writeVtt(dialogs)
}

View File

@ -1,11 +0,0 @@
package no.iktdev.mediaprocessing.converter
import no.iktdev.library.subtitle.classes.Dialog
import java.io.File
interface Exporter {
fun write(dialogs: List<Dialog>): MutableList<File>
fun writeSrt(dialogs: List<Dialog>): File
fun writeSmi(dialogs: List<Dialog>): File
fun writeVtt(dialogs: List<Dialog>): File
}

View File

@ -0,0 +1,8 @@
package no.iktdev.mediaprocessing.converter
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
import org.springframework.context.annotation.Configuration
import org.springframework.context.annotation.Import
@Configuration
class SocketLocalInit: SocketImplementation()

View File

@ -0,0 +1,84 @@
package no.iktdev.mediaprocessing.converter
import mu.KotlinLogging
import no.iktdev.mediaprocessing.shared.common.*
import no.iktdev.mediaprocessing.shared.common.database.cal.ActiveMode
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
import no.iktdev.mediaprocessing.shared.common.task.TaskType
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
import org.springframework.scheduling.annotation.EnableScheduling
import org.springframework.stereotype.Service
@Service
@EnableScheduling
class TaskCoordinator(): TaskCoordinatorBase() {
private val log = KotlinLogging.logger {}
override fun onProduceEvent(event: Event) {
taskManager.produceEvent(event)
}
override fun onCoordinatorReady() {
super.onCoordinatorReady()
runnerManager = RunnerManager(dataSource = getEventsDatabase(), name = ConvertApplication::class.java.simpleName)
runnerManager.assignRunner()
}
override val taskAvailabilityEventListener: MutableMap<TaskType, MutableList<TaskQueueListener>> = mutableMapOf(
TaskType.Convert to mutableListOf(),
)
private val taskListeners: MutableSet<TaskEvents> = mutableSetOf()
fun getTaskListeners(): List<TaskEvents> {
return taskListeners.toList()
}
fun addTaskEventListener(listener: TaskEvents) {
taskListeners.add(listener)
}
fun addConvertTaskListener(listener: TaskQueueListener) {
addTaskListener(TaskType.Convert, listener)
}
override fun addTaskListener(type: TaskType, listener: TaskQueueListener) {
super.addTaskListener(type, listener)
pullForAvailableTasks()
}
override fun pullForAvailableTasks() {
if (runnerManager.iAmSuperseded()) {
// This will let the application complete but not consume new
taskMode = ActiveMode.Passive
return
}
val available = taskManager.getClaimableTasks().asClaimable()
available.forEach { (type, list) ->
taskAvailabilityEventListener[type]?.forEach { listener ->
list.foreachOrUntilClaimed {
listener.onTaskAvailable(it)
}
}
}
}
override fun clearExpiredClaims() {
val expiredClaims = taskManager.getTasksWithExpiredClaim().filter { it.task in listOf(TaskType.Convert) }
expiredClaims.forEach {
log.info { "Found event with expired claim: ${it.referenceId}::${it.eventId}::${it.task}" }
}
expiredClaims.forEach {
val result = taskManager.deleteTaskClaim(referenceId = it.referenceId, eventId = it.eventId)
if (result) {
log.info { "Released claim on ${it.referenceId}::${it.eventId}::${it.task}" }
} else {
log.error { "Failed to release claim on ${it.referenceId}::${it.eventId}::${it.task}" }
}
}
}
interface TaskEvents {
fun onCancelOrStopProcess(eventId: String)
}
}

View File

@ -1,76 +0,0 @@
package no.iktdev.mediaprocessing.converter
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskPollerImplementation
import no.iktdev.eventi.tasks.TaskReporter
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.boot.ApplicationArguments
import org.springframework.boot.ApplicationRunner
import org.springframework.stereotype.Component
import org.springframework.stereotype.Service
import java.util.*
@Component
class PollerAdministrator(
private val taskPoller: TaskPoller,
): ApplicationRunner {
override fun run(args: ApplicationArguments?) {
CoroutineScope(Dispatchers.Default).launch {
taskPoller.start()
}
}
}
@Service
class TaskPoller(
private val reporter: TaskReporter,
) : TaskPollerImplementation(
taskStore = TaskStore,
reporterFactory = { reporter } // én reporter brukes for alle tasks
) {
}
@Component
class DefaultTaskReporter() : TaskReporter {
override fun markClaimed(taskId: UUID, workerId: String) {
TaskStore.claim(taskId, workerId)
}
override fun updateLastSeen(taskId: UUID) {
TaskStore.heartbeat(taskId)
}
override fun markCompleted(taskId: UUID) {
TaskStore.markConsumed(taskId, TaskStatus.Completed)
}
override fun markFailed(referenceId: UUID, taskId: UUID) {
TaskStore.markConsumed(taskId, TaskStatus.Failed)
}
override fun markCancelled(referenceId: UUID, taskId: UUID) {
TaskStore.markConsumed(taskId, TaskStatus.Cancelled)
}
override fun updateProgress(taskId: UUID, progress: Int) {
// Not to be implemented for this application
}
override fun log(taskId: UUID, message: String) {
// Not to be implemented for this application
}
override fun publishEvent(event: Event) {
EventStore.persist(event)
}
}

View File

@ -1,28 +0,0 @@
package no.iktdev.mediaprocessing.converter.controller
import org.springframework.boot.actuate.health.HealthEndpoint
import org.springframework.boot.actuate.health.Status
import org.springframework.http.HttpStatus
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("/system")
class ReadinessController(
private val healthEndpoint: HealthEndpoint
) {
@GetMapping("/ready")
fun ready(): ResponseEntity<String> {
val health = healthEndpoint.health()
return if (health.status == Status.UP) {
ResponseEntity.ok("READY")
} else {
ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
.body("NOT_READY: ${health.status}")
}
}
}

View File

@ -1,18 +0,0 @@
package no.iktdev.mediaprocessing.converter.convert
import no.iktdev.library.subtitle.reader.BaseReader
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
import java.io.File
abstract class Converter(val env: ConverterEnvironment, val listener: ConvertListener) {
var writtenUris: List<String>? = null
abstract fun getSubtitleReader(useFile: File): BaseReader?
abstract suspend fun convert(data: ConvertTask.Data)
class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException()
class FileUnavailableException(override val message: String): RuntimeException()
abstract fun getResult(): List<String>
}

View File

@ -4,70 +4,75 @@ import no.iktdev.library.subtitle.Configuration
import no.iktdev.library.subtitle.Syncro import no.iktdev.library.subtitle.Syncro
import no.iktdev.library.subtitle.classes.Dialog import no.iktdev.library.subtitle.classes.Dialog
import no.iktdev.library.subtitle.classes.DialogType import no.iktdev.library.subtitle.classes.DialogType
import no.iktdev.library.subtitle.export.Export
import no.iktdev.library.subtitle.reader.BaseReader import no.iktdev.library.subtitle.reader.BaseReader
import no.iktdev.library.subtitle.reader.Reader
import no.iktdev.mediaprocessing.converter.ConverterEnv import no.iktdev.mediaprocessing.converter.ConverterEnv
import no.iktdev.mediaprocessing.converter.ConverterEnvironment import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask import no.iktdev.mediaprocessing.shared.common.contract.dto.SubtitleFormats
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
import java.io.File import java.io.File
import kotlin.jvm.Throws
class Converter2( class Converter2(val data: ConvertData,
env: ConverterEnvironment, private val listener: ConvertListener) {
listener: ConvertListener
): Converter(env = env, listener = listener) {
@Throws(FileUnavailableException::class) @Throws(FileUnavailableException::class)
override fun getSubtitleReader(useFile: File): BaseReader? { private fun getReader(): BaseReader? {
if (!env.canRead(useFile)) { val file = File(data.inputFile)
if (!file.canRead())
throw FileUnavailableException("Can't open file for reading..") throw FileUnavailableException("Can't open file for reading..")
} return Reader(file).getSubtitleReader()
return env.getReader(useFile)
} }
private fun syncDialogs(input: List<Dialog>): List<Dialog> { private fun syncDialogs(input: List<Dialog>): List<Dialog> {
return if (ConverterEnv.syncDialogs) Syncro().sync(input) else input return if (ConverterEnv.syncDialogs) Syncro().sync(input) else input
} }
fun canRead(): Boolean {
try {
val reader = getReader()
return reader != null
} catch (e: FileUnavailableException) {
return false
}
}
@Throws(FileUnavailableException::class, FileIsNullOrEmpty::class) @Throws(FileUnavailableException::class, FileIsNullOrEmpty::class)
override suspend fun convert(data: ConvertTask.Data) { fun execute() {
val file = File(data.inputFile) val file = File(data.inputFile)
listener.onStarted(file.absolutePath) listener.onStarted(file.absolutePath)
try { try {
Configuration.exportJson = true Configuration.exportJson = true
val read = getSubtitleReader(file)?.read() ?: throw FileIsNullOrEmpty() val read = getReader()?.read() ?: throw FileIsNullOrEmpty()
if (read.isEmpty()) if (read.isEmpty())
throw FileIsNullOrEmpty() throw FileIsNullOrEmpty()
val filtered = read.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) } val filtered = read.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) }
val syncOrNotSync = syncDialogs(filtered) val syncOrNotSync = syncDialogs(filtered)
val exporter = env.createExporter(file, File(data.outputDirectory), data.outputFileName) val exporter = Export(file, File(data.outputDirectory), data.outputFileName)
val outFiles = if (data.formats.isEmpty()) { val outFiles = if (data.formats.isEmpty()) {
exporter.write(syncOrNotSync) exporter.write(syncOrNotSync)
} else { } else {
val exported = mutableListOf<File>() val exported = mutableListOf<File>()
if (data.formats.contains(SubtitleFormat.SRT)) { if (data.formats.contains(SubtitleFormats.SRT)) {
exported.add(exporter.writeSrt(syncOrNotSync)) exported.add(exporter.writeSrt(syncOrNotSync))
} }
if (data.formats.contains(SubtitleFormat.SMI)) { if (data.formats.contains(SubtitleFormats.SMI)) {
exported.add(exporter.writeSmi(syncOrNotSync)) exported.add(exporter.writeSmi(syncOrNotSync))
} }
if (data.formats.contains(SubtitleFormat.VTT)) { if (data.formats.contains(SubtitleFormats.VTT)) {
exported.add(exporter.writeVtt(syncOrNotSync)) exported.add(exporter.writeVtt(syncOrNotSync))
} }
exported exported
} }
writtenUris = outFiles.map { it.absolutePath } listener.onCompleted(file.absolutePath, outFiles.map { it.absolutePath })
listener.onCompleted(file.absolutePath, writtenUris!!)
} catch (e: Exception) { } catch (e: Exception) {
listener.onError(file.absolutePath, e.message ?: e.localizedMessage) listener.onError(file.absolutePath, e.message ?: e.localizedMessage)
} }
} }
override fun getResult(): List<String> {
if (writtenUris == null) { class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException()
throw IllegalStateException("Execute must be called before getting result") class FileUnavailableException(override val message: String): RuntimeException()
}
return writtenUris!!
}
} }

View File

@ -1,117 +0,0 @@
package no.iktdev.mediaprocessing.converter.listeners
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.Task
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskListener
import no.iktdev.eventi.tasks.TaskType
import no.iktdev.library.subtitle.export.Export
import no.iktdev.library.subtitle.reader.BaseReader
import no.iktdev.library.subtitle.reader.Reader
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
import no.iktdev.mediaprocessing.converter.ExportAdapter
import no.iktdev.mediaprocessing.converter.Exporter
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
import no.iktdev.mediaprocessing.converter.convert.Converter
import no.iktdev.mediaprocessing.converter.convert.Converter2
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
import org.springframework.stereotype.Component
import java.io.File
import java.util.*
@Component
open class ConvertTaskListener(): TaskListener(TaskType.CPU_INTENSIVE) {
override fun getWorkerId(): String {
return "${this::class.java.simpleName}-${TaskType.CPU_INTENSIVE}-${UUID.randomUUID()}"
}
override fun supports(task: Task): Boolean {
return task is ConvertTask
}
override suspend fun onTask(task: Task): Event? {
if (task !is ConvertTask) {
throw IllegalArgumentException("Invalid task type: ${task::class.java.name}")
}
val converter = getConverter()
withHeartbeatRunner {
reporter?.updateLastSeen(task.taskId)
}
withContext(Dispatchers.Unconfined) {
converter.convert(task.data)
}
return try {
val result = converter.getResult()
val newEvent = ConvertTaskResultEvent(
data = ConvertTaskResultEvent.ConvertedData(
language = task.data.language,
outputFiles = result,
baseName = task.data.outputFileName
),
status = TaskStatus.Completed
).producedFrom(task)
newEvent
} catch (e: Exception) {
e.printStackTrace()
val newEvent = ConvertTaskResultEvent(
data = null,
status = TaskStatus.Failed
).producedFrom(task)
newEvent
}
}
override fun createIncompleteStateTaskEvent(
task: Task,
status: TaskStatus,
exception: Exception?
): Event {
val message = when (status) {
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
TaskStatus.Cancelled -> "Canceled"
else -> ""
}
return ConvertTaskResultEvent(null, status, error = message)
}
open fun getConverter(): Converter {
return Converter2(getConverterEnvironment(), getListener())
}
open fun getListener(): ConvertListener {
return DefaultConvertListener()
}
class DefaultConvertListener: ConvertListener {
override fun onStarted(inputFile: String) {
}
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
}
}
open fun getConverterEnvironment(): ConverterEnvironment {
return DefaultConverterEnvironment()
}
class DefaultConverterEnvironment : ConverterEnvironment {
override fun canRead(file: File) = file.canRead()
override fun getReader(file: File): BaseReader? =
Reader(file).getSubtitleReader()
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
return ExportAdapter(Export(input, outputDir, name))
}
}
}

View File

@ -0,0 +1,133 @@
package no.iktdev.mediaprocessing.converter.tasks
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import mu.KotlinLogging
import no.iktdev.eventi.data.EventMetadata
import no.iktdev.eventi.data.EventStatus
import no.iktdev.mediaprocessing.converter.*
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
import no.iktdev.mediaprocessing.converter.convert.Converter2
import no.iktdev.mediaprocessing.shared.common.database.cal.Status
import no.iktdev.mediaprocessing.shared.common.services.TaskService
import no.iktdev.mediaprocessing.shared.common.task.Task
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertWorkPerformed
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertedData
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
@Service
class ConvertService(
@Autowired var tasks: TaskCoordinator,
) : TaskService(), ConvertListener, TaskCoordinator.TaskEvents {
fun getProducerName(): String {
return this::class.java.simpleName
}
override val log = KotlinLogging.logger {}
override val logDir = ConverterEnv.logDirectory
override fun getServiceId(serviceName: String): String {
return super.getServiceId(this::class.java.simpleName)
}
var worker: Converter2? = null
override fun onAttachListener() {
tasks.addConvertTaskListener(this)
tasks.addTaskEventListener(this)
}
override fun isReadyToConsume(): Boolean {
return worker == null
}
override fun isTaskClaimable(task: Task): Boolean {
return !taskManager.isTaskClaimed(referenceId = task.referenceId, eventId = task.eventId)
}
override fun onTaskAssigned(task: Task) {
startConvert(task)
}
fun startConvert(task: Task) {
val convert = task.data as ConvertData
worker = Converter2(convert, this)
worker?.execute()
}
override fun onStarted(inputFile: String) {
val task = assignedTask ?: return
taskManager.markTaskAsClaimed(task.referenceId, task.eventId, serviceId)
log.info { "Convert started for ${task.referenceId}" }
}
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
val task = assignedTask ?: return
log.info { "Convert completed for ${task.referenceId}" }
val claimSuccessful = taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
runBlocking {
delay(1000)
if (!claimSuccessful) {
taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
delay(1000)
}
var readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
while (!readbackIsSuccess) {
delay(1000)
readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
}
tasks.onProduceEvent(ConvertWorkPerformed(
metadata = EventMetadata(
referenceId = task.referenceId,
derivedFromEventId = task.eventId,
status = EventStatus.Success,
source = getProducerName()
),
data = ConvertedData(
outputFiles = outputFiles
)
))
onClearTask()
}
}
override fun onError(inputFile: String, message: String) {
val task = assignedTask ?: return
super.onError(inputFile, message)
log.info { "Convert error for ${task.referenceId}\nmessage: $message" }
taskManager.markTaskAsCompleted(task.referenceId, task.eventId, Status.ERROR)
tasks.onProduceEvent(ConvertWorkPerformed(
metadata = EventMetadata(
referenceId = task.referenceId,
derivedFromEventId = task.eventId,
status = EventStatus.Failed,
source = getProducerName()
)
))
onClearTask()
}
override fun onClearTask() {
super.onClearTask()
worker = null
}
override fun onCancelOrStopProcess(eventId: String) {
TODO("Not yet implemented")
}
}

View File

@ -0,0 +1,3 @@
spring.output.ansi.enabled=always
logging.level.org.apache.kafka=WARN
logging.level.root=INFO

View File

@ -1,33 +0,0 @@
spring:
output:
ansi:
enabled: always
flyway:
enabled: true
locations: classpath:flyway
baseline-on-migrate: false
management:
endpoints:
web:
exposure:
include: health
endpoint:
health:
show-details: always
logging:
level:
root: INFO
org.apache.kafka: INFO
Exposed: OFF
org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN
media:
cache: /src/cache
outgoing: /src/output
incoming: /src/input
streamit:
address: http://streamit.service

View File

@ -1,82 +0,0 @@
package no.iktdev.mediaprocessing.converter
import io.mockk.every
import io.mockk.mockk
import io.mockk.mockkObject
import io.mockk.verify
import no.iktdev.eventi.models.Task
import no.iktdev.mediaprocessing.shared.common.TestBase
import no.iktdev.mediaprocessing.shared.database.config.DatasourceConfiguration
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.jetbrains.exposed.sql.Database
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.springframework.boot.builder.SpringApplicationBuilder
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.test.context.TestPropertySource
import org.springframework.test.context.junit.jupiter.SpringExtension
import javax.sql.DataSource
@SpringBootTest(
classes = [ConverterApplication::class,
DatasourceConfiguration::class],
webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT
)
@TestPropertySource(properties = ["spring.flyway.enabled=true"])
@ExtendWith(SpringExtension::class)
class ConverterApplicationTest : TestBase() {
data class TestTask(
val success: Boolean
) : Task()
@Test
fun `context loads and common configuration is available`() {
// Hvis du har beans du vil verifisere, kan du autowire dem her
// @Autowired lateinit var database: Database
// Dummy assertion for å verifisere at konteksten starter
assertNotNull(Unit)
}
@Test
fun `Verify that we can access TaskStore`() {
val tasks = TaskStore.getPendingTasks()
assertNotNull(tasks)
assert(tasks.isEmpty())
TaskStore.persist(TestTask(success = true).newReferenceId())
val tasksAfter = TaskStore.getPendingTasks()
assertNotNull(tasksAfter)
assert(tasksAfter.isNotEmpty())
}
@Test
fun `ExposedInitializer should connect to database`() {
mockkObject(Database)
every {
Database.connect(
any<DataSource>(),
any(),
any(),
any(),
any()
)
} returns mockk()
val context = SpringApplicationBuilder(ConverterApplication::class.java)
.properties("spring.main.web-application-type=none")
.run()
verify(exactly = 1) { Database.connect(any<DataSource>(), any(), any(), any(), any()) }
}
}

View File

@ -1,38 +0,0 @@
package no.iktdev.mediaprocessing.converter
import kotlinx.coroutines.delay
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.library.subtitle.reader.BaseReader
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
import no.iktdev.mediaprocessing.converter.convert.Converter
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
import java.io.File
class MockConverter(
val delayMillis: Long = 0,
private val simulatedResult: List<String>? = null,
private val taskResultStatus: TaskStatus = TaskStatus.Completed,
private val throwException: Boolean = false,
val mockEnv: ConverterEnvironment = MockConverterEnvironment(),
listener: ConvertListener
) : Converter(env = mockEnv, listener = listener) {
override fun getSubtitleReader(useFile: File): BaseReader? {
TODO("Not yet implemented")
}
override suspend fun convert(data: ConvertTask.Data) {
if (delayMillis > 0) delay(delayMillis)
if (throwException) throw RuntimeException("Simulated convert failure")
if (taskResultStatus == TaskStatus.Failed) {
listener.onError(data.inputFile, "Failed state desired")
} else {
listener.onCompleted(data.inputFile, simulatedResult!!)
}
}
override fun getResult(): List<String> {
return simulatedResult!!
}
}

View File

@ -1,19 +0,0 @@
package no.iktdev.mediaprocessing.converter
import no.iktdev.library.subtitle.reader.BaseReader
import java.io.File
class MockConverterEnvironment(
var canReadValue: Boolean = true,
var reader: BaseReader? = null,
var exporter: Exporter? = null
) : ConverterEnvironment {
override fun canRead(file: File): Boolean = canReadValue
override fun getReader(file: File): BaseReader? = reader
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
return exporter ?: error("FakeEnv.exporter must be set before calling createExporter")
}
}

View File

@ -1,249 +0,0 @@
package no.iktdev.mediaprocessing.converter.convert
import kotlinx.coroutines.test.runTest
import no.iktdev.library.subtitle.classes.Dialog
import no.iktdev.library.subtitle.classes.DialogType
import no.iktdev.library.subtitle.classes.Time
import no.iktdev.library.subtitle.reader.BaseReader
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
import no.iktdev.mediaprocessing.converter.Exporter
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import java.io.File
class Converter2Test {
// -------------------------------------------------------------------------
// Fake implementations
// -------------------------------------------------------------------------
class FakeReader(
private val dialogs: List<Dialog>,
private val shouldThrow: Boolean = false
) : BaseReader() {
override fun read(): List<Dialog> {
if (shouldThrow) throw RuntimeException("reader failed")
return dialogs
}
}
class FakeExporter(
private val srtFile: File? = null,
private val smiFile: File? = null,
private val vttFile: File? = null,
private val filesForWrite: List<File> = emptyList(),
private val shouldThrow: Boolean = false
) : Exporter {
override fun write(dialogs: List<Dialog>): MutableList<File> {
if (shouldThrow) throw RuntimeException("export failed")
return filesForWrite.toMutableList()
}
override fun writeSrt(dialogs: List<Dialog>): File =
srtFile ?: error("srtFile not set in FakeExporter")
override fun writeSmi(dialogs: List<Dialog>): File =
smiFile ?: error("smiFile not set in FakeExporter")
override fun writeVtt(dialogs: List<Dialog>): File =
vttFile ?: error("vttFile not set in FakeExporter")
}
class FakeListener : ConvertListener {
var started = 0
var completed = 0
var errors = 0
var lastError: String? = null
override fun onStarted(inputFile: String) {
started++
}
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
completed++
}
override fun onError(inputFile: String, message: String) {
errors++
lastError = message
}
}
class FakeEnv(
var canReadValue: Boolean = true,
var reader: BaseReader? = null,
var exporter: Exporter? = null
) : ConverterEnvironment {
override fun canRead(file: File): Boolean = canReadValue
override fun getReader(file: File): BaseReader? = reader
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
return exporter ?: error("FakeEnv.exporter must be set before calling createExporter")
}
}
// -------------------------------------------------------------------------
// Helpers
// -------------------------------------------------------------------------
private fun makeTaskData(
input: String = "input.srt",
language: String = "no",
outDir: String = "out",
outName: String = "name",
formats: List<SubtitleFormat> = emptyList(),
allowOverwrite: Boolean = true
) = ConvertTask.Data(
inputFile = input,
language = language,
outputDirectory = outDir,
outputFileName = outName,
formats = formats,
allowOverwrite = allowOverwrite
)
// -------------------------------------------------------------------------
// Tests
// -------------------------------------------------------------------------
@Test
@DisplayName(
"""
Når execute() kjøres
Hvis reader returnerer tom liste
kalles onError
"""
)
fun execute_emptyFile() = runTest {
val env = FakeEnv(
canReadValue = true,
reader = FakeReader(emptyList())
)
val listener = FakeListener()
val converter = Converter2(
env = env,
listener = listener
)
converter.convert(makeTaskData())
assertEquals(1, listener.errors)
assertEquals(0, listener.completed)
}
@Test
@DisplayName(
"""
Når execute() kjøres
Hvis reader returnerer dialoger
Og exporter.write lykkes
kalles onCompleted
"""
)
fun execute_success() = runTest {
val dialogs = listOf(
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
)
val env = FakeEnv(
canReadValue = true,
reader = FakeReader(dialogs),
exporter = FakeExporter(srtFile = File("/fake/out.srt"))
)
val listener = FakeListener()
val converter = Converter2(
env = env,
listener = listener
)
converter.convert(
makeTaskData(
formats = listOf(SubtitleFormat.SRT)
)
)
assertEquals(1, listener.completed)
assertEquals(listOf("/fake/out.srt"), converter.getResult())
}
@Test
@DisplayName(
"""
Når execute() kjøres
Hvis exporter.write kaster exception
kalles onError
"""
)
fun execute_exportFails() = runTest {
val dialogs = listOf(
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
)
val env = FakeEnv(
canReadValue = true,
reader = FakeReader(dialogs),
exporter = FakeExporter(srtFile = File("/fake/out.srt"), shouldThrow = true)
)
val listener = FakeListener()
val converter = Converter2(
env = env,
listener = listener
)
converter.convert(makeTaskData())
assertEquals(1, listener.errors)
assertEquals(0, listener.completed)
}
@Test
@DisplayName(
"""
Når execute() kjøres
Hvis formats inneholder SRT og VTT
brukes writeSrt og writeVtt
"""
)
fun execute_multipleFormats() = runTest {
val dialogs = listOf(
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
)
val env = FakeEnv(
canReadValue = true,
reader = FakeReader(dialogs),
exporter = FakeExporter(srtFile = File("/fake/out.srt"), vttFile = File("/fake/out.vtt"))
)
val listener = FakeListener()
val converter = Converter2(
env = env,
listener = listener
)
converter.convert(
makeTaskData(
formats = listOf(SubtitleFormat.SRT, SubtitleFormat.VTT)
)
)
assertEquals(1, listener.completed)
assertEquals(listOf("/fake/out.srt", "/fake/out.vtt"), converter.getResult())
}
}

View File

@ -1,247 +0,0 @@
package no.iktdev.mediaprocessing.converter.listeners
import kotlinx.coroutines.test.runTest
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.Task
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskReporter
import no.iktdev.library.subtitle.classes.Dialog
import no.iktdev.library.subtitle.classes.DialogType
import no.iktdev.library.subtitle.classes.Time
import no.iktdev.library.subtitle.reader.BaseReader
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
import no.iktdev.mediaprocessing.converter.Exporter
import no.iktdev.mediaprocessing.converter.MockConverter
import no.iktdev.mediaprocessing.converter.MockConverterEnvironment
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
import no.iktdev.mediaprocessing.converter.convert.Converter
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
import org.junit.jupiter.api.Assertions.*
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import java.io.File
import java.util.*
import kotlin.system.measureTimeMillis
class ConvertTaskListenerTest {
class ConvertTaskListenerTestImplementation : ConvertTaskListener() {
fun getJob() = currentJob
var overrideEnv: ConverterEnvironment = DefaultConverterEnvironment()
override fun getConverterEnvironment(): ConverterEnvironment {
return overrideEnv
}
var overrideListener: ConvertListener? = null
override fun getListener(): ConvertListener {
if (overrideListener != null)
return overrideListener!!
else
return super.getListener()
}
var overrideConverter: Converter? = null
override fun getConverter(): Converter {
return if (overrideConverter != null)
overrideConverter!!
else
super.getConverter()
}
}
val listener = ConvertTaskListenerTestImplementation()
// ---------------------------------------------------------------------
// Fake environment + fake converter
// ---------------------------------------------------------------------
class FakeListener : ConvertListener {
var started = 0
var completed = 0
var errors = 0
override fun onStarted(inputFile: String) {
started++
}
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
completed++
}
override fun onError(inputFile: String, message: String) {
errors++
}
}
class FakeExporter(
private val files: List<File>,
private val shouldThrow: Boolean = false
) : Exporter {
override fun write(dialogs: List<Dialog>): MutableList<File> {
if (shouldThrow) throw RuntimeException("export failed")
return files.toMutableList()
}
override fun writeSrt(dialogs: List<Dialog>) = files[0]
override fun writeSmi(dialogs: List<Dialog>) = files[0]
override fun writeVtt(dialogs: List<Dialog>) = files[0]
}
class FakeReader(
private val dialogs: List<Dialog>,
private val shouldThrow: Boolean = false
) : BaseReader() {
override fun read(): List<Dialog> {
if (shouldThrow) throw RuntimeException("reader failed")
return dialogs
}
}
val overrideReporter = object : TaskReporter {
override fun markClaimed(taskId: UUID, workerId: String) {}
override fun updateLastSeen(taskId: UUID) {}
override fun markCompleted(taskId: UUID) {}
override fun markFailed(referenceId: UUID, taskId: UUID) {}
override fun markCancelled(referenceId: UUID, taskId: UUID) {}
override fun updateProgress(taskId: UUID, progress: Int) {}
override fun log(taskId: UUID, message: String) {}
override fun publishEvent(event: Event) {
}
}
// ---------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------
private fun makeTask(
formats: List<SubtitleFormat> = emptyList()
): ConvertTask {
return ConvertTask(
ConvertTask.Data(
inputFile = "input.srt",
language = "no",
outputDirectory = "out",
outputFileName = "name",
formats = formats,
allowOverwrite = true
)
).apply { newReferenceId() }
}
// ---------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------
@Test
@DisplayName("""
Når onTask kjøres og converterer
Hvis den bruker lengre tid
:
Skal koden vente til den er ferdig
""")
fun onTask_validate_delay() = runTest {
val delay = 1000L
val converter = MockConverter(
delay,
listOf("file:///potato.srt"),
listener = FakeListener()
)
listener.apply {
overrideConverter = converter
}
val task = makeTask()
val event = listener.onTask(task)
val time = measureTimeMillis {
val accepted = listener.accept(task, overrideReporter)
assertTrue(accepted, "Task listener did not accept the task.")
listener.getJob()?.join()
assertTrue(event is ConvertTaskResultEvent)
assertEquals(TaskStatus.Completed, (event as ConvertTaskResultEvent).status)
}
}
@Test
@DisplayName(
"""
Når onTask kjøres
Hvis converter lykkes
returneres Completed-event
"""
)
fun onTask_success() = runTest {
val dialogs = listOf(
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
)
val env = MockConverterEnvironment(
canReadValue = true,
reader = FakeReader(dialogs),
exporter = FakeExporter(listOf(File("/fake/out.srt")))
)
listener.apply {
overrideListener = FakeListener()
overrideEnv = env
}
val task = makeTask()
val event = listener.onTask(task) as ConvertTaskResultEvent
assertEquals(TaskStatus.Completed, event.status)
assertEquals(listOf("/fake/out.srt"), event.data!!.outputFiles)
}
@Test
@DisplayName(
"""
Når onTask kjøres
Hvis converter feiler
returneres Failed-event
"""
)
fun onTask_failure() = runTest {
val env = MockConverterEnvironment(
canReadValue = true,
reader = FakeReader(emptyList()) // triggers FileIsNullOrEmpty
)
listener.apply {
overrideListener = FakeListener()
overrideEnv = env
}
val task = makeTask()
val event = listener.onTask(task) as ConvertTaskResultEvent
assertEquals(TaskStatus.Failed, event.status)
assertNull(event.data)
}
@Test
@DisplayName(
"""
Når supports() kalles
returnerer den true for ConvertTask og false for andre typer
"""
)
fun supports_test() {
val listener = ConvertTaskListener()
assertTrue(listener.supports(makeTask()))
assertFalse(listener.supports(DummyTask()))
}
class DummyTask : Task()
}

View File

@ -1,41 +0,0 @@
spring:
main:
allow-bean-definition-overriding: true
flyway:
enabled: false
locations: classpath:flyway
autoconfigure:
exclude:
- org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
output:
ansi:
enabled: always
springdoc:
swagger-ui:
path: /open/swagger-ui
logging:
level:
org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping: DEBUG
management:
endpoints:
web:
exposure:
include:
- mappings
- health
endpoint:
health:
show-details: always
media:
cache: /src/cache
outgoing: /src/output
incoming: /src/input
streamit:
address: http://streamit.service

View File

@ -22,7 +22,7 @@ Only one instance is supported, while multiple processer's can be run at any tim
- Extracts info from filename - Extracts info from filename
- Extracts info from file media streams - Extracts info from file media streams
- Produces title and sanitized - Produces title and sanitized
- py-metadata: - pyMetadata:
- Picks up event - Picks up event
- Searches with sources using title and sanitized - Searches with sources using title and sanitized
- Produces result - Produces result

View File

@ -1,10 +1,10 @@
plugins { plugins {
id("java") id("java")
kotlin("jvm") kotlin("jvm")
kotlin("plugin.spring") kotlin("plugin.spring") version "1.5.31"
id("org.springframework.boot") id("org.springframework.boot") version "2.5.5"
id("io.spring.dependency-management") id("io.spring.dependency-management") version "1.0.11.RELEASE"
id("org.jetbrains.kotlin.plugin.serialization") id("org.jetbrains.kotlin.plugin.serialization") version "1.5.0" // Legg til Kotlin Serialization-plugin
} }
group = "no.iktdev.mediaprocessing" group = "no.iktdev.mediaprocessing"
@ -22,45 +22,44 @@ repositories {
} }
val exposedVersion = "0.44.0"
dependencies { dependencies {
/*Spring boot*/ /*Spring boot*/
implementation("org.springframework.boot:spring-boot-starter")
implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.boot:spring-boot-starter-webflux") implementation("org.springframework.boot:spring-boot-starter:3.2.0")
implementation("org.springframework.boot:spring-boot-starter-actuator") // implementation("org.springframework.kafka:spring-kafka:3.0.1")
implementation("org.springframework.boot:spring-boot-starter-websocket") implementation("org.springframework.kafka:spring-kafka:2.8.5")
implementation("org.springframework:spring-tx") implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3")
implementation("org.springframework.boot:spring-boot-starter-validation")
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11") implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
implementation("com.google.code.gson:gson:2.8.9") implementation("com.google.code.gson:gson:2.8.9")
implementation("org.json:json:20210307") implementation("org.json:json:20210307")
implementation(libs.exfl) implementation("no.iktdev:exfl:0.0.16-SNAPSHOT")
implementation("no.iktdev.streamit.library:streamit-library-db:1.0.0-alpha14") implementation("no.iktdev.streamit.library:streamit-library-db:1.0.0-alpha11")
implementation(libs.eventi)
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT") implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
//implementation(project(mapOf("path" to ":shared")))
implementation(project(mapOf("path" to ":shared:ffmpeg"))) implementation(project(mapOf("path" to ":shared:eventi")))
implementation(project(mapOf("path" to ":shared:common"))) implementation(project(mapOf("path" to ":shared:common")))
implementation(project(mapOf("path" to ":shared:database")))
implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion")
implementation ("mysql:mysql-connector-java:8.0.29")
implementation("org.jetbrains.kotlin:kotlin-stdlib") implementation("org.jetbrains.kotlin:kotlin-stdlib")
implementation(kotlin("stdlib-jdk8")) implementation(kotlin("stdlib-jdk8"))
testImplementation("org.assertj:assertj-core:3.21.0") testImplementation("org.assertj:assertj-core:3.21.0")
testImplementation("junit:junit:4.12") testImplementation("junit:junit:4.12")
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2")
testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.0") testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.0")
testImplementation("org.skyscreamer:jsonassert:1.5.0") testImplementation("org.skyscreamer:jsonassert:1.5.0")
testImplementation("org.mockito:mockito-core:3.+") testImplementation("org.mockito:mockito-core:3.+")
@ -70,35 +69,30 @@ dependencies {
testImplementation("org.mockito:mockito-core:3.+") testImplementation("org.mockito:mockito-core:3.+")
testImplementation("org.assertj:assertj-core:3.4.1") testImplementation("org.assertj:assertj-core:3.4.1")
/*testImplementation("org.junit.vintage:junit-vintage-engine")
testImplementation("org.junit.jupiter:junit-jupiter:5.10.1")
testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1")
testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.1")
testRuntimeOnly ("org.junit.jupiter:junit-jupiter-engine:5.10.1")
testImplementation("org.mockito:mockito-core:5.8.0") // Oppdater versjonen hvis det er nyere tilgjengelig
testImplementation("org.mockito:mockito-junit-jupiter:5.8.0")
testImplementation(platform("org.junit:junit-bom:5.10.1"))
testImplementation("org.junit.platform:junit-platform-runner:1.10.1")*/
testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation(platform("org.junit:junit-bom:5.9.1"))
testImplementation("org.junit.jupiter:junit-jupiter") testImplementation("org.junit.jupiter:junit-jupiter")
testImplementation("org.junit.jupiter:junit-jupiter-params")
testImplementation("junit:junit:4.13.2") testImplementation("junit:junit:4.13.2")
testImplementation("org.mockito:mockito-core:3.+") testImplementation("org.mockito:mockito-core:3.+")
testImplementation("org.assertj:assertj-core:3.4.1") testImplementation("org.assertj:assertj-core:3.4.1")
testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0") testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0")
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
testImplementation("io.mockk:mockk:1.13.9")
testImplementation("org.mockito:mockito-inline:5.2.0")
testImplementation("org.mockito.kotlin:mockito-kotlin:5.2.1")
testImplementation("org.mockito:mockito-junit-jupiter:5.11.0")
testImplementation(project(":shared:common", configuration = "testArtifacts"))
testImplementation(project(":shared:database", configuration = "testArtifacts"))
testImplementation("org.springframework.boot:spring-boot-starter-test")
val exposedVersion = "0.61.0"
testImplementation("org.jetbrains.exposed:exposed-core:${exposedVersion}")
} }
tasks.withType<Test> { tasks.withType<Test> {
useJUnitPlatform() useJUnitPlatform()
} }
kotlin { kotlin {
jvmToolchain(21) jvmToolchain(17)
} }
tasks.bootJar { tasks.bootJar {

View File

@ -1,33 +1,56 @@
package no.iktdev.mediaprocessing.coordinator package no.iktdev.mediaprocessing.coordinator
import mu.KotlinLogging import mu.KotlinLogging
import no.iktdev.eventi.events.EventTypeRegistry
import no.iktdev.eventi.tasks.TaskTypeRegistry
import no.iktdev.exfl.coroutines.CoroutinesDefault import no.iktdev.exfl.coroutines.CoroutinesDefault
import no.iktdev.exfl.coroutines.CoroutinesIO import no.iktdev.exfl.coroutines.CoroutinesIO
import no.iktdev.exfl.observable.Observables import no.iktdev.exfl.observable.Observables
import no.iktdev.mediaprocessing.coordinator.config.ExecutablesConfig import no.iktdev.mediaprocessing.shared.common.*
import no.iktdev.mediaprocessing.coordinator.config.ProcesserClientProperties import no.iktdev.eventi.database.MySqlDataSource
import no.iktdev.mediaprocessing.shared.common.configs.StreamItConfig import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
import no.iktdev.mediaprocessing.shared.common.getAppVersion import no.iktdev.streamit.library.db.tables.*
import no.iktdev.mediaprocessing.shared.database.DatabaseApplication import no.iktdev.streamit.library.db.tables.helper.cast_errors
import no.iktdev.mediaprocessing.shared.database.DatabasebasedMediaProcessingApp import no.iktdev.streamit.library.db.tables.helper.data_audio
import org.springframework.boot.context.properties.EnableConfigurationProperties import no.iktdev.streamit.library.db.tables.helper.data_video
import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.boot.runApplication import org.springframework.boot.runApplication
import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Bean
@DatabasebasedMediaProcessingApp val log = KotlinLogging.logger {}
class CoordinatorApplication: DatabaseApplication() { private lateinit var eventDatabase: EventsDatabase
private lateinit var eventsManager: EventsManager
lateinit var runnerManager: RunnerManager
@SpringBootApplication
class CoordinatorApplication {
@Bean
fun eventManager(): EventsManager {
return eventsManager
} }
}
private lateinit var storeDatabase: MySqlDataSource
val ioCoroutine = CoroutinesIO() val ioCoroutine = CoroutinesIO()
val defaultCoroutine = CoroutinesDefault() val defaultCoroutine = CoroutinesDefault()
private val log = KotlinLogging.logger {}
fun getStoreDatabase(): MySqlDataSource {
return storeDatabase
}
lateinit var taskManager: TasksManager
fun main(args: Array<String>) { fun main(args: Array<String>) {
printSharedConfig()
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> { ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
override fun onUpdated(value: Throwable) { override fun onUpdated(value: Throwable) {
value.printStackTrace() value.printStackTrace()
@ -38,31 +61,49 @@ fun main(args: Array<String>) {
value.printStackTrace() value.printStackTrace()
} }
}) })
eventDatabase = EventsDatabase().also {
eventsManager = EventsManager(it.database)
}
storeDatabase = DatabaseEnvConfig.toStoredDatabase()
storeDatabase.createDatabase()
taskManager = TasksManager(eventDatabase.database)
val tables = arrayOf(
catalog,
genre,
movie,
serie,
subtitle,
summary,
users,
progress,
data_audio,
data_video,
cast_errors,
titles
)
storeDatabase.createTables(*tables)
runnerManager = RunnerManager(dataSource = eventDatabase.database, name = CoordinatorApplication::class.java.simpleName)
runnerManager.assignRunner()
runApplication<CoordinatorApplication>(*args) runApplication<CoordinatorApplication>(*args)
log.info { "App Version: ${getAppVersion()}" } log.info { "App Version: ${getAppVersion()}" }
} }
//private val logger = KotlinLogging.logger {}
@Configuration fun printSharedConfig() {
open class ApplicationConfiguration() { log.info { "File Input: ${SharedConfig.incomingContent}" }
init { log.info { "File Output: ${SharedConfig.outgoingContent}" }
EventRegistry.getEvents().let { log.info { "Ffprobe: ${SharedConfig.ffprobe}" }
EventTypeRegistry.register(it) log.info { "Ffmpeg: ${SharedConfig.ffmpeg}" }
}
TaskRegistry.getTasks().let {
TaskTypeRegistry.register(it)
}
}
}
@Configuration /*log.info { "Database: ${DatabaseConfig.database} @ ${DatabaseConfig.address}:${DatabaseConfig.port}" }
@EnableConfigurationProperties( log.info { "Username: ${DatabaseConfig.username}" }
value = [ log.info { "Password: ${if (DatabaseConfig.password.isNullOrBlank()) "Is not set" else "Is set"}" }*/
ExecutablesConfig::class, }
StreamItConfig::class,
ProcesserClientProperties::class
]
)
class CoordinatorConfig

View File

@ -1,23 +0,0 @@
package no.iktdev.mediaprocessing.coordinator
import no.iktdev.mediaprocessing.coordinator.config.ExecutablesConfig
import no.iktdev.mediaprocessing.shared.common.configs.MediaPaths
import no.iktdev.mediaprocessing.shared.common.configs.StreamItConfig
import org.springframework.stereotype.Service
import java.io.File
@Service
class CoordinatorEnv(
val streamIt: StreamItConfig,
val exec: ExecutablesConfig,
val media: MediaPaths
) {
val streamitAddress = streamIt.address
val ffprobe = exec.ffprobe
val cachedContent = File(media.cache)
val outgoingContent = File(media.outgoing)
val incomingContent = File(media.incoming)
val preference: File = File("/data/config/preference.json")
}

View File

@ -0,0 +1,97 @@
package no.iktdev.mediaprocessing.coordinator
import no.iktdev.eventi.data.EventMetadata
import no.iktdev.eventi.data.EventStatus
import no.iktdev.eventi.data.eventId
import no.iktdev.eventi.implementations.ActiveMode
import no.iktdev.eventi.implementations.EventCoordinator
import no.iktdev.mediaprocessing.shared.common.contract.Events
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaProcessStartEvent
import no.iktdev.mediaprocessing.shared.common.contract.data.PermitWorkCreationEvent
import no.iktdev.mediaprocessing.shared.common.contract.data.StartEventData
import no.iktdev.mediaprocessing.shared.common.contract.dto.StartOperationEvents
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.context.ApplicationContext
import org.springframework.stereotype.Component
import java.io.File
import java.util.*
@Component
class Coordinator(
@Autowired
override var applicationContext: ApplicationContext,
@Autowired
override var eventManager: EventsManager
) : EventCoordinator<Event, EventsManager>() {
init {
}
fun getProducerName(): String {
return this::class.java.simpleName
}
public fun startProcess(file: File, type: ProcessType) {
val operations: List<StartOperationEvents> = listOf(
StartOperationEvents.ENCODE,
StartOperationEvents.EXTRACT,
StartOperationEvents.CONVERT
)
startProcess(file, type, operations)
}
fun startProcess(file: File, type: ProcessType, operations: List<StartOperationEvents>): UUID {
val referenceId: UUID = UUID.randomUUID()
val event = MediaProcessStartEvent(
metadata = EventMetadata(
referenceId = referenceId.toString(),
status = EventStatus.Success,
source = getProducerName()
),
data = StartEventData(
file = file.absolutePath,
type = type,
operations = operations
)
)
produceNewEvent(event)
return referenceId
}
fun permitWorkToProceedOn(referenceId: String, events: List<Event>, message: String) {
val defaultRequiredBy = listOf(Events.EventMediaParameterEncodeCreated, Events.EventMediaParameterExtractCreated)
val eventToAttachTo = if (events.any { it.eventType in defaultRequiredBy }) {
events.findLast { it.eventType in defaultRequiredBy }
} else events.find { it.eventType == Events.EventMediaProcessStarted }
if (eventToAttachTo == null) {
log.error { "No event to attach permit to" }
return
}
produceNewEvent(
PermitWorkCreationEvent(
metadata = EventMetadata(
referenceId = referenceId,
derivedFromEventId = eventToAttachTo.eventId(),
status = EventStatus.Success,
source = getProducerName()
),
data = message
)
)
}
override fun getActiveTaskMode(): ActiveMode {
if (runnerManager.iAmSuperseded()) {
// This will let the application complete but not consume new
taskMode = ActiveMode.Passive
}
return taskMode
}
}

View File

@ -0,0 +1,11 @@
package no.iktdev.mediaprocessing.coordinator
import no.iktdev.mediaprocessing.shared.common.contract.Events
import no.iktdev.mediaprocessing.shared.common.contract.EventsListenerContract
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
abstract class CoordinatorEventListener(): EventsListenerContract<EventsManager, Coordinator>() {
abstract override val produceEvent: Events
abstract override val listensForEvents: List<Events>
abstract override var coordinator: Coordinator?
}

View File

@ -1,21 +0,0 @@
package no.iktdev.mediaprocessing.coordinator
import no.iktdev.mediaprocessing.shared.common.model.ProgressUpdate
import org.springframework.stereotype.Service
import java.util.concurrent.ConcurrentHashMap
@Service
class CoordinatorService {
private val progressMap = ConcurrentHashMap<String, ProgressUpdate>()
fun updateProgress(update: ProgressUpdate) {
progressMap[update.taskId] = update
}
fun getProgress(taskId: String): ProgressUpdate? =
progressMap[taskId]
fun getProgress(): List<ProgressUpdate> =
progressMap.values.toList()
}

View File

@ -1,45 +0,0 @@
package no.iktdev.mediaprocessing.coordinator
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import no.iktdev.eventi.events.EventDispatcher
import no.iktdev.eventi.events.EventPollerImplementation
import no.iktdev.eventi.events.SequenceDispatchQueue
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
import org.springframework.context.SmartLifecycle
import org.springframework.context.annotation.DependsOn
import org.springframework.stereotype.Component
@Component
class EventPollerAdministrator(
private val eventPoller: EventPoller
) : SmartLifecycle {
private var running = false
var job: Job? = null
override fun start() {
job = CoroutineScope(Dispatchers.Default).launch {
eventPoller.start()
}
running = true
}
override fun stop() {
job?.cancel()
}
override fun isRunning() = running
}
val sequenceDispatcher = SequenceDispatchQueue(8)
val dispatcher = EventDispatcher(eventStore = EventStore)
@Component
@DependsOn("ExposedInit")
class EventPoller: EventPollerImplementation(eventStore = EventStore, dispatchQueue = sequenceDispatcher, dispatcher = dispatcher) {
}

View File

@ -0,0 +1,23 @@
package no.iktdev.mediaprocessing.coordinator
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
import no.iktdev.mediaprocessing.shared.common.database.tables.*
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
class EventsDatabase() {
val database = DatabaseEnvConfig.toEventsDatabase()
val tables = listOf(
events, // For kafka
allEvents,
tasks,
runners,
processed
)
init {
database.createDatabase()
database.createTables(*tables.toTypedArray())
}
}

View File

@ -0,0 +1,11 @@
package no.iktdev.mediaprocessing.coordinator
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
import org.springframework.context.annotation.Configuration
import org.springframework.context.annotation.Import
@Configuration
class SocketLocalInit: SocketImplementation() {
}

View File

@ -1,190 +0,0 @@
package no.iktdev.mediaprocessing.coordinator
import com.google.gson.Gson
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import no.iktdev.mediaprocessing.ffmpeg.dsl.AudioCodec
import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoCodec
import org.springframework.stereotype.Component
import java.io.File
import java.io.IOException
// ------------------------------------------------------------
// MODELLER
// ------------------------------------------------------------
data class PeferenceConfig(
val processer: ProcesserPreference,
val language: LanguagePreference
)
data class LanguagePreference(
val preferredAudio: List<String>,
val preferredSubtitles: List<String>,
val preferOriginal: Boolean = true,
val avoidDub: Boolean = true,
// NEW: Prioritet for hvilket subtitle-format som skal brukes som master
val subtitleFormatPriority: List<String> = listOf("ass", "srt", "vtt", "smi"),
// NEW: Hvordan subtitles skal velges
val subtitleSelectionMode: SubtitleSelectionMode = SubtitleSelectionMode.DialogueOnly
) {
companion object {
fun default() = LanguagePreference(
preferredAudio = listOf("eng", "nor", "jpn"),
preferredSubtitles = listOf("eng", "nor"),
preferOriginal = true,
avoidDub = true,
subtitleFormatPriority = listOf("ass", "srt", "vtt", "smi"),
subtitleSelectionMode = SubtitleSelectionMode.DialogueOnly
)
}
}
enum class SubtitleSelectionMode {
DialogueOnly, // Kun dialog
DialogueAndForced, // Dialog + forced
All // Alle typer
}
data class ProcesserPreference(
val videoPreference: VideoPreference? = null,
val audioPreference: AudioPreference? = null
) {
companion object {
fun default(): ProcesserPreference {
return ProcesserPreference(
videoPreference = VideoPreference(VideoCodec.Hevc(), false),
audioPreference = AudioPreference(AudioCodec.Aac())
)
}
}
}
data class VideoPreference(
val codec: VideoCodec,
val enforceMkv: Boolean = false
)
data class AudioPreference(
val codec: AudioCodec
)
// ------------------------------------------------------------
// PREFERENCE COMPONENT
// ------------------------------------------------------------
@Component
class Preference(private val coordinatorEnv: CoordinatorEnv) {
private val gson = Gson()
private val lock = Any()
/**
* Leser hele configen, men bevarer ukjente felter.
*/
fun getFullConfig(): PeferenceConfig {
val file = coordinatorEnv.preference
if (!file.exists()) {
val default = PeferenceConfig(
processer = ProcesserPreference.default(),
language = LanguagePreference.default()
)
writeJsonObject(JsonObject().apply {
add("processer", gson.toJsonTree(default.processer))
add("language", gson.toJsonTree(default.language))
}, file)
return default
}
val root = try {
JsonParser.parseString(file.readText()).asJsonObject
} catch (e: Exception) {
return PeferenceConfig(
processer = ProcesserPreference.default(),
language = LanguagePreference.default()
)
}
val processer = try {
gson.fromJson(root.get("processer"), ProcesserPreference::class.java)
?: ProcesserPreference.default()
} catch (_: Exception) {
ProcesserPreference.default()
}
val language = try {
gson.fromJson(root.get("language"), LanguagePreference::class.java)
?: LanguagePreference.default()
} catch (_: Exception) {
LanguagePreference.default()
}
return PeferenceConfig(processer, language)
}
fun getProcesserPreference(): ProcesserPreference =
getFullConfig().processer
fun getLanguagePreference(): LanguagePreference =
getFullConfig().language
/**
* Oppdaterer kun processer-delen og bevarer resten av JSON.
*/
fun saveProcesserPreference(pref: ProcesserPreference) {
val file = coordinatorEnv.preference
synchronized(lock) {
val root = readOrEmpty(file)
root.add("processer", gson.toJsonTree(pref))
writeJsonObject(root, file)
}
}
/**
* Oppdaterer kun language-delen og bevarer resten av JSON.
*/
fun saveLanguagePreference(pref: LanguagePreference) {
val file = coordinatorEnv.preference
synchronized(lock) {
val root = readOrEmpty(file)
root.add("language", gson.toJsonTree(pref))
writeJsonObject(root, file)
}
}
/**
* Overskriver hele configen (brukes hvis FE sender alt).
*/
fun saveFullConfig(cfg: PeferenceConfig) {
val file = coordinatorEnv.preference
synchronized(lock) {
val root = JsonObject()
root.add("processer", gson.toJsonTree(cfg.processer))
root.add("language", gson.toJsonTree(cfg.language))
writeJsonObject(root, file)
}
}
private fun readOrEmpty(file: File): JsonObject =
if (file.exists()) {
try {
JsonParser.parseString(file.readText()).asJsonObject
} catch (_: Exception) {
JsonObject()
}
} else JsonObject()
private fun writeJsonObject(obj: JsonObject, file: File) {
try {
file.parentFile?.mkdirs()
file.writeText(gson.toJson(obj))
} catch (e: IOException) {
throw RuntimeException("Failed to write preference file", e)
}
}
}

View File

@ -1,23 +0,0 @@
package no.iktdev.mediaprocessing.coordinator
import org.springframework.stereotype.Component
import org.springframework.web.reactive.function.client.WebClient
import reactor.core.publisher.Mono
@Component
class ProcesserClient(
private val processerWebClient: WebClient
) {
fun fetchLog(path: String): Mono<String> =
processerWebClient.get()
.uri { it.path("/state/log").queryParam("path", path).build() }
.retrieve()
.bodyToMono(String::class.java)
fun ping(): Mono<String> =
processerWebClient.get()
.uri("/actuator/health")
.retrieve()
.bodyToMono(String::class.java)
}

View File

@ -1,24 +0,0 @@
package no.iktdev.mediaprocessing.coordinator
import org.springframework.boot.web.client.RestTemplateBuilder
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.web.client.RestTemplate
@Configuration
class RestTemplateConfig {
@Configuration
class RestTemplateConfig(
private val coordinatorEnv: CoordinatorEnv
) {
@Bean
fun streamitRestTemplate(): RestTemplate {
return RestTemplateBuilder()
.rootUri(coordinatorEnv.streamitAddress)
.build()
}
}
}

View File

@ -1,74 +0,0 @@
package no.iktdev.mediaprocessing.coordinator
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskPollerImplementation
import no.iktdev.eventi.tasks.TaskReporter
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.boot.ApplicationArguments
import org.springframework.boot.ApplicationRunner
import org.springframework.stereotype.Component
import org.springframework.stereotype.Service
import java.util.*
@Component
class TaskPollerAdministrator(
private val taskPoller: TaskPoller,
): ApplicationRunner {
override fun run(args: ApplicationArguments?) {
CoroutineScope(Dispatchers.Default).launch {
taskPoller.start()
}
}
}
@Service
class TaskPoller(
private val reporter: TaskReporter,
) : TaskPollerImplementation(
taskStore = TaskStore,
reporterFactory = { reporter } // én reporter brukes for alle tasks
) {
}
@Component
class DefaultTaskReporter() : TaskReporter {
override fun markClaimed(taskId: UUID, workerId: String) {
TaskStore.claim(taskId, workerId)
}
override fun updateLastSeen(taskId: UUID) {
TaskStore.heartbeat(taskId)
}
override fun markCompleted(taskId: UUID) {
TaskStore.markConsumed(taskId, TaskStatus.Completed)
}
override fun markFailed(referenceId: UUID, taskId: UUID) {
TaskStore.markConsumed(taskId, TaskStatus.Failed)
}
override fun markCancelled(referenceId: UUID, taskId: UUID) {
TaskStore.markConsumed(taskId, TaskStatus.Cancelled)
}
override fun updateProgress(taskId: UUID, progress: Int) {
// Not to be implemented for this application
}
override fun log(taskId: UUID, message: String) {
// Not to be implemented for this application
}
override fun publishEvent(event: Event) {
EventStore.persist(event)
}
}

View File

@ -1,8 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.config
import org.springframework.boot.context.properties.ConfigurationProperties
@ConfigurationProperties(prefix = "executables")
data class ExecutablesConfig(
val ffprobe: String
)

View File

@ -1,12 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.config
import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.web.reactive.function.client.WebClient
@ConfigurationProperties(prefix = "processer")
data class ProcesserClientProperties(
val baseUrl: String
)

View File

@ -1,21 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.config
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.web.reactive.function.client.WebClient
@Configuration
class WebClients(
private val processerClientProperties: ProcesserClientProperties
) {
@Bean
fun webClient(): WebClient.Builder =
WebClient
.builder()
.codecs { it.defaultCodecs().maxInMemorySize(10 * 1024 * 1024) }
@Bean
fun processerWebClient(builder: WebClient.Builder): WebClient {
return builder.baseUrl(processerClientProperties.baseUrl).build()
}
}

View File

@ -0,0 +1,31 @@
package no.iktdev.mediaprocessing.coordinator.controller
import com.google.gson.Gson
import no.iktdev.mediaprocessing.coordinator.Coordinator
import no.iktdev.mediaprocessing.shared.common.contract.dto.RequestWorkProceed
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.http.HttpStatus
import org.springframework.http.ResponseEntity
import org.springframework.stereotype.Controller
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
@Controller
@RequestMapping(path = ["/action"])
class ActionEventController(@Autowired var coordinator: Coordinator, @Autowired var eventsManager: EventsManager) {
@RequestMapping("/flow/proceed")
fun permitRunOnSequence(@RequestBody data: RequestWorkProceed): ResponseEntity<String> {
val set = eventsManager.getEventsWith(data.referenceId)
if (set.isEmpty()) {
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(data))
}
coordinator.permitWorkToProceedOn(data.referenceId, set, "Requested by ${data.source}")
//EVENT_MEDIA_WORK_PROCEED_PERMITTED("event:media-work-proceed:permitted")
return ResponseEntity.ok(null)
}
}

View File

@ -1,43 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.eventi.models.store.PersistedEvent
import no.iktdev.mediaprocessing.coordinator.services.EventService
import no.iktdev.mediaprocessing.shared.common.dto.EventQuery
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
import no.iktdev.mediaprocessing.shared.common.dto.SequenceEvent
import org.springframework.web.bind.annotation.*
import java.util.*
@RestController
@RequestMapping("/events")
class EventsController(
private val paging: EventService
) {
@GetMapping()
fun getEvents(query: EventQuery): Paginated<PersistedEvent> {
return paging.getEvents(query)
}
@GetMapping("/sequence/{referenceId}")
fun getEventSequence(
@PathVariable referenceId: UUID,
@RequestParam(required = false) beforeEventId: UUID?,
@RequestParam(required = false) afterEventId: UUID?,
@RequestParam(defaultValue = "50") limit: Int
): List<SequenceEvent> {
return paging.getPagedEvents(
referenceId = referenceId,
beforeEventId = beforeEventId,
afterEventId = afterEventId,
limit = limit
)
}
@GetMapping("/history/{referenceId}/effective")
fun getEffectiveHistory(
@PathVariable referenceId: UUID,
): List<PersistedEvent> {
return paging.getEffectiveHistory(referenceId)
}
}

View File

@ -1,19 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.shared.common.dto.FileTableItem
import no.iktdev.mediaprocessing.shared.database.queries.FilesTableQueries
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("/files")
class FilesController {
@GetMapping()
fun getFilesInDatabase(): ResponseEntity<List<FileTableItem>?>? {
val files = FilesTableQueries().getFiles()
return ResponseEntity.ok(files)
}
}

View File

@ -1,27 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
import no.iktdev.mediaprocessing.coordinator.services.CoordinatorHealthService
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("/health")
class HealthController(
private val healthService: CoordinatorHealthService
) {
@GetMapping
fun getHealth(): CoordinatorHealth {
return healthService.getHealth()
}
@GetMapping("/events")
fun getEventRate(): EventRate = healthService.getEventRate()
@GetMapping("/storage")
fun getDiskStatus(): List<DiskInfo> = healthService.getDiskHealth()
}

View File

@ -1,34 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.CoordinatorService
import no.iktdev.mediaprocessing.coordinator.services.SseHub
import no.iktdev.mediaprocessing.shared.common.model.ProgressUpdate
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.*
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter
@RestController
@RequestMapping("/internal")
class InternalProcesserController(
private val coordinator: CoordinatorService,
private val hub: SseHub
) {
@PostMapping("/progress")
fun receiveProgress(@RequestBody update: ProgressUpdate): ResponseEntity<Void> {
coordinator.updateProgress(update)
hub.broadcast("progress", update)
return ResponseEntity.ok().build()
}
@GetMapping("/progress")
fun getAllProgress(): List<ProgressUpdate> {
return coordinator.getProgress()
}
@GetMapping("/sse")
fun stream(): SseEmitter {
return hub.createEmitter()
}
}

View File

@ -1,21 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.ProcesserClient
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RequestParam
import org.springframework.web.bind.annotation.RestController
import reactor.core.publisher.Mono
@RestController
@RequestMapping("/log")
class LogController(
private val processerClient: ProcesserClient
) {
@GetMapping
fun getLog(@RequestParam path: String): Mono<String> {
return processerClient.fetchLog(path)
}
}

View File

@ -1,42 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.services.CommandService
import no.iktdev.mediaprocessing.shared.common.dto.requests.StartProcessRequest
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("/operations")
class OperationsController(
private val commandService: CommandService
) {
@PostMapping("/start")
fun startProcess(@RequestBody req: StartProcessRequest): ResponseEntity<Map<String, String>> {
return when (val result = commandService.startProcess(req)) {
is CommandService.StartResult.Accepted -> ResponseEntity
.accepted()
.body(
mapOf(
"referenceId" to result.referenceId.toString(),
"status" to "accepted",
"message" to "Process accepted and StartedEvent created"
)
)
is CommandService.StartResult.Rejected -> ResponseEntity
.badRequest()
.body(
mapOf(
"status" to "rejected",
"message" to result.reason
)
)
}
}
}

View File

@ -0,0 +1,18 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.Coordinator
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Controller
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
@Controller
@RequestMapping(path = ["/polls"])
class PollController(@Autowired var coordinator: Coordinator) {
@GetMapping()
fun polls(): String {
val stat = coordinator.getActivePolls()
return "Active Polls ${stat.active}/${stat.total}"
}
}

View File

@ -1,34 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.PeferenceConfig
import no.iktdev.mediaprocessing.coordinator.Preference
import no.iktdev.mediaprocessing.coordinator.ProcesserPreference
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.*
@RestController
@RequestMapping("/preference")
class PreferenceController(
private val preference: Preference
) {
@GetMapping
fun getFull(): ResponseEntity<PeferenceConfig> =
ResponseEntity.ok(preference.getFullConfig())
@PutMapping
fun putFull(@RequestBody body: PeferenceConfig): ResponseEntity<PeferenceConfig> {
preference.saveFullConfig(body)
return ResponseEntity.ok(preference.getFullConfig())
}
@GetMapping("/processer")
fun getProcesser(): ResponseEntity<ProcesserPreference> =
ResponseEntity.ok(preference.getProcesserPreference())
@PutMapping("/processer")
fun putProcesser(@RequestBody body: ProcesserPreference): ResponseEntity<ProcesserPreference> {
preference.saveProcesserPreference(body)
return ResponseEntity.ok(preference.getProcesserPreference())
}
}

View File

@ -1,28 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import org.springframework.boot.actuate.health.HealthEndpoint
import org.springframework.boot.actuate.health.Status
import org.springframework.http.HttpStatus
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("/system")
class ReadinessController(
private val healthEndpoint: HealthEndpoint
) {
@GetMapping("/ready")
fun ready(): ResponseEntity<String> {
val health = healthEndpoint.health()
return if (health.status == Status.UP) {
ResponseEntity.ok("READY")
} else {
ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
.body("NOT_READY: ${health.status}")
}
}
}

View File

@ -0,0 +1,78 @@
package no.iktdev.mediaprocessing.coordinator.controller
import com.google.gson.Gson
import no.iktdev.mediaprocessing.coordinator.Coordinator
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventRequest
import no.iktdev.mediaprocessing.shared.common.contract.dto.StartOperationEvents
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.http.HttpStatus
import org.springframework.http.ResponseEntity
import org.springframework.stereotype.Controller
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.ResponseStatus
import java.io.File
@Controller
@RequestMapping(path = ["/request"])
class RequestEventController(@Autowired var coordinator: Coordinator) {
@PostMapping("/convert")
@ResponseStatus(HttpStatus.OK)
fun requestConvert(@RequestBody payload: String): ResponseEntity<String> {
var convert: EventRequest? = null
var referenceId: String?
try {
convert = Gson().fromJson(payload, EventRequest::class.java)
val file = File(convert.file)
if (!file.exists()) {
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(convert.file)
}
referenceId = coordinator.startProcess(file, ProcessType.FLOW, listOf(StartOperationEvents.CONVERT)).toString()
} catch (e: Exception) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(convert))
}
return ResponseEntity.ok(referenceId)
}
@PostMapping("/extract")
@ResponseStatus(HttpStatus.OK)
fun requestExtract(@RequestBody payload: String): ResponseEntity<String> {
var request: EventRequest? = null
var referenceId: String?
try {
request = Gson().fromJson(payload, EventRequest::class.java)
val file = File(request.file)
if (!file.exists()) {
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload)
}
referenceId = coordinator.startProcess(file, ProcessType.MANUAL, listOf(StartOperationEvents.EXTRACT)).toString()
} catch (e: Exception) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(payload)
}
return ResponseEntity.ok(referenceId)
}
@PostMapping("/all")
@ResponseStatus(HttpStatus.OK)
fun requestAll(@RequestBody payload: String): ResponseEntity<String> {
var request: EventRequest? = null
var referenceId: String?
try {
request = Gson().fromJson(payload, EventRequest::class.java)
val file = File(request.file)
if (!file.exists()) {
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload)
}
referenceId = coordinator.startProcess(file, type = ProcessType.MANUAL).toString()
} catch (e: Exception) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(payload)
}
return ResponseEntity.ok(referenceId)
}
}

View File

@ -1,84 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.dto.translate.ApiResponse
import no.iktdev.mediaprocessing.coordinator.services.SequenceAggregatorService
import no.iktdev.mediaprocessing.shared.common.dto.SequenceSummary
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
import org.springframework.http.HttpStatus
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.*
import java.util.*
@RestController
@RequestMapping("/sequences")
class SequenceController(
private val aggregator: SequenceAggregatorService
) {
@GetMapping("/active")
fun getActive(): List<SequenceSummary> {
return aggregator.getActiveSequences()
}
@GetMapping("/recent")
fun getRecent(
@RequestParam(defaultValue = "15") limit: Int
): List<SequenceSummary> {
return aggregator.getRecentSequences(limit)
}
@PostMapping("/{referenceId}/continue")
fun continueSequence(
@PathVariable referenceId: UUID
): ResponseEntity<ApiResponse> {
return try {
val id = EventStore.createManuallyContinueEvent(referenceId)
ResponseEntity.ok(
ApiResponse(
ok = true,
message = "Sequence continued, event $id created!"
)
)
} catch (ex: Exception) {
ResponseEntity
.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(
ApiResponse(
ok = false,
message = ex.message ?: "Unknown error"
)
)
}
}
@PostMapping("/{referenceId}/delete")
fun deleteSequences(
@PathVariable referenceId: UUID
): ResponseEntity<ApiResponse> {
return try {
val id = EventStore.deleteSequence(referenceId)
ResponseEntity.ok(
ApiResponse(
ok = true,
message = "Sequence deleted, Event id for deletion marking is $id"
)
)
} catch (ex: Exception) {
ResponseEntity
.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(
ApiResponse(
ok = false,
message = ex.message ?: "Unknown error"
)
)
}
}
}

View File

@ -1,91 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.controller
import no.iktdev.mediaprocessing.coordinator.dto.translate.CoordinatorTaskTransferDto
import no.iktdev.mediaprocessing.coordinator.dto.translate.toCoordinatorTransferDto
import no.iktdev.mediaprocessing.coordinator.services.EventService
import no.iktdev.mediaprocessing.coordinator.services.TaskService
import no.iktdev.mediaprocessing.ffmpeg.util.UtcNow
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
import no.iktdev.mediaprocessing.shared.common.dto.ResetTaskResponse
import no.iktdev.mediaprocessing.shared.common.dto.TaskQuery
import no.iktdev.mediaprocessing.shared.common.dto.map
import org.springframework.http.HttpStatus
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.PathVariable
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
import java.util.*
@RestController
@RequestMapping("/tasks")
class TaskController(
private val taskService: TaskService,
private val eventService: EventService
) {
@GetMapping("/active")
fun getActiveTasks(): List<CoordinatorTaskTransferDto> {
val tasks = taskService.getActiveTasks()
val logEvents = eventService.getTaskEventResultsWithLogs(tasks.map { it.referenceId }.toSet())
return tasks.map { it.toCoordinatorTransferDto(logEvents) }
}
@GetMapping
fun getPagedTasks(query: TaskQuery): Paginated<CoordinatorTaskTransferDto> {
val paginatedTasks = taskService.getPagedTasks(query)
val logEvents = eventService.getTaskEventResultsWithLogs(paginatedTasks.items.map { it.referenceId }.toSet())
return paginatedTasks.map { it.toCoordinatorTransferDto(logEvents) }
}
@GetMapping("/{id}")
fun getTask(@PathVariable id: UUID): CoordinatorTaskTransferDto? {
val tasks = taskService.getTaskById(id) ?: return null
val logEvents = eventService.getTaskEventResultsWithLogs(setOf(tasks.referenceId))
return tasks.toCoordinatorTransferDto(logEvents)
}
@GetMapping("/{taskId}/reset")
fun resetTask(@PathVariable taskId: UUID, forced: Boolean = false): ResponseEntity<ResetTaskResponse> {
val task = taskService.getTaskById(taskId)
?: return ResponseEntity.notFound().build()
val referenceId = task.referenceId
if (eventService.isSequenceDeleted(referenceId)) {
return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build()
}
// 1. Opprett DeleteEvent
val deletedId = eventService.deleteTaskFailureForReset(referenceId, taskId)
if (deletedId == null) {
if (forced) {
eventService.createForcedTaskResetAuditEvent(referenceId, taskId)
} else {
return ResponseEntity.status(HttpStatus.CONFLICT).build()
}
}
// 2. Reset task
val success = taskService.resetFailedTask(taskId)
return ResponseEntity.ok(
ResetTaskResponse(
taskId = taskId,
referenceId = referenceId,
reset = success,
deletedEventId = deletedId,
resetAt = UtcNow()
)
)
}
@GetMapping("/{taskId}/reset/force")
fun resetTaskForce(@PathVariable taskId: UUID): ResponseEntity<ResetTaskResponse> {
return resetTask(taskId, true)
}
}

View File

@ -0,0 +1,11 @@
package no.iktdev.mediaprocessing.coordinator.coordination
/**
* Class to handle messages from websockets, produced by Processer instances.
* This is due to keep a overview of progress by processer
*/
class ProcesserSocketMessageListener {
}

View File

@ -1,9 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.dto
import java.util.*
data class LogAssociatedIds(
val referenceId: UUID,
val ids: Set<UUID>,
val logFile: String
)

View File

@ -1,26 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.dto.health
import java.time.Instant
data class CoordinatorHealth(
val status: CoordinatorHealthStatus,
val abandonedTasks: Int,
val stalledTasks: Int,
val activeTasks: Int,
val queuedTasks: Int,
val failedTasks: Int,
val sequencesOnHold: Int,
val lastActivity: Instant?,
// IDs for UI linking
val abandonedTaskIds: List<String>,
val stalledTaskIds: List<String>,
val sequencesOnHoldIds: List<String>,
val overdueSequenceIds: List<String>,
// Detailed sequence info
val overdueSequences: List<SequenceHealth>,
val details: Map<String, Any?>
)

View File

@ -1,7 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.dto.health
enum class CoordinatorHealthStatus {
HEALTHY,
DEGRADED,
UNHEALTHY
}

View File

@ -1,20 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.dto.health
import java.time.Duration
import java.time.Instant
data class SequenceHealth(
val referenceId: String,
val age: Duration,
val expected: Duration,
val lastEventAt: Instant,
val eventCount: Int,
// nye felter
val startTime: Instant,
val expectedFinishTime: Instant,
val overdueDuration: Duration,
val isOverdue: Boolean
)

View File

@ -1,6 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.dto.rate
data class EventRate(
val lastMinute: Long,
val lastFiveMinutes: Long
)

View File

@ -1,6 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.dto.translate
data class ApiResponse(
val ok: Boolean,
val message: String
)

View File

@ -1,46 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.dto.translate
import no.iktdev.eventi.models.store.PersistedTask
import no.iktdev.mediaprocessing.coordinator.dto.LogAssociatedIds
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
import java.time.Instant
import java.util.*
data class CoordinatorTaskTransferDto(
val id: Long,
val referenceId: UUID,
val status: String,
val taskId: UUID,
val task: String,
val data: String,
val claimed: Boolean,
val claimedBy: String?,
val consumed: Boolean,
val lastCheckIn: Instant?,
val persistedAt: Instant,
val logs: List<String> = emptyList(),
val abandoned: Boolean,
) {
}
fun PersistedTask.toCoordinatorTransferDto(logs: List<LogAssociatedIds>): CoordinatorTaskTransferDto {
val matchingLogs = logs
.filter { log -> log.ids.contains(taskId) }
.map { it.logFile }
return CoordinatorTaskTransferDto(
id = id,
referenceId = referenceId,
status = status.name,
taskId = taskId,
task = task,
data = data,
claimed = claimed,
claimedBy = claimedBy,
consumed = consumed,
lastCheckIn = lastCheckIn,
persistedAt = persistedAt,
logs = matchingLogs,
abandoned = TaskLifecycleRules.isAbandoned(consumed, persistedAt, lastCheckIn)
)
}

View File

@ -1,33 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import mu.KotlinLogging
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
import no.iktdev.mediaprocessing.shared.common.listeners.SummaryEventListener
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
import org.springframework.stereotype.Component
@Component
class CollectEventsListener(eventStore: no.iktdev.eventi.stores.EventStore = EventStore) : SummaryEventListener(eventStore) {
private val log = KotlinLogging.logger {}
override fun shouldSummarize(fullHistory: List<Event>): Boolean {
val projection = CollectProjection(fullHistory)
if (projection.startedWith == null) return false
if (!projection.isWorkflowComplete()) return false
return true
}
override fun produceSummary(fullHistory: List<Event>): Event {
// Must have all relevant tasks completed
val eventIds = fullHistory.map { it.eventId }.toSet()
return CollectedEvent(eventIds).derivedOf(fullHistory.last())
}
override fun summaryAlreadyExists(fullHistory: List<Event>): Boolean {
return fullHistory.any { it is CollectedEvent }
}
}

View File

@ -1,31 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import mu.KotlinLogging
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CompletedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent
import no.iktdev.mediaprocessing.shared.common.getName
import org.springframework.stereotype.Component
@Component
class CompletedListener: EventListener() {
val log = KotlinLogging.logger {}
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
if (event !is StoreContentAndMetadataTaskResultEvent)
return null
val useEvent = event as StoreContentAndMetadataTaskResultEvent
if (useEvent.status != TaskStatus.Completed) {
log.info { "${useEvent.referenceId} - ${MigrateContentToStoreTaskResultEvent::class.getName()} is failed, thus no task will be created" }
return null
}
return CompletedEvent().derivedOf(event)
}
}

View File

@ -1,60 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import mu.KotlinLogging
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskCreatedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Component
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
@Component
class MediaCreateConvertTaskListener: EventListener() {
private val log = KotlinLogging.logger {}
fun allowOverwrite(): Boolean {
return true
}
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
if (startedEvent.data.operation.isNotEmpty()) {
if (!startedEvent.data.operation.contains(OperationType.ConvertSubtitles))
return null
}
val selectedEvent = event as? ProcesserExtractResultEvent ?: return null
if (selectedEvent.status != TaskStatus.Completed)
return null
val result = selectedEvent.data ?: return null
if (!Files.exists(Path.of(result.cachedOutputFile)))
return null
val useFile = File(result.cachedOutputFile)
val convertTask = ConvertTask(
data = ConvertTask.Data(
inputFile = result.cachedOutputFile,
language = result.language,
allowOverwrite = allowOverwrite(),
outputDirectory = useFile.parentFile.absolutePath,
outputFileName = useFile.nameWithoutExtension,
)
).derivedOf(event)
TaskStore.persist(convertTask)
return ConvertTaskCreatedEvent(convertTask.taskId).derivedOf(event)
}
}

View File

@ -1,49 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import mu.KotlinLogging
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadTaskCreatedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Component
@Component
class MediaCreateCoverDownloadTaskListener: EventListener() {
private val log = KotlinLogging.logger {}
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val useEvent = event as? MetadataSearchResultEvent ?: return null
if (useEvent.status != TaskStatus.Completed) {
log.warn("MetadataResult on ${event.referenceId} did not complete successfully")
return null
}
val downloadData = useEvent.results.map {
val data = it.metadata
val outputFileName = "${data.title}-${data.source}"
CoverDownloadTask.CoverDownloadData(
url = it.metadata.cover,
source = it.metadata.source,
outputFileName = outputFileName
)
}
val downloadTasks = downloadData.map {
CoverDownloadTask(it)
.derivedOf(useEvent)
}
downloadTasks.forEach { TaskStore.persist(it) }
return CoverDownloadTaskCreatedEvent(
downloadTasks.map { it.taskId }
).derivedOf(event)
}
}

View File

@ -1,73 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.coordinator.Preference
import no.iktdev.mediaprocessing.ffmpeg.dsl.*
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeData
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Component
import java.io.File
@Component
class MediaCreateEncodeTaskListener(
private val preference: Preference
) : EventListener() {
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val preference = preference.getProcesserPreference()
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
if (startedEvent.data.operation.isNotEmpty()) {
if (!startedEvent.data.operation.contains(OperationType.Encode))
return null
}
val selectedEvent = event as? MediaTracksEncodeSelectedEvent ?: return null
val streams = history.filterIsInstance<MediaStreamParsedEvent>().firstOrNull()?.data ?: return null
val videoPreference = preference.videoPreference?.codec ?: VideoCodec.Hevc()
val audioPreference = preference.audioPreference?.codec ?: AudioCodec.Aac(channels = 2)
val audioTargets = mutableListOf<AudioTarget>(
AudioTarget(
index = selectedEvent.selectedAudioTrack,
codec = audioPreference
)
)
selectedEvent.selectedAudioExtendedTrack?.let {
audioTargets.add(AudioTarget(
index = it,
codec = audioPreference
))
}
val plan = MediaPlan(
videoTrack = VideoTarget(index = selectedEvent.selectedVideoTrack, codec = videoPreference),
audioTracks = audioTargets
)
val args = plan.toFfmpegArgs(streams.videoStream, streams.audioStream)
val filename = startedEvent.data.fileUri.let { File(it) }.nameWithoutExtension
val extension = plan.toContainer()
val task = EncodeTask(
data = EncodeData(
arguments = args,
outputFileName = "$filename.$extension",
inputFile = startedEvent.data.fileUri
)
).derivedOf(event)
TaskStore.persist(task)
return ProcesserEncodeTaskCreatedEvent(
taskCreated = task.taskId
).derivedOf(event)
}
}

View File

@ -1,79 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
import no.iktdev.mediaprocessing.ffmpeg.dsl.SubtitleCodec
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleData
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleTask
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Component
import java.io.File
import java.util.*
@Component
class MediaCreateExtractTaskListener(): EventListener() {
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
if (startedEvent.data.operation.isNotEmpty()) {
if (!startedEvent.data.operation.contains(OperationType.ExtractSubtitles))
return null
}
val selectedEvent = event as? MediaTracksExtractSelectedEvent ?: return null
val streams = history.filterIsInstance<MediaStreamParsedEvent>().firstOrNull()?.data ?: return null
val selectedStreams: Map<Int, SubtitleStream> =
selectedEvent.selectedSubtitleTracks.mapNotNull { streamIndex ->
val stream = streams.subtitleStream.firstOrNull { it.index == streamIndex }
stream?.let { streams.subtitleStream.indexOf(it) to it }
}.toMap()
val entries = selectedStreams.mapNotNull { (idx, stream )->
toSubtitleArgumentData(idx, startedEvent.data.fileUri.let { File(it) }, stream)
}
val createdTaskIds: MutableList<UUID> = mutableListOf()
entries.forEach { entry ->
ExtractSubtitleTask(data = entry).derivedOf(event).also {
TaskStore.persist(it)
createdTaskIds.add(it.taskId)
}
}
return ProcesserExtractTaskCreatedEvent(
tasksCreated = createdTaskIds
).derivedOf(event)
}
fun toSubtitleArgumentData(index: Int, inputFile: File, stream: SubtitleStream): ExtractSubtitleData? {
val codec = SubtitleCodec.getCodec(stream.codec_name) ?: return null
val extension = codec.getExtension()
// ffmpeg-args for å mappe og copy akkurat dette subtitle-sporet
val args = mutableListOf<String>()
args += listOf("-map", "0:s:$index")
args += codec.buildFfmpegArgs(stream)
val language = stream.tags.language?: return null
// outputfilnavn basert på index og extension
val outputFileName = "${inputFile.nameWithoutExtension}-${language}.${extension}"
return ExtractSubtitleData(
inputFile = inputFile.absolutePath,
arguments = args,
outputFileName = outputFileName,
language = language
)
}
}

View File

@ -1,84 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.ListenerOrder
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MetadataSearchTask
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.jetbrains.annotations.VisibleForTesting
import org.springframework.stereotype.Component
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit
@Component
@ListenerOrder(5)
class MediaCreateMetadataSearchTaskListener: EventListener() {
@VisibleForTesting
internal val scheduledExpiries = ConcurrentHashMap<UUID, ScheduledFuture<*>>()
private val scheduler = Executors.newScheduledThreadPool(1)
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
if (startedEvent.data.operation.isNotEmpty()) {
if (!startedEvent.data.operation.contains(OperationType.MetadataSearch))
return null
}
// For replay
if (event is MetadataSearchTaskCreatedEvent) {
val hasResult = history.filter { it is MetadataSearchResultEvent }
.any { it.metadata.derivedFromId?.contains(event.taskId) == true }
if (!hasResult) {
scheduleTaskExpiry(event.taskId, event.eventId, event.referenceId)
}
} else if (event is MetadataSearchResultEvent) {
val cancelKeys = event.metadata.derivedFromId ?: emptySet()
scheduledExpiries.filter { it -> it.key in cancelKeys }.keys.forEach { key ->
scheduledExpiries.remove(key)?.cancel(true)
}
return null
}
val useEvent = event as? MediaParsedInfoEvent ?: return null
val task = MetadataSearchTask(
MetadataSearchTask.SearchData(
searchTitles = useEvent.data.parsedSearchTitles,
collection = useEvent.data.parsedCollection
)
).derivedOf(useEvent)
TaskStore.persist(task)
val finalResult = MetadataSearchTaskCreatedEvent(task.taskId).derivedOf(useEvent)
scheduleTaskExpiry(task.taskId, finalResult.eventId, task.referenceId)
return finalResult
}
private fun scheduleTaskExpiry(taskId: UUID, eventId: UUID, referenceId: UUID) {
if (scheduledExpiries.containsKey(taskId)) return
val future = scheduler.schedule({
// Hvis tasken fortsatt ikke har result/failed → marker som failed
TaskStore.claim(taskId, "Coordinator-MetadataSearchTaskListener-TimeoutScheduler")
TaskStore.markConsumed(taskId, TaskStatus.Failed)
val failureEvent = MetadataSearchResultEvent(
status = TaskStatus.Failed,
).apply { setFailed(listOf(taskId)) }
//publishEvent(MetadataSearchFailedEvent(taskId, "Timeout").derivedOf(referenceId))
scheduledExpiries.remove(taskId)
}, 10, TimeUnit.MINUTES)
scheduledExpiries[taskId] = future
}
}

View File

@ -1,76 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent
import no.iktdev.mediaprocessing.shared.common.model.SubtitleItem
import no.iktdev.mediaprocessing.shared.common.model.SubtitleType
import org.springframework.stereotype.Component
@Component
class MediaDetermineSubtitleTrackTypeListener: EventListener() {
fun ignoreSHD(): Boolean = true
fun ignoreCC(): Boolean = true
fun ignoreSongs(): Boolean = true
fun ignoreCommentary(): Boolean = true
val supportedCodecs = setOf(
"ass", "subrip", "webvtt", "vtt", "smi"
)
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val useEvent = event as? MediaStreamParsedEvent ?: return null
val collected = useEvent.data.subtitleStream
.mapToType()
.excludeTypes()
.onlySupportedCodecs()
return MediaTracksDetermineSubtitleTypeEvent(
subtitleTrackItems = collected
).derivedOf(event)
}
fun getCommentaryFilters(): Set<String> = setOf("commentary", "comentary", "kommentar", "kommentaar")
fun getSongFilters(): Set<String> = setOf("song", "sign", "lyrics")
fun getClosedCaptionFilters(): Set<String> = setOf("closed caption", "cc", "close caption", "closed-caption", "cc.")
fun getSHDFilters(): Set<String> = setOf("shd", "hh", "hard of hearing", "hard-of-hearing")
private fun List<SubtitleItem>.excludeTypes(): List<SubtitleItem> {
return this.filter {
when (it.type) {
SubtitleType.Song -> !ignoreSongs()
SubtitleType.Commentary -> !ignoreCommentary()
SubtitleType.ClosedCaption -> !ignoreCC()
SubtitleType.SHD -> !ignoreSHD()
SubtitleType.Dialogue -> true
}
}
}
private fun List<SubtitleStream>.mapToType(): List<SubtitleItem> {
return this.map {
val title = it.tags.title?.lowercase() ?: ""
val type = when {
getCommentaryFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.Commentary
getSongFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.Song
getClosedCaptionFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.ClosedCaption
getSHDFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.SHD
else -> SubtitleType.Dialogue
}
SubtitleItem(it, type)
}
}
private fun List<SubtitleItem>.onlySupportedCodecs(): List<SubtitleItem> {
return this.filter { it.stream.codec_name in supportedCodecs }
}
}

View File

@ -1,82 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import com.google.gson.Gson
import com.google.gson.JsonObject
import mu.KotlinLogging
import no.iktdev.eventi.ListenerOrder
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream
import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
import org.springframework.stereotype.Component
@ListenerOrder(4)
@Component
class MediaParseStreamsListener: EventListener() {
val log = KotlinLogging.logger {}
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
if (event !is CoordinatorReadStreamsResultEvent) return null
if (event.status != TaskStatus.Completed)
return null
if (event.data == null) {
log.error { "No data to parse in CoordinatorReadStreamsResultEvent" }
return null
}
val streams = parseStreams(event.data)
return MediaStreamParsedEvent(
data = streams
).derivedOf(event)
}
fun parseStreams(data: JsonObject?): ParsedMediaStreams {
val ignoreCodecs = listOf("png", "mjpeg")
val gson = Gson()
return try {
val jStreams = data!!.getAsJsonArray("streams")
val videoStreams = mutableListOf<VideoStream>()
val audioStreams = mutableListOf<AudioStream>()
val subtitleStreams = mutableListOf<SubtitleStream>()
for (streamJson in jStreams) {
val streamObject = streamJson.asJsonObject
if (!streamObject.has("codec_name")) continue
val codecName = streamObject.get("codec_name").asString
val codecType = streamObject.get("codec_type").asString
if (codecName in ignoreCodecs) continue
when (codecType) {
"video" -> videoStreams.add(gson.fromJson(streamObject, VideoStream::class.java))
"audio" -> audioStreams.add(gson.fromJson(streamObject, AudioStream::class.java))
"subtitle" -> subtitleStreams.add(gson.fromJson(streamObject, SubtitleStream::class.java))
}
}
val parsedStreams = ParsedMediaStreams(
videoStream = videoStreams,
audioStream = audioStreams,
subtitleStream = subtitleStreams
)
parsedStreams
} catch (e: Exception) {
"Failed to parse data, its either not a valid json structure or expected and required fields are not present.".also {
log.error { it }
}
throw e
}
}
}

View File

@ -1,271 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.ListenerOrder
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
import no.iktdev.mediaprocessing.shared.common.model.MediaType
import org.springframework.stereotype.Component
import java.io.File
@ListenerOrder(2)
@Component
class MediaParsedInfoListener : EventListener() {
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val started = event as? StartProcessingEvent ?: return null
val file = File(started.data.fileUri)
val filename = file.guessDesiredFileName()
val collection = file.getDesiredCollection()
val searchTitles = file.guessSearchableTitle()
val mediaType = file.guessMovieOrSeries()
val episodeInfo = if (mediaType == MediaType.Serie) {
val serieInfo = file.guessSerieInfo()
MediaParsedInfoEvent.ParsedData.EpisodeInfo(
episodeNumber = serieInfo.episodeNumber,
seasonNumber = serieInfo.seasonNumber,
episodeTitle = serieInfo.episodeTitle,
)
} else null
return MediaParsedInfoEvent(
MediaParsedInfoEvent.ParsedData(
parsedFileName = filename,
parsedCollection = collection,
parsedSearchTitles = searchTitles,
mediaType = mediaType,
episodeInfo = episodeInfo,
)
).derivedOf(event)
}
fun String.getCleanedTitle(): String {
return this
.noBrackets()
.noParens()
.noResolutionAndAfter()
.noSourceTags()
.noYear()
.noDots()
.noTrailingOrLeading()
.noUnderscores()
.noExtraSpaces()
.trim()
}
fun String.noBrackets() = Regex("\\[.*?]").replace(this, " ")
fun String.noParens() = Regex("\\(.*?\\)").replace(this, " ")
fun String.noTrailingOrLeading() = Regex("^[^a-zA-Z0-9!,]+|[^a-zA-Z0-9!~,]+\$").replace(this, " ")
fun String.noResolutionAndAfter() = Regex("[0-9]+[pk].*", RegexOption.IGNORE_CASE).replace(this, "")
fun String.noSourceTags() =
Regex("(?i)(bluray|laserdisc|dvd|web|uhd|hd|htds|imax).*", RegexOption.IGNORE_CASE).replace(this, " ")
fun String.noUnderscores() = this.replace("_", " ")
fun String.noYear() = Regex("\\b\\d{4}\\b").replace(this.takeIf { !it.matches(Regex("^\\d{4}")) } ?: this, "")
fun String.noDots() = Regex("(?<!\\b(?:Dr|Mr|Ms|Mrs|Lt|Capt|Prof|St|Ave))\\.").replace(this, " ")
fun String.noExtraSpaces() = Regex("\\s{2,}").replace(this, " ")
fun String.fullTrim() = this.trim('.', ',', ' ', '_', '-')
fun File.guessMovieOrSeries(): MediaType {
val name = this.nameWithoutExtension.lowercase()
// Serie-mønstre: dekker alle vanlige shorthand og varianter
val seriesPatterns = listOf(
Regex("s\\d{1,2}e\\d{1,2}"), // S01E03, s1e5
Regex("\\d{1,2}x\\d{1,2}"), // 1x03, 2x10
Regex("season\\s*\\d+"), // Season 2
Regex("episode\\s*\\d+"), // Episode 5
Regex("ep\\s*\\d+"), // Ep05, Ep 5
Regex("s\\d{1,2}\\s*[- ]\\s*e\\d{1,2}"), // S1 - E5, S01 - E05
Regex("s\\d{1,2}\\s*ep\\s*\\d{1,2}"), // S1 Ep05
Regex("series\\s*\\d+"), // Series 2 (britisk stil)
Regex("s\\d{1,2}[. ]e\\d{1,2}") // S01.E02 eller S01 E02
)
if (seriesPatterns.any { it.containsMatchIn(name) }) {
return MediaType.Serie
}
// Film-mønstre: årstall (19002099) etter tittel
val moviePattern = Regex("\\b(19|20)\\d{2}\\b")
if (moviePattern.containsMatchIn(name)) {
return MediaType.Movie
}
// Fallback: hvis ingen mønstre passer, anta film
return MediaType.Movie
}
fun File.guessDesiredFileName(): String {
val type = this.guessMovieOrSeries()
return when (type) {
MediaType.Movie -> this.guessDesiredMovieTitle()
MediaType.Serie -> this.guessDesiredSerieTitle()
MediaType.Subtitle -> {
this.nameWithoutExtension
}
}
}
fun File.getDesiredCollection(): String {
val collection = when (this.guessMovieOrSeries()) {
MediaType.Movie -> this.guessDesiredMovieTitle()
MediaType.Serie -> this.guessDesiredSerieTitle()
MediaType.Subtitle -> {
this.parentFile.parentFile.nameWithoutExtension
}
}
return collection.noParens().noYear().split(" - ").first().trim()
}
/**
* @return A fully cleaned title suitable to use for collection
*/
fun File.guessDesiredMovieTitle(): String {
val cleaned = this.nameWithoutExtension.getCleanedTitle()
val yearRegex = Regex("\\b(19|20)\\d{2}\\b")
val yearMatch = yearRegex.find(cleaned)
return if (yearMatch != null) {
val title = cleaned.replace(yearRegex, "").trim()
"${title} (${yearMatch.value})"
} else {
cleaned
}
}
/**
* @return A fully cleaned title including season and episode with possible episode title
*/
fun File.guessDesiredSerieTitle(): String {
val parsedSerieInfo = this.guessSerieInfo()
val tag = buildString {
append("S${(parsedSerieInfo.seasonNumber ?: 1).toString().padStart(2, '0')}")
append("E${(parsedSerieInfo.episodeNumber ?: 1).toString().padStart(2, '0')}")
if (parsedSerieInfo.revision != null) append(" (v$parsedSerieInfo.revision)")
}
return buildString {
append(parsedSerieInfo.serieTitle)
append(" - ")
append(tag)
if (parsedSerieInfo.episodeTitle.isNotEmpty()) {
append(" - ")
append(parsedSerieInfo.episodeTitle)
}
}.trim()
}
fun File.guessSerieInfo(): ParsedSerieInfo {
val raw = this.nameWithoutExtension
val seasonRegex = Regex("""(?i)(?:S|Season|Series)\s*(\d{1,2})""")
val episodeRegex = Regex("""(?i)(?:E|Episode|Ep)\s*(\d{1,3})""")
val revisionRegex = Regex("""(?i)\bv(\d+)\b""")
val seasonEpisodeRegex = Regex("""(?i)(\d{1,2})x(\d{1,2})(?:[vV](\d+))?""")
var season: Int? = null
var episode: Int? = null
var revision: Int? = null
var baseTitle = raw.getCleanedTitle()
var episodeTitle = ""
val seMatch = seasonEpisodeRegex.find(raw)
if (seMatch != null) {
season = seMatch.groupValues[1].toIntOrNull()
episode = seMatch.groupValues[2].toIntOrNull()
revision = seMatch.groupValues.getOrNull(3)?.toIntOrNull()
baseTitle = raw.substring(0, seMatch.range.first).getCleanedTitle()
episodeTitle = raw.substring(seMatch.range.last + 1).getCleanedTitle()
} else {
val seasonMatch = seasonRegex.find(raw)
val episodeMatch = episodeRegex.find(raw)
val revisionMatch = revisionRegex.find(raw)
season = seasonMatch?.groupValues?.get(1)?.toIntOrNull()
episode = episodeMatch?.groupValues?.get(1)?.toIntOrNull()
revision = revisionMatch?.groupValues?.get(1)?.toIntOrNull()
baseTitle = if (seasonMatch != null) {
raw.substring(0, seasonMatch.range.first).getCleanedTitle()
} else raw.getCleanedTitle()
episodeTitle = if (episodeMatch != null) {
raw.substring(episodeMatch.range.last + 1).getCleanedTitle()
} else ""
}
// Fallback: hvis baseTitle er tom eller bare inneholder S/E, bruk parentmappe
if (baseTitle.isBlank() || baseTitle.matches(Regex("""(?i)^s?\d+e?\d+$"""))) {
baseTitle = this.parentFile?.name?.getCleanedTitle() ?: "Dumb ways to die"
}
return ParsedSerieInfo(
serieTitle = baseTitle,
episodeNumber = episode ?: 1,
seasonNumber = season ?: 1,
revision = revision,
episodeTitle
)
}
fun File.guessSearchableTitle(): List<String> {
val cleaned = this.guessDesiredFileName()
.noResolutionAndAfter()
.noSourceTags()
.noDots()
.noExtraSpaces()
.fullTrim()
val titles = mutableListOf<String>()
val yearRegex = Regex("""\b(19|20)\d{2}\b""")
val hasYear = yearRegex.containsMatchIn(cleaned)
// 1. Hvis årstall finnes, legg hele cleaned først
if (hasYear) {
titles.add(cleaned)
}
// 2. Første del før bindestrek
val firstPart = cleaned.split(" - ").firstOrNull()?.trim() ?: cleaned
titles.add(firstPart)
// 3. Hele cleaned (hvis ikke allerede lagt inn først)
if (!hasYear) {
titles.add(cleaned)
}
// 4. Variant uten årstall
val noYear = yearRegex.replace(cleaned, "")
.noParens().trim()
if (noYear.isNotEmpty() && noYear != cleaned) {
titles.add(noYear)
}
return titles.distinct()
}
data class ParsedSerieInfo(
val serieTitle: String,
val episodeNumber: Int,
val seasonNumber: Int,
val revision: Int? = null,
val episodeTitle: String,
)
}

View File

@ -1,33 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.ListenerOrder
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsTaskCreatedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Component
@ListenerOrder(3)
@Component
class MediaReadStreamsTaskCreatedListener: EventListener() {
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
if (event !is MediaParsedInfoEvent) return null
val startEvent = history.lastOrNull { it is StartProcessingEvent } as? StartProcessingEvent
?: return null
val readTask = MediaReadTask(
fileUri = startEvent.data.fileUri
).derivedOf(event)
TaskStore.persist(readTask)
return CoordinatorReadStreamsTaskCreatedEvent(readTask.taskId).derivedOf(event) // Create task instead of event
}
}

View File

@ -1,158 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.coordinator.Preference
import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream
import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksEncodeSelectedEvent
import org.springframework.stereotype.Component
@Component
class MediaSelectEncodeTracksListener(
private val preference: Preference
) : EventListener() {
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val useEvent = event as? MediaStreamParsedEvent ?: return null
val videoTrackIndex = getVideoTrackToUse(useEvent.data.videoStream)
val audioDefaultTrack = getAudioDefaultTrackToUse(useEvent.data.audioStream)
val audioExtendedTrack = getAudioExtendedTrackToUse(
useEvent.data.audioStream,
selectedDefaultTrack = audioDefaultTrack
)
return MediaTracksEncodeSelectedEvent(
selectedVideoTrack = videoTrackIndex,
selectedAudioTrack = audioDefaultTrack,
selectedAudioExtendedTrack = audioExtendedTrack
).derivedOf(event)
}
// ------------------------------------------------------------
// AUDIO SELECTION
// ------------------------------------------------------------
private fun getAudioDefaultTrackToUse(audioStreams: List<AudioStream>): Int {
val pref = preference.getLanguagePreference()
val selected = selectBestAudioStream(
streams = audioStreams,
preferredLanguages = pref.preferredAudio,
preferOriginal = pref.preferOriginal,
avoidDub = pref.avoidDub,
mode = AudioSelectMode.DEFAULT
) ?: audioStreams.firstOrNull()
return audioStreams.indexOf(selected)
}
private fun getAudioExtendedTrackToUse(
audioStreams: List<AudioStream>,
selectedDefaultTrack: Int
): Int? {
val pref = preference.getLanguagePreference()
val candidates = audioStreams
.filter { it.index != selectedDefaultTrack }
.filter { (it.duration_ts ?: 0) > 0 }
.filter { it.channels > 2 }
val selected = selectBestAudioStream(
streams = candidates,
preferredLanguages = pref.preferredAudio,
preferOriginal = pref.preferOriginal,
avoidDub = pref.avoidDub,
mode = AudioSelectMode.EXTENDED
) ?: return null
return audioStreams.indexOf(selected)
}
// ------------------------------------------------------------
// CORE AUDIO SELECTION LOGIC
// ------------------------------------------------------------
private enum class AudioSelectMode { DEFAULT, EXTENDED }
private fun selectBestAudioStream(
streams: List<AudioStream>,
preferredLanguages: List<String>,
preferOriginal: Boolean,
avoidDub: Boolean,
mode: AudioSelectMode
): AudioStream? {
if (streams.isEmpty()) return null
// 1. Originalspråk
if (preferOriginal) {
val originals = streams.filter { it.disposition.original == 1 }
if (originals.isNotEmpty()) {
return when (mode) {
AudioSelectMode.DEFAULT -> originals.minByOrNull { it.channels }
AudioSelectMode.EXTENDED -> originals.maxByOrNull { it.channels }
}
}
}
// 2. Filtrer bort dub
val filtered = if (avoidDub) {
streams.filter { it.disposition.dub != 1 }
} else streams
// 3. Foretrukne språk
for (lang in preferredLanguages) {
val match = filtered.filter {
it.tags.language?.equals(lang, ignoreCase = true) == true
}
if (match.isNotEmpty()) {
return when (mode) {
AudioSelectMode.DEFAULT -> match.minByOrNull { it.channels }
AudioSelectMode.EXTENDED -> match.maxByOrNull { it.channels }
}
}
}
// 4. Default-flagget
val default = filtered.firstOrNull { it.disposition.default == 1 }
if (default != null) return default
// 5. Fallback
return filtered.firstOrNull()
}
// ------------------------------------------------------------
// QUALITY SCORE (no bitrate)
// ------------------------------------------------------------
private fun qualityScore(s: AudioStream): Int {
val channelsScore = s.channels * 10
val bitsScore = s.bits_per_sample
val sampleRateScore = s.sample_rate.toIntOrNull()?.div(1000) ?: 0
return channelsScore + bitsScore + sampleRateScore
}
// ------------------------------------------------------------
// VIDEO SELECTION
// ------------------------------------------------------------
private fun getVideoTrackToUse(streams: List<VideoStream>): Int {
val selectStream = streams
.filter { (it.duration_ts ?: 0) > 0 }
.maxByOrNull { it.duration_ts ?: 0 }
?: streams.minByOrNull { it.index }
?: throw Exception("No video streams found")
return streams.indexOf(selectStream)
}
}

View File

@ -1,130 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.coordinator.Preference
import no.iktdev.mediaprocessing.coordinator.SubtitleSelectionMode
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksExtractSelectedEvent
import no.iktdev.mediaprocessing.shared.common.model.SubtitleItem
import no.iktdev.mediaprocessing.shared.common.model.SubtitleType
import org.springframework.stereotype.Component
@Component
class MediaSelectExtractTracksListener(
private val preference: Preference
) : EventListener() {
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val useEvent = event as? MediaTracksDetermineSubtitleTypeEvent ?: return null
val pref = preference.getLanguagePreference()
// 1. Filter by subtitle type (dialogue, forced, etc.)
val filteredByType = filterBySelectionMode(
items = useEvent.subtitleTrackItems,
mode = pref.subtitleSelectionMode
)
// 2. Extract streams
val streams = filteredByType.map { it.stream }
// 3. Select subtitles based on language + format priority
val selected = selectSubtitleStreams(
streams = streams,
preferredLanguages = pref.preferredSubtitles,
preferOriginal = pref.preferOriginal,
formatPriority = pref.subtitleFormatPriority
)
return MediaTracksExtractSelectedEvent(
selectedSubtitleTracks = selected.map { it.index }
).derivedOf(event)
}
// ------------------------------------------------------------
// TYPE FILTERING
// ------------------------------------------------------------
private fun filterBySelectionMode(
items: List<SubtitleItem>,
mode: SubtitleSelectionMode
): List<SubtitleItem> {
return when (mode) {
SubtitleSelectionMode.DialogueOnly ->
items.filter { it.type == SubtitleType.Dialogue }
SubtitleSelectionMode.DialogueAndForced ->
items.filter {
it.type == SubtitleType.Dialogue || it.stream.disposition?.forced == 1
}
SubtitleSelectionMode.All ->
items
}
}
// ------------------------------------------------------------
// SUBTITLE SELECTION LOGIC
// ------------------------------------------------------------
private fun selectSubtitleStreams(
streams: List<SubtitleStream>,
preferredLanguages: List<String>,
preferOriginal: Boolean,
formatPriority: List<String>
): List<SubtitleStream> {
if (streams.isEmpty()) return emptyList()
// 1. Originalspråk
if (preferOriginal) {
val originals = streams.filter { it.disposition?.original == 1 }
if (originals.isNotEmpty()) {
return originals.uniquePerLanguageBestFormat(formatPriority)
}
}
// 2. Preferred languages
for (lang in preferredLanguages) {
val match = streams.filter {
it.tags.language?.equals(lang, ignoreCase = true) == true ||
it.subtitle_tags.language?.equals(lang, ignoreCase = true) == true
}
if (match.isNotEmpty()) {
return match.uniquePerLanguageBestFormat(formatPriority)
}
}
// 3. Default subtitles
val defaults = streams.filter { it.disposition?.default == 1 }
if (defaults.isNotEmpty()) {
return defaults.uniquePerLanguageBestFormat(formatPriority)
}
// 4. Fallback: all subtitles
return streams.uniquePerLanguageBestFormat(formatPriority)
}
// ------------------------------------------------------------
// UNIQUE PER LANGUAGE + FORMAT PRIORITY
// ------------------------------------------------------------
private fun List<SubtitleStream>.uniquePerLanguageBestFormat(
formatPriority: List<String>
): List<SubtitleStream> {
return this
.groupBy { it.tags.language ?: it.subtitle_tags.language ?: "unknown" }
.mapNotNull { (_, langGroup) ->
langGroup
.sortedBy { s ->
val idx = formatPriority.indexOf(s.codec_name.lowercase())
if (idx == -1) Int.MAX_VALUE else idx
}
.firstOrNull()
}
}
}

View File

@ -1,79 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import mu.KotlinLogging
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ManualAllowCompletionEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskCreatedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
import no.iktdev.mediaprocessing.shared.common.projection.MigrateContentProject
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Component
@Component
class MigrateCreateStoreTaskListener(
private val coordinatorEnv: CoordinatorEnv,
): EventListener() {
private val log = KotlinLogging.logger {}
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
if (event !is CollectedEvent && event !is ManualAllowCompletionEvent)
return null
val useEvent = if (event is ManualAllowCompletionEvent) {
history.lastOrNull { it is CollectedEvent } as? CollectedEvent
?: return null
} else {
event as CollectedEvent
}
val useHistory = history.filter { useEvent.eventIds.contains(it.eventId) }
val collectProjection = CollectProjection(useHistory)
log.info { collectProjection.prettyPrint() }
val statusAcceptable = collectProjection.getTaskStatus().none { it == CollectProjection.TaskStatus.Failed }
if (!statusAcceptable) {
log.warn { "One or more tasks have failed in ${event.referenceId}" }
}
val migrateContentProjection = MigrateContentProject(useHistory, coordinatorEnv.outgoingContent)
val collection = migrateContentProjection.useStore?.name ?:
throw RuntimeException("No content store configured for migration in ${event.referenceId}")
val videContent = migrateContentProjection.getVideoStoreFile()?.let { MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath) }
val subtitleContent = migrateContentProjection.getSubtitleStoreFiles()?.map {
MigrateToContentStoreTask.Data.SingleSubtitle(it.language, it.cts.cachedFile.absolutePath, it.cts.storeFile.absolutePath, )
}
val coverContent = migrateContentProjection.getCoverStoreFiles()?.map {
MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath)
}
val storeTask = MigrateToContentStoreTask(
MigrateToContentStoreTask.Data(
collection = collection,
videoContent = videContent,
subtitleContent = subtitleContent,
coverContent = coverContent
)
).derivedOf(event)
if (!CollectProjection(history).isStorePermitted()) {
log.info { "\uD83D\uDED1 Not storing content and metadata automatically for collection: $collection @ ${useEvent.referenceId}" }
log.info { "A manual allow completion event is required to proceed." }
return null
}
TaskStore.persist(storeTask)
return MigrateContentToStoreTaskCreatedEvent(storeTask.taskId).derivedOf(useEvent)
}
}

View File

@ -1,31 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import no.iktdev.eventi.ListenerOrder
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
import org.springframework.stereotype.Component
@ListenerOrder(1)
@Component
class StartedListener : EventListener() {
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
val useEvent = event as? FileReadyEvent ?: return null
return StartProcessingEvent(
data = StartData(
flow = StartFlow.Auto,
fileUri = useEvent.data.fileUri,
operation = setOf(
OperationType.ExtractSubtitles,
OperationType.ConvertSubtitles,
OperationType.Encode,
OperationType.MetadataSearch
)
)
)
}
}

View File

@ -1,78 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.events
import mu.KotlinLogging
import no.iktdev.eventi.events.EventListener
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ManualAllowCompletionEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskCreatedEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask
import no.iktdev.mediaprocessing.shared.common.getName
import no.iktdev.mediaprocessing.shared.common.model.ContentExport
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
import no.iktdev.mediaprocessing.shared.common.projection.StoreProjection
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Component
@Component
class StoreContentAndMetadataListener: EventListener() {
val log = KotlinLogging.logger {}
override fun onEvent(
event: Event,
history: List<Event>
): Event? {
if (event !is MigrateContentToStoreTaskResultEvent && event !is ManualAllowCompletionEvent)
return null
val useEvent = if (event is ManualAllowCompletionEvent) {
history.lastOrNull { it is MigrateContentToStoreTaskResultEvent } as? MigrateContentToStoreTaskResultEvent
?: return null
} else {
event as MigrateContentToStoreTaskResultEvent
}
val collectionEvent = history.lastOrNull { it is CollectedEvent } as? CollectedEvent
?: return null
val useHistory = (history.filter { collectionEvent.eventIds.contains(it.eventId) }) + listOf(useEvent)
val projection = StoreProjection(useHistory)
val collection = projection.getCollection()
if (collection.isNullOrBlank()) {
log.error { "Collection is null @ ${useEvent.referenceId}" }
return null
}
val metadata = projection.projectMetadata()
if (metadata == null) {
log.error { "Metadata is null @ ${useEvent.referenceId}"}
return null
}
if (useEvent.status != TaskStatus.Completed) {
log.info { "${useEvent.referenceId} - ${MigrateContentToStoreTaskResultEvent::class.getName()} is failed, thus no task will be created" }
return null
}
if (!CollectProjection(history).isStorePermitted()) {
log.info { "\uD83D\uDED1 Not storing content and metadata automatically for collection: $collection @ ${useEvent.referenceId}" }
log.info { "A manual allow completion event is required to proceed." }
return null
}
val exportInfo = ContentExport(
collection = collection,
media = projection.projectMediaFiles(),
episodeInfo = projection.projectEpisodeInfo(),
metadata = metadata
)
val task = StoreContentAndMetadataTask(exportInfo).derivedOf(useEvent)
TaskStore.persist(task)
return StoreContentAndMetadataTaskCreatedEvent(task.taskId).derivedOf(useEvent)
}
}

View File

@ -1,90 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
import mu.KotlinLogging
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.Task
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskListener
import no.iktdev.eventi.tasks.TaskType
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
import no.iktdev.mediaprocessing.shared.common.DownloadClient
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask
import no.iktdev.mediaprocessing.shared.common.notExist
import org.springframework.stereotype.Component
import java.util.*
@Component
class DownloadCoverTaskListener(
private val coordinatorEnv: CoordinatorEnv,
): TaskListener(TaskType.MIXED) {
val log = KotlinLogging.logger {}
override fun getWorkerId(): String {
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
}
override fun supports(task: Task): Boolean {
return task is CoverDownloadTask
}
override suspend fun onTask(task: Task): Event? {
val pickedTask = task as? CoverDownloadTask ?: return null
log.info { "Downloading cover from ${pickedTask.data.url}" }
val taskData = pickedTask.data
val downloadClient = getDownloadClient()
val downloadResult = try {
downloadClient.download(taskData.url, taskData.outputFileName)
} catch (e: Exception) {
return CoverDownloadResultEvent(status = TaskStatus.Failed)
}
val downloadedFile = downloadResult.result
if (downloadResult.success && downloadedFile != null) {
log.info { "Downloaded cover to ${downloadedFile.absolutePath}" }
return CoverDownloadResultEvent(
status = TaskStatus.Completed,
data = CoverDownloadResultEvent.CoverDownloadedData(
source = taskData.source,
outputFile = downloadedFile.absolutePath
)
).producedFrom(pickedTask)
} else {
log.error { "Failed to download cover from ${taskData.url}" }
return CoverDownloadResultEvent(
status = TaskStatus.Failed,
)
}
}
override fun createIncompleteStateTaskEvent(
task: Task,
status: TaskStatus,
exception: Exception?
): Event {
val message = when (status) {
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
TaskStatus.Cancelled -> "Canceled"
else -> ""
}
return CoverDownloadResultEvent(null, status, error = message)
}
open fun getDownloadClient(): DownloadClient {
return DefaultDownloadClient(coordinatorEnv)
}
class DefaultDownloadClient(private val coordinatorEnv: CoordinatorEnv) : DownloadClient(
outDir = coordinatorEnv.cachedContent,
connectionFactory = DefaultConnectionFactory(),) {
override fun onCreate() {
super.onCreate()
if (outDir.notExist()) {
outDir.mkdirs()
}
}
}
}

View File

@ -1,9 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
import no.iktdev.eventi.tasks.TaskListener
import no.iktdev.eventi.tasks.TaskType
import no.iktdev.mediaprocessing.ffmpeg.FFprobe
abstract class FfprobeTaskListener(taskType: TaskType): TaskListener(taskType) {
abstract fun getFfprobe(): FFprobe
}

View File

@ -1,71 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
import mu.KotlinLogging
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.Task
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskType
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
import no.iktdev.mediaprocessing.ffmpeg.FFprobe
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask
import org.springframework.stereotype.Component
import java.util.*
@Component
class MediaStreamReadTaskListener(
private val coordinatorEnv: CoordinatorEnv
): FfprobeTaskListener(TaskType.CPU_INTENSIVE) {
val log = KotlinLogging.logger {}
override fun getWorkerId(): String {
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
}
override fun supports(task: Task): Boolean {
return task is MediaReadTask
}
override suspend fun onTask(task: Task): Event? {
val pickedTask = task as? MediaReadTask ?: return null
try {
val probeResult = getFfprobe()
.readJsonStreams(pickedTask.fileUri)
val result =
probeResult.data ?: throw RuntimeException("No data returned from ffprobe for ${pickedTask.fileUri}")
return CoordinatorReadStreamsResultEvent(
status = TaskStatus.Completed,
data = result
).producedFrom(task)
} catch (e: Exception) {
log.error(e) { "Error reading media streams for ${pickedTask.fileUri}" }
return CoordinatorReadStreamsResultEvent(
status = TaskStatus.Failed,
data = null
)
}
}
override fun createIncompleteStateTaskEvent(
task: Task,
status: TaskStatus,
exception: Exception?
): Event {
val message = when (status) {
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
TaskStatus.Cancelled -> "Canceled"
else -> ""
}
return CoordinatorReadStreamsResultEvent(null, status, error = message)
}
override fun getFfprobe(): FFprobe {
return JsonFfinfo(coordinatorEnv.ffprobe)
}
class JsonFfinfo(executable: String): FFprobe(executable) {
}
}

View File

@ -1,197 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.Task
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskListener
import no.iktdev.eventi.tasks.TaskType
import no.iktdev.mediaprocessing.coordinator.util.FileServiceException
import no.iktdev.mediaprocessing.coordinator.util.FileSystemService
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask
import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus
import no.iktdev.mediaprocessing.shared.common.silentTry
import org.springframework.stereotype.Component
import java.io.File
import java.nio.file.Files
import java.util.*
@Component
class MigrateContentToStoreTaskListener : TaskListener(TaskType.IO_INTENSIVE) {
override fun getWorkerId(): String =
"${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
override fun supports(task: Task): Boolean =
task is MigrateToContentStoreTask
override suspend fun onTask(task: Task): Event? {
val picked = task as? MigrateToContentStoreTask ?: return null
val fs = getFileSystemService()
val video = migrateVideo(fs, picked.data.videoContent)
val subs = migrateSubtitle(fs, picked.data.subtitleContent ?: emptyList())
val covers = migrateCover(fs, picked.data.coverContent ?: emptyList())
deleteCache(fs, picked)
return MigrateContentToStoreTaskResultEvent(
status = TaskStatus.Completed,
migrateData = MigrateContentToStoreTaskResultEvent.MigrateData(
collection = picked.data.collection,
videoMigrate = video,
subtitleMigrate = subs,
coverMigrate = covers
)
).producedFrom(task)
}
override fun createIncompleteStateTaskEvent(
task: Task,
status: TaskStatus,
exception: Exception?
): Event {
val message = when (status) {
TaskStatus.Failed -> exception?.message ?: "Unknown error"
TaskStatus.Cancelled -> "Canceled"
else -> ""
}
return MigrateContentToStoreTaskResultEvent(
migrateData = null,
status = status,
error = message
).producedFrom(task)
}
private fun deleteCache(fs: FileSystemService, task: MigrateToContentStoreTask) {
task.data.videoContent?.cachedUri?.let { silentTry { fs.delete(File(it)) } }
task.data.subtitleContent?.forEach { silentTry { fs.delete(File(it.cachedUri)) } }
task.data.coverContent?.forEach { silentTry { fs.delete(File(it.cachedUri)) } }
}
// -------------------------------------------------------------------------
// MIGRATION HELPERS
// -------------------------------------------------------------------------
private fun migrateFile(fs: FileSystemService, source: File, destination: File) {
if (destination.exists()) {
try {
fs.verifyIdentical(source, destination)
return
} catch (e: FileServiceException.VerificationFailed) {
throw FileServiceException.DestinationExistsButDifferent(source, destination)
}
}
fs.copy(source, destination)
fs.verifyIdentical(source, destination)
}
internal fun migrateVideo(
fs: FileSystemService,
content: MigrateToContentStoreTask.Data.SingleContent?
): MigrateContentToStoreTaskResultEvent.FileMigration {
if (content == null) {
return MigrateContentToStoreTaskResultEvent.FileMigration(null, MigrateStatus.NotPresent)
}
val source = File(content.cachedUri)
val dest = File(content.storeUri)
migrateFile(fs, source, dest)
return MigrateContentToStoreTaskResultEvent.FileMigration(
storedUri = dest.absolutePath,
status = MigrateStatus.Completed
)
}
internal fun migrateSubtitle(
fs: FileSystemService,
subs: List<MigrateToContentStoreTask.Data.SingleSubtitle>
): List<MigrateContentToStoreTaskResultEvent.SubtitleMigration> {
if (subs.isEmpty()) {
return listOf(
MigrateContentToStoreTaskResultEvent.SubtitleMigration(
language = null,
storedUri = null,
status = MigrateStatus.NotPresent
)
)
}
return subs.map { sub ->
val source = File(sub.cachedUri)
val dest = File(sub.storeUri)
migrateFile(fs, source, dest)
MigrateContentToStoreTaskResultEvent.SubtitleMigration(
language = sub.language,
storedUri = dest.absolutePath,
status = MigrateStatus.Completed
)
}
}
internal fun migrateCover(
fs: FileSystemService,
covers: List<MigrateToContentStoreTask.Data.SingleContent>
): List<MigrateContentToStoreTaskResultEvent.FileMigration> {
if (covers.isEmpty()) {
return listOf(
MigrateContentToStoreTaskResultEvent.FileMigration(
storedUri = null,
status = MigrateStatus.NotPresent
)
)
}
return covers.map { cover ->
val source = File(cover.cachedUri)
val dest = File(cover.storeUri)
migrateFile(fs, source, dest)
MigrateContentToStoreTaskResultEvent.FileMigration(
storedUri = dest.absolutePath,
status = MigrateStatus.Completed
)
}
}
open fun getFileSystemService(): FileSystemService =
DefaultFileSystemService()
class DefaultFileSystemService : FileSystemService {
override fun copy(source: File, destination: File) {
if (!source.exists()) {
throw FileServiceException.SourceMissing(source)
}
try {
source.copyTo(destination, overwrite = true)
} catch (e: Exception) {
throw FileServiceException.CopyFailed(source, destination, e)
}
}
override fun verifyIdentical(source: File, destination: File) {
val mismatch = Files.mismatch(source.toPath(), destination.toPath())
if (mismatch != -1L) {
throw FileServiceException.VerificationFailed(source, destination)
}
}
override fun delete(file: File) {
file.delete()
}
}
}

View File

@ -1,73 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
import no.iktdev.eventi.models.Event
import no.iktdev.eventi.models.Task
import no.iktdev.eventi.models.store.TaskStatus
import no.iktdev.eventi.tasks.TaskListener
import no.iktdev.eventi.tasks.TaskType
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.http.HttpEntity
import org.springframework.http.HttpHeaders
import org.springframework.http.HttpMethod
import org.springframework.http.MediaType
import org.springframework.stereotype.Component
import org.springframework.web.client.RestTemplate
import java.util.*
@Component
class StoreContentAndMetadataTaskListener : TaskListener(TaskType.MIXED) {
@Autowired
lateinit var streamitRestTemplate: RestTemplate
override fun getWorkerId(): String {
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
}
override fun supports(task: Task): Boolean {
return task is StoreContentAndMetadataTask
}
override suspend fun onTask(task: Task): Event? {
val pickedTask = task as? StoreContentAndMetadataTask ?: return null
val headers = HttpHeaders().apply { contentType = MediaType.APPLICATION_JSON }
val entity = HttpEntity(pickedTask.data, headers)
// ❗ Ikke fang exceptions — la TaskListener håndtere dem
val response = streamitRestTemplate.exchange(
"/api/mediaprocesser/import",
HttpMethod.POST,
entity,
Void::class.java,
)
if (!response.statusCode.is2xxSuccessful) {
throw IllegalStateException("StreamIt returned ${response.statusCode}")
}
// Hvis vi kommer hit → alt OK
return StoreContentAndMetadataTaskResultEvent(
status = TaskStatus.Completed
).producedFrom(task)
}
override fun createIncompleteStateTaskEvent(
task: Task,
status: TaskStatus,
exception: Exception?
): Event {
val message = when (status) {
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
TaskStatus.Cancelled -> "Canceled"
else -> ""
}
return StoreContentAndMetadataTaskResultEvent(
status = status,
error = message
).producedFrom(task)
}
}

View File

@ -1,49 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.services
import mu.KotlinLogging
import no.iktdev.mediaprocessing.shared.common.dto.requests.StartProcessRequest
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartFlow
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
import no.iktdev.mediaprocessing.shared.common.notExist
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
import org.springframework.stereotype.Service
import java.io.File
import java.util.*
@Service
class CommandService {
val log = KotlinLogging.logger {}
fun startProcess(request: StartProcessRequest): StartResult {
return try {
val file = File(request.fileUri)
if (file.notExist()) {
throw IllegalArgumentException("File does not exists at ${request.fileUri}")
}
if (!file.canRead()) {
throw IllegalStateException("File is not readable ${request.fileUri}")
}
val startProcessingEvent = StartProcessingEvent(
data = StartData(
fileUri = request.fileUri,
operation = request.operationTypes,
flow = StartFlow.Manual
)
).newReferenceId()
EventStore.persist(startProcessingEvent)
StartResult.Accepted(startProcessingEvent.referenceId)
} catch (e: Exception) {
log.error { e }
StartResult.Rejected("Failed to start process for file ${request.fileUri}, with the following reason: ${e.message}")
}
}
sealed class StartResult {
data class Accepted(val referenceId: UUID) : StartResult()
data class Rejected(val reason: String) : StartResult()
}
}

View File

@ -1,132 +0,0 @@
package no.iktdev.mediaprocessing.coordinator.services
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealthStatus
import no.iktdev.mediaprocessing.coordinator.dto.health.SequenceHealth
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
import no.iktdev.mediaprocessing.coordinator.util.getDiskInfoFor
import no.iktdev.mediaprocessing.shared.common.dto.CurrentState
import no.iktdev.mediaprocessing.shared.common.rules.EventLifecycleRules
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
import org.springframework.stereotype.Service
import java.time.Duration
import java.time.Instant
@Service
class CoordinatorHealthService(
private val taskService: TaskService,
private val eventService: EventService,
private val aggregator: SequenceAggregatorService,
private val coordinatorEnv: CoordinatorEnv
) {
fun getHealth(): CoordinatorHealth {
val tasks = taskService.getActiveTasks()
val incompleteSequences = eventService.getIncompleteSequences()
.groupBy { it.referenceId }
.values
// --- TASK HEALTH ---
val abandonedTaskIds = tasks
.filter { TaskLifecycleRules.isAbandoned(it.consumed, it.persistedAt, it.lastCheckIn) }
.map { it.taskId }
val stalledTaskIds = tasks
.filter { TaskLifecycleRules.isStalled(it) }
.map { it.taskId }
val failedTasks = taskService.getFailedTasks()
val sequencesOnHold = aggregator.getActiveSequences().filter { it.currentState == CurrentState.OnHold }
// --- SEQUENCE HEALTH ---
val overdueSequences = incompleteSequences
.filter { EventLifecycleRules.isOverdue(it) }
.map { seq ->
val refId = seq.first().referenceId
val firstEventAt = seq.minOf { it.persistedAt }
val lastEventAt = seq.maxOf { it.persistedAt }
val expectedWindow = EventLifecycleRules.expectedCompletionTimeWindow(seq)
val actualAge = Duration.between(firstEventAt, Instant.now())
// Operasjonelle verdier frontend trenger
val expectedFinish = firstEventAt.plus(expectedWindow)
val overdueDuration = actualAge.minus(expectedWindow).coerceAtLeast(Duration.ZERO)
val isOverdue = overdueDuration > Duration.ZERO
SequenceHealth(
referenceId = refId.toString(),
// eksisterende felter
age = actualAge,
expected = expectedWindow,
lastEventAt = lastEventAt,
eventCount = seq.size,
// nye operasjonelle felter
startTime = firstEventAt,
expectedFinishTime = expectedFinish,
overdueDuration = overdueDuration,
isOverdue = isOverdue
)
}
val overdueSequenceIds = overdueSequences.map { it.referenceId }
// --- AGGREGATED STATUS ---
val status = when {
abandonedTaskIds.isNotEmpty() ||
stalledTaskIds.isNotEmpty() ||
overdueSequenceIds.isNotEmpty() -> CoordinatorHealthStatus.DEGRADED
else -> CoordinatorHealthStatus.HEALTHY
}
val lastActivityCandidates = listOfNotNull(
tasks.maxOfOrNull { it.persistedAt },
eventService.getLastEventTimestamp()
)
return CoordinatorHealth(
status = status,
abandonedTasks = abandonedTaskIds.size,
stalledTasks = stalledTaskIds.size,
activeTasks = tasks.count { !it.consumed },
failedTasks = failedTasks.count(),
queuedTasks = TaskStore.getPendingTasks().size,
lastActivity = lastActivityCandidates.maxOrNull(),
abandonedTaskIds = abandonedTaskIds.map { it.toString() },
stalledTaskIds = stalledTaskIds.map { it.toString() },
sequencesOnHold = sequencesOnHold.size,
sequencesOnHoldIds = sequencesOnHold.map { it.referenceId },
overdueSequenceIds = overdueSequenceIds,
overdueSequences = overdueSequences,
details = mapOf(
"oldestActiveTaskAgeMinutes" to tasks.minOfOrNull {
Duration.between(it.persistedAt, Instant.now()).toMinutes()
}
)
)
}
fun getEventRate(): EventRate {
return EventRate(
lastMinute = eventService.getEventsLast(1),
lastFiveMinutes = eventService.getEventsLast(5)
)
}
fun getDiskHealth(): List<DiskInfo> {
val paths = listOf(coordinatorEnv.incomingContent,
coordinatorEnv.cachedContent,
coordinatorEnv.outgoingContent)
.map { it -> it.absolutePath }
return getDiskInfoFor(paths)
}
}

Some files were not shown because too many files have changed in this diff Show More