Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 77b086cd82 |
100
.github/workflows/build-java-app.yml
vendored
100
.github/workflows/build-java-app.yml
vendored
@ -1,100 +0,0 @@
|
|||||||
name: Build Java App
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_call:
|
|
||||||
inputs:
|
|
||||||
app:
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
dockerTag:
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
enabled:
|
|
||||||
required: true
|
|
||||||
type: boolean
|
|
||||||
shouldBuild:
|
|
||||||
required: true
|
|
||||||
type: boolean
|
|
||||||
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-java:
|
|
||||||
if: ${{ inputs.enabled && inputs.shouldBuild }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Detect frontend
|
|
||||||
id: detect_web
|
|
||||||
run: |
|
|
||||||
if [ -d "apps/${{ inputs.app }}/web" ]; then
|
|
||||||
echo "has_web=true" >> $GITHUB_OUTPUT
|
|
||||||
else
|
|
||||||
echo "has_web=false" >> $GITHUB_OUTPUT
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Build React frontend
|
|
||||||
if: ${{ steps.detect_web.outputs.has_web == 'true' }}
|
|
||||||
run: |
|
|
||||||
cd apps/${{ inputs.app }}/web
|
|
||||||
npm install
|
|
||||||
export CI=false
|
|
||||||
npm run build
|
|
||||||
|
|
||||||
- name: Copy React build into Spring Boot resources
|
|
||||||
if: ${{ steps.detect_web.outputs.has_web == 'true' }}
|
|
||||||
run: |
|
|
||||||
rm -rf apps/${{ inputs.app }}/src/main/resources/static
|
|
||||||
mkdir -p apps/${{ inputs.app }}/src/main/resources/static
|
|
||||||
cp -r apps/${{ inputs.app }}/web/build/* apps/${{ inputs.app }}/src/main/resources/static
|
|
||||||
|
|
||||||
- name: Extract version
|
|
||||||
run: |
|
|
||||||
VERSION=$(grep '^version' apps/${{ inputs.app }}/build.gradle.kts | sed 's/.*"\(.*\)".*/\1/')
|
|
||||||
echo "VERSION=$VERSION" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
- name: Run unit tests
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :apps:${{ inputs.app }}:test --info --stacktrace
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build Java module
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :apps:${{ inputs.app }}:bootJar --info --stacktrace
|
|
||||||
|
|
||||||
- name: Build Docker image locally
|
|
||||||
run: |
|
|
||||||
docker build \
|
|
||||||
-f ./dockerfiles/DebianJava \
|
|
||||||
-t local-${{ inputs.app }}:${{ inputs.dockerTag }} \
|
|
||||||
--build-arg MODULE_NAME=${{ inputs.app }} \
|
|
||||||
--build-arg PASS_APP_VERSION=${{ env.VERSION }} \
|
|
||||||
.
|
|
||||||
|
|
||||||
- name: Test Docker container
|
|
||||||
run: |
|
|
||||||
docker run --rm local-${{ inputs.app }}:${{ inputs.dockerTag }} /bin/sh -c "echo 'Smoke test OK'"
|
|
||||||
|
|
||||||
- name: Docker login
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
|
||||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
|
||||||
|
|
||||||
|
|
||||||
- name: Push Docker image
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/DebianJavaFfmpeg
|
|
||||||
build-args: |
|
|
||||||
MODULE_NAME=${{ inputs.app }}
|
|
||||||
PASS_APP_VERSION=${{ env.VERSION }}
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
bskjon/mediaprocessing-${{ inputs.app }}:v5
|
|
||||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ inputs.dockerTag }}
|
|
||||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ github.sha }}
|
|
||||||
127
.github/workflows/build-python-app.yml
vendored
127
.github/workflows/build-python-app.yml
vendored
@ -1,127 +0,0 @@
|
|||||||
name: Build Python App
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_call:
|
|
||||||
inputs:
|
|
||||||
app:
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
dockerTag:
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
enabled:
|
|
||||||
required: true
|
|
||||||
type: boolean
|
|
||||||
shouldBuild:
|
|
||||||
required: true
|
|
||||||
type: boolean
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-python:
|
|
||||||
if: ${{ inputs.enabled && inputs.shouldBuild }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Cache pip per app
|
|
||||||
# -----------------------------
|
|
||||||
- name: Cache pip
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: ~/.cache/pip
|
|
||||||
key: ${{ runner.os }}-pip-${{ inputs.app }}-${{ hashFiles('apps/${{ inputs.app }}/requirements*.txt') }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-pip-${{ inputs.app }}-
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Install Python deps
|
|
||||||
# -----------------------------
|
|
||||||
- name: Install dependencies
|
|
||||||
working-directory: apps/${{ inputs.app }}
|
|
||||||
run: |
|
|
||||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
|
||||||
if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Run tests
|
|
||||||
# -----------------------------
|
|
||||||
- name: Run Python tests
|
|
||||||
working-directory: apps/${{ inputs.app }}
|
|
||||||
run: python -m pytest -q
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Setup Buildx
|
|
||||||
# -----------------------------
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Cache Docker layers per app
|
|
||||||
# -----------------------------
|
|
||||||
- name: Cache Docker layers
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: /tmp/.buildx-cache-${{ inputs.app }}
|
|
||||||
key: ${{ runner.os }}-buildx-${{ inputs.app }}-${{ github.sha }}
|
|
||||||
restore-keys: |
|
|
||||||
${{ runner.os }}-buildx-${{ inputs.app }}-
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Build image (load locally for smoke test)
|
|
||||||
# -----------------------------
|
|
||||||
- name: Build Docker image (local load)
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/Python
|
|
||||||
build-args: |
|
|
||||||
MODULE_NAME=${{ inputs.app }}
|
|
||||||
load: true
|
|
||||||
tags: local-${{ inputs.app }}:${{ inputs.dockerTag }}
|
|
||||||
cache-from: type=local,src=/tmp/.buildx-cache-${{ inputs.app }}
|
|
||||||
cache-to: type=local,dest=/tmp/.buildx-cache-${{ inputs.app }}-new
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Smoke test
|
|
||||||
# -----------------------------
|
|
||||||
- name: Smoke test container
|
|
||||||
run: |
|
|
||||||
docker run --rm local-${{ inputs.app }}:${{ inputs.dockerTag }} \
|
|
||||||
/bin/sh -c "echo 'Smoke test OK'"
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Docker login
|
|
||||||
# -----------------------------
|
|
||||||
- name: Docker login
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
|
||||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Push final image (no rebuild)
|
|
||||||
# -----------------------------
|
|
||||||
- name: Push Docker image
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/Python
|
|
||||||
build-args: |
|
|
||||||
MODULE_NAME=${{ inputs.app }}
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
bskjon/mediaprocessing-${{ inputs.app }}:v5
|
|
||||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ inputs.dockerTag }}
|
|
||||||
bskjon/mediaprocessing-${{ inputs.app }}:v5-${{ github.sha }}
|
|
||||||
cache-from: type=local,src=/tmp/.buildx-cache-${{ inputs.app }}
|
|
||||||
cache-to: type=local,dest=/tmp/.buildx-cache-${{ inputs.app }}-new
|
|
||||||
|
|
||||||
# -----------------------------
|
|
||||||
# Move Docker cache
|
|
||||||
# -----------------------------
|
|
||||||
- name: Move Docker cache
|
|
||||||
run: |
|
|
||||||
rm -rf /tmp/.buildx-cache-${{ inputs.app }}
|
|
||||||
mv /tmp/.buildx-cache-${{ inputs.app }}-new /tmp/.buildx-cache-${{ inputs.app }}
|
|
||||||
26
.github/workflows/build-shared.yml
vendored
26
.github/workflows/build-shared.yml
vendored
@ -1,26 +0,0 @@
|
|||||||
name: Build Shared
|
|
||||||
|
|
||||||
on:
|
|
||||||
workflow_call:
|
|
||||||
inputs:
|
|
||||||
dockerTag:
|
|
||||||
required: true
|
|
||||||
type: string
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-shared:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Cache Gradle
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: ~/.gradle/caches
|
|
||||||
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
|
|
||||||
|
|
||||||
- name: Build Shared module
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :shared:build --info --stacktrace
|
|
||||||
115
.github/workflows/build-v5.yml
vendored
115
.github/workflows/build-v5.yml
vendored
@ -1,115 +0,0 @@
|
|||||||
name: Build v5
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [ v5 ]
|
|
||||||
pull_request:
|
|
||||||
branches: [ v5 ]
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
pre-check:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
outputs:
|
|
||||||
sharedDefinitions: ${{ steps.filter.outputs.sharedDefinitions }}
|
|
||||||
shared: ${{ steps.filter.outputs.shared }}
|
|
||||||
processer: ${{ steps.filter.outputs.processer }}
|
|
||||||
converter: ${{ steps.filter.outputs.converter }}
|
|
||||||
coordinator: ${{ steps.filter.outputs.coordinator }}
|
|
||||||
ui: ${{ steps.filter.outputs.ui }}
|
|
||||||
py-metadata: ${{ steps.filter.outputs.metadata }}
|
|
||||||
py-watcher: ${{ steps.filter.outputs.watcher }}
|
|
||||||
dockerTag: ${{ steps.tag.outputs.tag }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: ${{ github.sha }}
|
|
||||||
base: ${{ github.event.before }}
|
|
||||||
|
|
||||||
- name: Detect changes
|
|
||||||
id: filter
|
|
||||||
uses: dorny/paths-filter@v3
|
|
||||||
with:
|
|
||||||
filters: |
|
|
||||||
shared:
|
|
||||||
- 'shared/**'
|
|
||||||
sharedDefinitions:
|
|
||||||
- 'gradle/**'
|
|
||||||
processer:
|
|
||||||
- 'apps/processer/**'
|
|
||||||
converter:
|
|
||||||
- 'apps/converter/**'
|
|
||||||
coordinator:
|
|
||||||
- 'apps/coordinator/**'
|
|
||||||
ui:
|
|
||||||
- 'apps/ui/**'
|
|
||||||
metadata:
|
|
||||||
- 'apps/py-metadata/**'
|
|
||||||
watcher:
|
|
||||||
- 'apps/py-watcher/**'
|
|
||||||
|
|
||||||
- name: Generate docker tag
|
|
||||||
id: tag
|
|
||||||
run: echo "tag=$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" >> $GITHUB_OUTPUT
|
|
||||||
|
|
||||||
|
|
||||||
build-shared:
|
|
||||||
needs: pre-check
|
|
||||||
if: ${{
|
|
||||||
needs.pre-check.outputs.shared == 'true'
|
|
||||||
|| needs.pre-check.outputs.sharedDefinitions == 'true'
|
|
||||||
|| needs.pre-check.outputs.processer == 'true'
|
|
||||||
|| needs.pre-check.outputs.converter == 'true'
|
|
||||||
|| needs.pre-check.outputs.coordinator == 'true'
|
|
||||||
|| needs.pre-check.outputs.ui == 'true'
|
|
||||||
|| github.event_name == 'workflow_dispatch'
|
|
||||||
}}
|
|
||||||
uses: ./.github/workflows/build-shared.yml
|
|
||||||
with:
|
|
||||||
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
|
|
||||||
|
|
||||||
build-java:
|
|
||||||
needs: [pre-check, build-shared]
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- app: processer
|
|
||||||
enabled: true
|
|
||||||
- app: converter
|
|
||||||
enabled: true
|
|
||||||
- app: coordinator
|
|
||||||
enabled: true
|
|
||||||
- app: ui
|
|
||||||
enabled: false
|
|
||||||
|
|
||||||
uses: ./.github/workflows/build-java-app.yml
|
|
||||||
with:
|
|
||||||
app: ${{ matrix.app }}
|
|
||||||
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
|
|
||||||
enabled: ${{ matrix.enabled }}
|
|
||||||
shouldBuild: ${{ needs.pre-check.outputs[matrix.app] == 'true'
|
|
||||||
|| needs.pre-check.outputs.shared == 'true'
|
|
||||||
|| needs.pre-check.outputs.sharedDefinitions == 'true'
|
|
||||||
|| github.event_name == 'workflow_dispatch' }}
|
|
||||||
secrets: inherit
|
|
||||||
|
|
||||||
|
|
||||||
build-python:
|
|
||||||
needs: [pre-check]
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- app: py-metadata
|
|
||||||
enabled: true
|
|
||||||
- app: py-watcher
|
|
||||||
enabled: true
|
|
||||||
|
|
||||||
uses: ./.github/workflows/build-python-app.yml
|
|
||||||
with:
|
|
||||||
app: ${{ matrix.app }}
|
|
||||||
dockerTag: ${{ needs.pre-check.outputs.dockerTag }}
|
|
||||||
enabled: ${{ matrix.enabled }}
|
|
||||||
shouldBuild: ${{ needs.pre-check.outputs[matrix.app] == 'true'
|
|
||||||
|| github.event_name == 'workflow_dispatch' }}
|
|
||||||
secrets: inherit
|
|
||||||
386
.github/workflows/v4.yml
vendored
386
.github/workflows/v4.yml
vendored
@ -1,386 +0,0 @@
|
|||||||
name: Build v4
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- v4
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- v4
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
pre-check:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
outputs:
|
|
||||||
pyMetadata: ${{ steps.checkAppChanges.outputs.metadata }}
|
|
||||||
sharedLibrary: ${{ steps.checkAppChanges.outputs.shared }}
|
|
||||||
coordinator: ${{ steps.checkAppChanges.outputs.coordinator }}
|
|
||||||
processer: ${{ steps.checkAppChanges.outputs.processer }}
|
|
||||||
converter: ${{ steps.checkAppChanges.outputs.converter }}
|
|
||||||
ui: ${{ steps.checkAppChanges.outputs.ui }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- uses: dorny/paths-filter@v3
|
|
||||||
name: "Detect app changes"
|
|
||||||
id: checkAppChanges
|
|
||||||
with:
|
|
||||||
base: ''
|
|
||||||
filters: |
|
|
||||||
metadata:
|
|
||||||
- 'apps/pyMetadata/**'
|
|
||||||
coordinator:
|
|
||||||
- 'apps/coordinator/**'
|
|
||||||
processer:
|
|
||||||
- 'apps/processer/**'
|
|
||||||
converter:
|
|
||||||
- 'apps/converter/**'
|
|
||||||
ui:
|
|
||||||
- 'apps/ui/**'
|
|
||||||
shared:
|
|
||||||
- 'shared/**'
|
|
||||||
|
|
||||||
# Step to print the outputs from "pre-check" job
|
|
||||||
- name: Print Outputs from pre-check job
|
|
||||||
run: |
|
|
||||||
echo "Apps\n"
|
|
||||||
echo "app:pyMetadata: ${{ steps.checkAppChanges.outputs.metadata }}"
|
|
||||||
echo "app:coordinator: ${{ steps.checkAppChanges.outputs.coordinator }}"
|
|
||||||
echo "app:processer: ${{ steps.checkAppChanges.outputs.processer }}"
|
|
||||||
echo "app:converter: ${{ steps.checkAppChanges.outputs.converter }}"
|
|
||||||
echo "app:ui: ${{ steps.checkAppChanges.outputs.ui }}"
|
|
||||||
|
|
||||||
echo "Shared"
|
|
||||||
echo "shared: ${{ steps.checkAppChanges.outputs.shared }}"
|
|
||||||
echo "${{ steps.checkAppChanges.outputs }}"
|
|
||||||
|
|
||||||
build-shared:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: pre-check
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Cache Shared code Gradle dependencies
|
|
||||||
id: cache-gradle
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: ~/.gradle/caches
|
|
||||||
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
|
|
||||||
|
|
||||||
- name: Build Shared code
|
|
||||||
if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.sharedLibrary == 'true' || github.event_name == 'workflow_dispatch'
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :shared:build --stacktrace --info
|
|
||||||
|
|
||||||
|
|
||||||
build-processer:
|
|
||||||
needs:
|
|
||||||
- build-shared
|
|
||||||
- pre-check
|
|
||||||
if: ${{ needs.pre-check.outputs.processer == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
#if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Cache Shared Gradle dependencies
|
|
||||||
id: cache-gradle
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: ~/.gradle/caches
|
|
||||||
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
|
|
||||||
|
|
||||||
- name: Extract version from build.gradle.kts
|
|
||||||
id: extract_version
|
|
||||||
run: |
|
|
||||||
VERSION=$(cat ./apps/processer/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
|
|
||||||
echo "VERSION=$VERSION"
|
|
||||||
echo "VERSION=$VERSION" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build Processer module
|
|
||||||
id: build-processer
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :apps:processer:bootJar --info --stacktrace
|
|
||||||
echo "Build completed"
|
|
||||||
|
|
||||||
|
|
||||||
- name: Generate Docker image tag
|
|
||||||
id: docker-tag
|
|
||||||
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
|
|
||||||
|
|
||||||
- name: Docker login
|
|
||||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
|
||||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push Docker image
|
|
||||||
uses: docker/build-push-action@v2
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/DebianJavaFfmpeg
|
|
||||||
build-args: |
|
|
||||||
MODULE_NAME=processer
|
|
||||||
PASS_APP_VERSION=${{ env.VERSION }}
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
bskjon/mediaprocessing-processer:v4
|
|
||||||
bskjon/mediaprocessing-processer:v4-${{ github.sha }}
|
|
||||||
bskjon/mediaprocessing-processer:v4-${{ steps.docker-tag.outputs.tag }}
|
|
||||||
|
|
||||||
build-converter:
|
|
||||||
needs:
|
|
||||||
- build-shared
|
|
||||||
- pre-check
|
|
||||||
if: ${{ needs.pre-check.outputs.converter == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
#if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v3
|
|
||||||
|
|
||||||
- name: Cache Shared Gradle dependencies
|
|
||||||
id: cache-gradle
|
|
||||||
uses: actions/cache@v3
|
|
||||||
with:
|
|
||||||
path: ~/.gradle/caches
|
|
||||||
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
|
|
||||||
|
|
||||||
- name: Extract version from build.gradle.kts
|
|
||||||
id: extract_version
|
|
||||||
run: |
|
|
||||||
VERSION=$(cat ./apps/converter/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
|
|
||||||
echo "VERSION=$VERSION"
|
|
||||||
echo "VERSION=$VERSION" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build Converter module
|
|
||||||
id: build-converter
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :apps:converter:bootJar --info --debug
|
|
||||||
echo "Build completed"
|
|
||||||
|
|
||||||
|
|
||||||
- name: Generate Docker image tag
|
|
||||||
id: docker-tag
|
|
||||||
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
|
|
||||||
|
|
||||||
- name: Docker login
|
|
||||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
|
||||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build and push Docker image
|
|
||||||
uses: docker/build-push-action@v2
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/DebianJava
|
|
||||||
build-args: |
|
|
||||||
MODULE_NAME=converter
|
|
||||||
PASS_APP_VERSION=${{ env.VERSION }}
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
bskjon/mediaprocessing-converter:v4
|
|
||||||
bskjon/mediaprocessing-converter:v4-${{ github.sha }}
|
|
||||||
bskjon/mediaprocessing-converter:v4-${{ steps.docker-tag.outputs.tag }}
|
|
||||||
|
|
||||||
build-coordinator:
|
|
||||||
needs:
|
|
||||||
- build-shared
|
|
||||||
- pre-check
|
|
||||||
if: ${{ needs.pre-check.outputs.coordinator == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
#if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Cache Shared Gradle dependencies
|
|
||||||
id: cache-gradle
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: ~/.gradle/caches
|
|
||||||
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
|
|
||||||
|
|
||||||
- name: Extract version from build.gradle.kts
|
|
||||||
id: extract_version
|
|
||||||
run: |
|
|
||||||
VERSION=$(cat ./apps/coordinator/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
|
|
||||||
echo "VERSION=$VERSION"
|
|
||||||
echo "VERSION=$VERSION" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
- name: Build Coordinator module
|
|
||||||
id: build-coordinator
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :apps:coordinator:bootJar
|
|
||||||
echo "Build completed"
|
|
||||||
|
|
||||||
|
|
||||||
- name: Generate Docker image tag
|
|
||||||
id: docker-tag
|
|
||||||
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
|
|
||||||
|
|
||||||
- name: Docker login
|
|
||||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
|
||||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Debug Check extracted version
|
|
||||||
run: |
|
|
||||||
echo "Extracted version: ${{ env.VERSION }}"
|
|
||||||
|
|
||||||
- name: Build and push Docker image
|
|
||||||
uses: docker/build-push-action@v2
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/DebianJavaFfmpeg
|
|
||||||
build-args: |
|
|
||||||
MODULE_NAME=coordinator
|
|
||||||
PASS_APP_VERSION=${{ env.VERSION }}
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
bskjon/mediaprocessing-coordinator:v4
|
|
||||||
bskjon/mediaprocessing-coordinator:v4-${{ github.sha }}
|
|
||||||
bskjon/mediaprocessing-coordinator:v4-${{ steps.docker-tag.outputs.tag }}
|
|
||||||
|
|
||||||
build-pymetadata:
|
|
||||||
needs:
|
|
||||||
- pre-check
|
|
||||||
if: ${{ needs.pre-check.outputs.pyMetadata == 'true' || github.event_name == 'workflow_dispatch' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Build pyMetadata module
|
|
||||||
id: build-pymetadata
|
|
||||||
run: |
|
|
||||||
if [[ "${{ needs.pre-check.outputs.pyMetadata }}" == "true" || "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
|
||||||
cd apps/pyMetadata
|
|
||||||
# Add the necessary build steps for your Python module here
|
|
||||||
echo "Build completed"
|
|
||||||
else
|
|
||||||
echo "pyMetadata has not changed. Skipping pyMetadata module build."
|
|
||||||
echo "::set-output name=job_skipped::true"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Generate Docker image tag
|
|
||||||
id: docker-tag
|
|
||||||
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
|
|
||||||
|
|
||||||
- name: Docker login
|
|
||||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
|
||||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push Docker image
|
|
||||||
uses: docker/build-push-action@v5.1.0
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/Python
|
|
||||||
build-args:
|
|
||||||
MODULE_NAME=pyMetadata
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
bskjon/mediaprocessing-pymetadata:v4
|
|
||||||
bskjon/mediaprocessing-pymetadata:v4-${{ github.sha }}
|
|
||||||
bskjon/mediaprocessing-pymetadata:v4-${{ steps.docker-tag.outputs.tag }}
|
|
||||||
|
|
||||||
build-ui:
|
|
||||||
needs:
|
|
||||||
- build-shared
|
|
||||||
- pre-check
|
|
||||||
if: ${{ needs.pre-check.outputs.ui == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.sharedLibrary == 'true' }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout repository
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Cache Shared Gradle dependencies
|
|
||||||
id: cache-gradle
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: ~/.gradle/caches
|
|
||||||
key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }}
|
|
||||||
|
|
||||||
|
|
||||||
- name: Setup Node.js
|
|
||||||
uses: actions/setup-node@v2
|
|
||||||
with:
|
|
||||||
node-version: '14'
|
|
||||||
- name: Install dependencies
|
|
||||||
run: npm install
|
|
||||||
working-directory: ./apps/ui/web
|
|
||||||
|
|
||||||
- name: Build React app
|
|
||||||
run: |
|
|
||||||
export CI=false
|
|
||||||
npm run build
|
|
||||||
working-directory: ./apps/ui/web
|
|
||||||
|
|
||||||
- name : Copy build files
|
|
||||||
run: |
|
|
||||||
mkdir -p ./apps/ui/src/main/resources/static
|
|
||||||
cp -r ./apps/ui/web/build/* ./apps/ui/src/main/resources/static
|
|
||||||
|
|
||||||
- name: Extract version from build.gradle.kts
|
|
||||||
id: extract_version
|
|
||||||
run: |
|
|
||||||
VERSION=$(cat ./apps/ui/build.gradle.kts | grep '^version\s*=\s*\".*\"' | sed 's/^version\s*=\s*\"\(.*\)\"/\1/')
|
|
||||||
echo "VERSION=$VERSION"
|
|
||||||
echo "VERSION=$VERSION" >> $GITHUB_ENV
|
|
||||||
|
|
||||||
- name: Build UI module
|
|
||||||
id: build-ui
|
|
||||||
run: |
|
|
||||||
chmod +x ./gradlew
|
|
||||||
./gradlew :apps:ui:bootJar
|
|
||||||
echo "Build completed"
|
|
||||||
|
|
||||||
|
|
||||||
- name: Generate Docker image tag
|
|
||||||
id: docker-tag
|
|
||||||
run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)"
|
|
||||||
|
|
||||||
- name: Docker login
|
|
||||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
||||||
with:
|
|
||||||
username: ${{ secrets.DOCKER_HUB_NAME }}
|
|
||||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Debug Check extracted version
|
|
||||||
run: |
|
|
||||||
echo "Extracted version: ${{ env.VERSION }}"
|
|
||||||
|
|
||||||
- name: Build and push Docker image
|
|
||||||
uses: docker/build-push-action@v2
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./dockerfiles/DebianJava
|
|
||||||
build-args: |
|
|
||||||
MODULE_NAME=ui
|
|
||||||
PASS_APP_VERSION=${{ env.VERSION }}
|
|
||||||
push: true
|
|
||||||
tags: |
|
|
||||||
bskjon/mediaprocessing-ui:v4
|
|
||||||
bskjon/mediaprocessing-ui:v4-${{ github.sha }}
|
|
||||||
bskjon/mediaprocessing-ui:v4-${{ steps.docker-tag.outputs.tag }}
|
|
||||||
7
.gitignore
vendored
7
.gitignore
vendored
@ -36,12 +36,7 @@ bin/
|
|||||||
/.nb-gradle/
|
/.nb-gradle/
|
||||||
|
|
||||||
### VS Code ###
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
### Mac OS ###
|
### Mac OS ###
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
|
||||||
.idea/runConfigurations
|
|
||||||
/apps/py-metadata/venv/
|
|
||||||
/apps/py-watcher/venv/
|
|
||||||
|
|||||||
6
.idea/copilot.data.migration.agent.xml
generated
6
.idea/copilot.data.migration.agent.xml
generated
@ -1,6 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="AgentMigrationStateService">
|
|
||||||
<option name="migrationStatus" value="COMPLETED" />
|
|
||||||
</component>
|
|
||||||
</project>
|
|
||||||
6
.idea/copilot.data.migration.ask.xml
generated
6
.idea/copilot.data.migration.ask.xml
generated
@ -1,6 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="AskMigrationStateService">
|
|
||||||
<option name="migrationStatus" value="COMPLETED" />
|
|
||||||
</component>
|
|
||||||
</project>
|
|
||||||
6
.idea/copilot.data.migration.ask2agent.xml
generated
6
.idea/copilot.data.migration.ask2agent.xml
generated
@ -1,6 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="Ask2AgentMigrationStateService">
|
|
||||||
<option name="migrationStatus" value="COMPLETED" />
|
|
||||||
</component>
|
|
||||||
</project>
|
|
||||||
6
.idea/copilot.data.migration.edit.xml
generated
6
.idea/copilot.data.migration.edit.xml
generated
@ -1,6 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project version="4">
|
|
||||||
<component name="EditMigrationStateService">
|
|
||||||
<option name="migrationStatus" value="COMPLETED" />
|
|
||||||
</component>
|
|
||||||
</project>
|
|
||||||
4
.idea/gradle.xml
generated
4
.idea/gradle.xml
generated
@ -5,7 +5,6 @@
|
|||||||
<option name="linkedExternalProjectsSettings">
|
<option name="linkedExternalProjectsSettings">
|
||||||
<GradleProjectSettings>
|
<GradleProjectSettings>
|
||||||
<option name="externalProjectPath" value="$PROJECT_DIR$" />
|
<option name="externalProjectPath" value="$PROJECT_DIR$" />
|
||||||
<option name="gradleJvm" value="17" />
|
|
||||||
<option name="modules">
|
<option name="modules">
|
||||||
<set>
|
<set>
|
||||||
<option value="$PROJECT_DIR$" />
|
<option value="$PROJECT_DIR$" />
|
||||||
@ -16,8 +15,7 @@
|
|||||||
<option value="$PROJECT_DIR$/apps/ui" />
|
<option value="$PROJECT_DIR$/apps/ui" />
|
||||||
<option value="$PROJECT_DIR$/shared" />
|
<option value="$PROJECT_DIR$/shared" />
|
||||||
<option value="$PROJECT_DIR$/shared/common" />
|
<option value="$PROJECT_DIR$/shared/common" />
|
||||||
<option value="$PROJECT_DIR$/shared/event-task-contract" />
|
<option value="$PROJECT_DIR$/shared/eventi" />
|
||||||
<option value="$PROJECT_DIR$/shared/ffmpeg" />
|
|
||||||
</set>
|
</set>
|
||||||
</option>
|
</option>
|
||||||
</GradleProjectSettings>
|
</GradleProjectSettings>
|
||||||
|
|||||||
2
.idea/kotlinc.xml
generated
2
.idea/kotlinc.xml
generated
@ -1,6 +1,6 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project version="4">
|
<project version="4">
|
||||||
<component name="KotlinJpsPluginSettings">
|
<component name="KotlinJpsPluginSettings">
|
||||||
<option name="version" value="2.1.0" />
|
<option name="version" value="1.9.20" />
|
||||||
</component>
|
</component>
|
||||||
</project>
|
</project>
|
||||||
2
.idea/misc.xml
generated
2
.idea/misc.xml
generated
@ -1,5 +1,5 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<project version="4">
|
<project version="4">
|
||||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" project-jdk-name="azul-17" project-jdk-type="JavaSDK" />
|
<component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="azul-17" project-jdk-type="JavaSDK" />
|
||||||
</project>
|
</project>
|
||||||
26
.idea/runConfigurations/UIApplicationKt.xml
generated
26
.idea/runConfigurations/UIApplicationKt.xml
generated
@ -1,26 +0,0 @@
|
|||||||
<component name="ProjectRunConfigurationManager">
|
|
||||||
<configuration default="false" name="UIApplicationKt" type="JetRunConfigurationType" nameIsGenerated="true">
|
|
||||||
<envs>
|
|
||||||
<env name="DATABASE_ADDRESS" value="192.168.2.250" />
|
|
||||||
<env name="DATABASE_NAME_E" value="eventsV4" />
|
|
||||||
<env name="DATABASE_NAME_S" value="streamitv3" />
|
|
||||||
<env name="DATABASE_PASSWORD" value="shFZ27eL2x2NoxyEDBMfDWkvFO" />
|
|
||||||
<env name="DATABASE_PORT" value="3306" />
|
|
||||||
<env name="DATABASE_USERNAME" value="root" />
|
|
||||||
<env name="DIRECTORY_CONTENT_INCOMING" value="G:\MediaProcessingPlayground\input" />
|
|
||||||
<env name="DIRECTORY_CONTENT_OUTGOING" value="G:\MediaProcessingPlayground\output" />
|
|
||||||
<env name="DISABLE_COMPLETE" value="true" />
|
|
||||||
<env name="DISABLE_PRODUCE" value="true" />
|
|
||||||
<env name="EncoderWs" value="ws://192.168.2.250:6081/ws" />
|
|
||||||
<env name="METADATA_TIMEOUT" value="0" />
|
|
||||||
<env name="SUPPORTING_EXECUTABLE_FFMPEG" value="G:\MediaProcessingPlayground\ffmpeg.exe" />
|
|
||||||
<env name="SUPPORTING_EXECUTABLE_FFPROBE" value="G:\MediaProcessingPlayground\ffprobe.exe" />
|
|
||||||
</envs>
|
|
||||||
<option name="MAIN_CLASS_NAME" value="no.iktdev.mediaprocessing.ui.UIApplicationKt" />
|
|
||||||
<module name="MediaProcessing.apps.ui.main" />
|
|
||||||
<shortenClasspath name="NONE" />
|
|
||||||
<method v="2">
|
|
||||||
<option name="Make" enabled="true" />
|
|
||||||
</method>
|
|
||||||
</configuration>
|
|
||||||
</component>
|
|
||||||
1182
.idea/workspace.xml
generated
1182
.idea/workspace.xml
generated
File diff suppressed because it is too large
Load Diff
@ -20,5 +20,5 @@ tasks.test {
|
|||||||
useJUnitPlatform()
|
useJUnitPlatform()
|
||||||
}
|
}
|
||||||
kotlin {
|
kotlin {
|
||||||
jvmToolchain(21)
|
jvmToolchain(17)
|
||||||
}
|
}
|
||||||
@ -1,8 +1,9 @@
|
|||||||
plugins {
|
plugins {
|
||||||
id("java")
|
id("java")
|
||||||
kotlin("jvm")
|
kotlin("jvm")
|
||||||
id("org.springframework.boot")
|
kotlin("plugin.spring") version "1.5.31"
|
||||||
id("io.spring.dependency-management")
|
id("org.springframework.boot") version "2.5.5"
|
||||||
|
id("io.spring.dependency-management") version "1.0.11.RELEASE"
|
||||||
}
|
}
|
||||||
|
|
||||||
group = "no.iktdev.mediaprocessing.apps"
|
group = "no.iktdev.mediaprocessing.apps"
|
||||||
@ -26,41 +27,40 @@ repositories {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
val exposedVersion = "0.44.0"
|
||||||
dependencies {
|
dependencies {
|
||||||
|
|
||||||
/*Spring boot*/
|
/*Spring boot*/
|
||||||
implementation("org.springframework.boot:spring-boot-starter-web")
|
implementation("org.springframework.boot:spring-boot-starter-web")
|
||||||
implementation("org.springframework.boot:spring-boot-starter-actuator")
|
implementation("org.springframework.boot:spring-boot-starter:2.7.0")
|
||||||
implementation("org.springframework.boot:spring-boot-starter-websocket")
|
// implementation("org.springframework.kafka:spring-kafka:3.0.1")
|
||||||
implementation("org.springframework:spring-tx")
|
implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3")
|
||||||
|
implementation("org.springframework.kafka:spring-kafka:2.8.5")
|
||||||
|
|
||||||
|
|
||||||
|
implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
|
||||||
|
implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
|
||||||
|
implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
|
||||||
|
implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion")
|
||||||
|
implementation ("mysql:mysql-connector-java:8.0.29")
|
||||||
|
|
||||||
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
|
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
|
||||||
implementation("com.google.code.gson:gson:2.8.9")
|
implementation("com.google.code.gson:gson:2.8.9")
|
||||||
implementation("org.json:json:20210307")
|
implementation("org.json:json:20210307")
|
||||||
|
|
||||||
implementation(libs.exfl)
|
implementation("no.iktdev:exfl:0.0.16-SNAPSHOT")
|
||||||
implementation("no.iktdev.library:subtitle:1.8.1-SNAPSHOT")
|
implementation("no.iktdev.library:subtitle:1.7.9-SNAPSHOT")
|
||||||
implementation(libs.eventi)
|
|
||||||
|
|
||||||
|
|
||||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
|
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
|
||||||
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
|
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
|
||||||
implementation("com.github.pgreze:kotlin-process:1.4.1")
|
implementation("com.github.pgreze:kotlin-process:1.4.1")
|
||||||
|
|
||||||
|
implementation(project(mapOf("path" to ":shared:eventi")))
|
||||||
implementation(project(mapOf("path" to ":shared:common")))
|
implementation(project(mapOf("path" to ":shared:common")))
|
||||||
implementation(project(mapOf("path" to ":shared:database")))
|
|
||||||
|
|
||||||
implementation(kotlin("stdlib-jdk8"))
|
implementation(kotlin("stdlib-jdk8"))
|
||||||
|
|
||||||
testImplementation("io.mockk:mockk:1.12.0")
|
|
||||||
testImplementation("org.springframework.boot:spring-boot-starter-test")
|
|
||||||
testImplementation(project(":shared:common", configuration = "testArtifacts"))
|
|
||||||
testImplementation(project(":shared:database", configuration = "testArtifacts"))
|
|
||||||
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
|
|
||||||
|
|
||||||
val exposedVersion = "0.61.0"
|
|
||||||
testImplementation("org.jetbrains.exposed:exposed-core:${exposedVersion}")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tasks.test {
|
tasks.test {
|
||||||
@ -78,5 +78,5 @@ tasks.jar {
|
|||||||
}
|
}
|
||||||
|
|
||||||
kotlin {
|
kotlin {
|
||||||
jvmToolchain(21)
|
jvmToolchain(17)
|
||||||
}
|
}
|
||||||
@ -1,28 +1,37 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
package no.iktdev.mediaprocessing.converter
|
||||||
|
|
||||||
import mu.KotlinLogging
|
import mu.KotlinLogging
|
||||||
import no.iktdev.eventi.events.EventTypeRegistry
|
|
||||||
import no.iktdev.eventi.tasks.TaskTypeRegistry
|
|
||||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||||
import no.iktdev.exfl.observable.Observables
|
import no.iktdev.exfl.observable.Observables
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry
|
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry
|
import no.iktdev.eventi.database.MySqlDataSource
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.tables.runners
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.tables.tasks
|
||||||
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
||||||
import no.iktdev.mediaprocessing.shared.database.DatabaseApplication
|
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||||
import no.iktdev.mediaprocessing.shared.database.DatabasebasedMediaProcessingApp
|
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||||
import org.springframework.boot.runApplication
|
import org.springframework.boot.runApplication
|
||||||
import org.springframework.context.annotation.Configuration
|
|
||||||
|
|
||||||
@DatabasebasedMediaProcessingApp
|
@SpringBootApplication
|
||||||
open class ConverterApplication: DatabaseApplication() {
|
class ConvertApplication
|
||||||
}
|
|
||||||
|
|
||||||
val ioCoroutine = CoroutinesIO()
|
val ioCoroutine = CoroutinesIO()
|
||||||
val defaultCoroutine = CoroutinesDefault()
|
val defaultCoroutine = CoroutinesDefault()
|
||||||
|
|
||||||
|
lateinit var taskManager: TasksManager
|
||||||
|
lateinit var runnerManager: RunnerManager
|
||||||
|
|
||||||
|
|
||||||
|
private lateinit var eventsDatabase: MySqlDataSource
|
||||||
private val log = KotlinLogging.logger {}
|
private val log = KotlinLogging.logger {}
|
||||||
|
|
||||||
|
fun getEventsDatabase(): MySqlDataSource {
|
||||||
|
return eventsDatabase
|
||||||
|
}
|
||||||
|
|
||||||
fun main(args: Array<String>) {
|
fun main(args: Array<String>) {
|
||||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||||
override fun onUpdated(value: Throwable) {
|
override fun onUpdated(value: Throwable) {
|
||||||
@ -36,19 +45,15 @@ fun main(args: Array<String>) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
runApplication<ConverterApplication>(*args)
|
eventsDatabase = DatabaseEnvConfig.toEventsDatabase()
|
||||||
|
eventsDatabase.createDatabase()
|
||||||
|
eventsDatabase.createTables(tasks, runners)
|
||||||
|
taskManager = TasksManager(eventsDatabase)
|
||||||
|
|
||||||
|
runnerManager = RunnerManager(dataSource = getEventsDatabase(), name = ConvertApplication::class.java.simpleName)
|
||||||
|
runnerManager.assignRunner()
|
||||||
|
|
||||||
|
runApplication<ConvertApplication>(*args)
|
||||||
log.info { "App Version: ${getAppVersion()}" }
|
log.info { "App Version: ${getAppVersion()}" }
|
||||||
}
|
}
|
||||||
//private val logger = KotlinLogging.logger {}
|
//private val logger = KotlinLogging.logger {}
|
||||||
|
|
||||||
@Configuration
|
|
||||||
open class ApplicationConfiguration() {
|
|
||||||
init {
|
|
||||||
EventRegistry.getEvents().let {
|
|
||||||
EventTypeRegistry.register(it)
|
|
||||||
}
|
|
||||||
TaskRegistry.getTasks().let {
|
|
||||||
TaskTypeRegistry.register(it)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt
Executable file → Normal file
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/ConverterEnv.kt
Executable file → Normal file
@ -1,10 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
|
||||||
|
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
interface ConverterEnvironment {
|
|
||||||
fun canRead(file: File): Boolean
|
|
||||||
fun getReader(file: File): BaseReader?
|
|
||||||
fun createExporter(input: File, outputDir: File, name: String): Exporter
|
|
||||||
}
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
|
||||||
|
|
||||||
import no.iktdev.library.subtitle.classes.Dialog
|
|
||||||
import no.iktdev.library.subtitle.export.Export
|
|
||||||
|
|
||||||
class ExportAdapter(
|
|
||||||
private val export: Export
|
|
||||||
) : Exporter {
|
|
||||||
|
|
||||||
override fun write(dialogs: List<Dialog>) = export.write(dialogs)
|
|
||||||
override fun writeSrt(dialogs: List<Dialog>) = export.writeSrt(dialogs)
|
|
||||||
override fun writeSmi(dialogs: List<Dialog>) = export.writeSmi(dialogs)
|
|
||||||
override fun writeVtt(dialogs: List<Dialog>) = export.writeVtt(dialogs)
|
|
||||||
}
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
|
||||||
|
|
||||||
import no.iktdev.library.subtitle.classes.Dialog
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
interface Exporter {
|
|
||||||
fun write(dialogs: List<Dialog>): MutableList<File>
|
|
||||||
fun writeSrt(dialogs: List<Dialog>): File
|
|
||||||
fun writeSmi(dialogs: List<Dialog>): File
|
|
||||||
fun writeVtt(dialogs: List<Dialog>): File
|
|
||||||
}
|
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
package no.iktdev.mediaprocessing.converter
|
||||||
|
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||||
|
import org.springframework.context.annotation.Configuration
|
||||||
|
import org.springframework.context.annotation.Import
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
class SocketLocalInit: SocketImplementation()
|
||||||
@ -0,0 +1,84 @@
|
|||||||
|
package no.iktdev.mediaprocessing.converter
|
||||||
|
|
||||||
|
import mu.KotlinLogging
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.*
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.ActiveMode
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.task.TaskType
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||||
|
import org.springframework.scheduling.annotation.EnableScheduling
|
||||||
|
import org.springframework.stereotype.Service
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@EnableScheduling
|
||||||
|
class TaskCoordinator(): TaskCoordinatorBase() {
|
||||||
|
private val log = KotlinLogging.logger {}
|
||||||
|
override fun onProduceEvent(event: Event) {
|
||||||
|
taskManager.produceEvent(event)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun onCoordinatorReady() {
|
||||||
|
super.onCoordinatorReady()
|
||||||
|
runnerManager = RunnerManager(dataSource = getEventsDatabase(), name = ConvertApplication::class.java.simpleName)
|
||||||
|
runnerManager.assignRunner()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
override val taskAvailabilityEventListener: MutableMap<TaskType, MutableList<TaskQueueListener>> = mutableMapOf(
|
||||||
|
TaskType.Convert to mutableListOf(),
|
||||||
|
)
|
||||||
|
|
||||||
|
private val taskListeners: MutableSet<TaskEvents> = mutableSetOf()
|
||||||
|
fun getTaskListeners(): List<TaskEvents> {
|
||||||
|
return taskListeners.toList()
|
||||||
|
}
|
||||||
|
fun addTaskEventListener(listener: TaskEvents) {
|
||||||
|
taskListeners.add(listener)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun addConvertTaskListener(listener: TaskQueueListener) {
|
||||||
|
addTaskListener(TaskType.Convert, listener)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun addTaskListener(type: TaskType, listener: TaskQueueListener) {
|
||||||
|
super.addTaskListener(type, listener)
|
||||||
|
pullForAvailableTasks()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
override fun pullForAvailableTasks() {
|
||||||
|
if (runnerManager.iAmSuperseded()) {
|
||||||
|
// This will let the application complete but not consume new
|
||||||
|
taskMode = ActiveMode.Passive
|
||||||
|
return
|
||||||
|
}
|
||||||
|
val available = taskManager.getClaimableTasks().asClaimable()
|
||||||
|
available.forEach { (type, list) ->
|
||||||
|
taskAvailabilityEventListener[type]?.forEach { listener ->
|
||||||
|
list.foreachOrUntilClaimed {
|
||||||
|
listener.onTaskAvailable(it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun clearExpiredClaims() {
|
||||||
|
val expiredClaims = taskManager.getTasksWithExpiredClaim().filter { it.task in listOf(TaskType.Convert) }
|
||||||
|
expiredClaims.forEach {
|
||||||
|
log.info { "Found event with expired claim: ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||||
|
}
|
||||||
|
expiredClaims.forEach {
|
||||||
|
val result = taskManager.deleteTaskClaim(referenceId = it.referenceId, eventId = it.eventId)
|
||||||
|
if (result) {
|
||||||
|
log.info { "Released claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||||
|
} else {
|
||||||
|
log.error { "Failed to release claim on ${it.referenceId}::${it.eventId}::${it.task}" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TaskEvents {
|
||||||
|
fun onCancelOrStopProcess(eventId: String)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -1,76 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
|
||||||
|
|
||||||
import kotlinx.coroutines.CoroutineScope
|
|
||||||
import kotlinx.coroutines.Dispatchers
|
|
||||||
import kotlinx.coroutines.launch
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskPollerImplementation
|
|
||||||
import no.iktdev.eventi.tasks.TaskReporter
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
import org.springframework.boot.ApplicationArguments
|
|
||||||
import org.springframework.boot.ApplicationRunner
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import org.springframework.stereotype.Service
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class PollerAdministrator(
|
|
||||||
private val taskPoller: TaskPoller,
|
|
||||||
): ApplicationRunner {
|
|
||||||
override fun run(args: ApplicationArguments?) {
|
|
||||||
CoroutineScope(Dispatchers.Default).launch {
|
|
||||||
taskPoller.start()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Service
|
|
||||||
class TaskPoller(
|
|
||||||
private val reporter: TaskReporter,
|
|
||||||
) : TaskPollerImplementation(
|
|
||||||
taskStore = TaskStore,
|
|
||||||
reporterFactory = { reporter } // én reporter brukes for alle tasks
|
|
||||||
) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class DefaultTaskReporter() : TaskReporter {
|
|
||||||
override fun markClaimed(taskId: UUID, workerId: String) {
|
|
||||||
TaskStore.claim(taskId, workerId)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun updateLastSeen(taskId: UUID) {
|
|
||||||
TaskStore.heartbeat(taskId)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun markCompleted(taskId: UUID) {
|
|
||||||
TaskStore.markConsumed(taskId, TaskStatus.Completed)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun markFailed(referenceId: UUID, taskId: UUID) {
|
|
||||||
TaskStore.markConsumed(taskId, TaskStatus.Failed)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun markCancelled(referenceId: UUID, taskId: UUID) {
|
|
||||||
TaskStore.markConsumed(taskId, TaskStatus.Cancelled)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
override fun updateProgress(taskId: UUID, progress: Int) {
|
|
||||||
// Not to be implemented for this application
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun log(taskId: UUID, message: String) {
|
|
||||||
// Not to be implemented for this application
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun publishEvent(event: Event) {
|
|
||||||
EventStore.persist(event)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,28 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter.controller
|
|
||||||
|
|
||||||
import org.springframework.boot.actuate.health.HealthEndpoint
|
|
||||||
import org.springframework.boot.actuate.health.Status
|
|
||||||
import org.springframework.http.HttpStatus
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
|
||||||
import org.springframework.web.bind.annotation.RestController
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/system")
|
|
||||||
class ReadinessController(
|
|
||||||
private val healthEndpoint: HealthEndpoint
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping("/ready")
|
|
||||||
fun ready(): ResponseEntity<String> {
|
|
||||||
val health = healthEndpoint.health()
|
|
||||||
|
|
||||||
return if (health.status == Status.UP) {
|
|
||||||
ResponseEntity.ok("READY")
|
|
||||||
} else {
|
|
||||||
ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
|
|
||||||
.body("NOT_READY: ${health.status}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt
Executable file → Normal file
0
apps/converter/src/main/kotlin/no/iktdev/mediaprocessing/converter/convert/ConvertListener.kt
Executable file → Normal file
@ -1,18 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter.convert
|
|
||||||
|
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
|
||||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
abstract class Converter(val env: ConverterEnvironment, val listener: ConvertListener) {
|
|
||||||
var writtenUris: List<String>? = null
|
|
||||||
|
|
||||||
abstract fun getSubtitleReader(useFile: File): BaseReader?
|
|
||||||
abstract suspend fun convert(data: ConvertTask.Data)
|
|
||||||
|
|
||||||
class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException()
|
|
||||||
class FileUnavailableException(override val message: String): RuntimeException()
|
|
||||||
|
|
||||||
abstract fun getResult(): List<String>
|
|
||||||
}
|
|
||||||
@ -4,70 +4,75 @@ import no.iktdev.library.subtitle.Configuration
|
|||||||
import no.iktdev.library.subtitle.Syncro
|
import no.iktdev.library.subtitle.Syncro
|
||||||
import no.iktdev.library.subtitle.classes.Dialog
|
import no.iktdev.library.subtitle.classes.Dialog
|
||||||
import no.iktdev.library.subtitle.classes.DialogType
|
import no.iktdev.library.subtitle.classes.DialogType
|
||||||
|
import no.iktdev.library.subtitle.export.Export
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
import no.iktdev.library.subtitle.reader.BaseReader
|
||||||
|
import no.iktdev.library.subtitle.reader.Reader
|
||||||
import no.iktdev.mediaprocessing.converter.ConverterEnv
|
import no.iktdev.mediaprocessing.converter.ConverterEnv
|
||||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
import no.iktdev.mediaprocessing.shared.common.contract.dto.SubtitleFormats
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
|
|
||||||
import java.io.File
|
import java.io.File
|
||||||
|
import kotlin.jvm.Throws
|
||||||
|
|
||||||
class Converter2(
|
class Converter2(val data: ConvertData,
|
||||||
env: ConverterEnvironment,
|
private val listener: ConvertListener) {
|
||||||
listener: ConvertListener
|
|
||||||
): Converter(env = env, listener = listener) {
|
|
||||||
|
|
||||||
@Throws(FileUnavailableException::class)
|
@Throws(FileUnavailableException::class)
|
||||||
override fun getSubtitleReader(useFile: File): BaseReader? {
|
private fun getReader(): BaseReader? {
|
||||||
if (!env.canRead(useFile)) {
|
val file = File(data.inputFile)
|
||||||
|
if (!file.canRead())
|
||||||
throw FileUnavailableException("Can't open file for reading..")
|
throw FileUnavailableException("Can't open file for reading..")
|
||||||
}
|
return Reader(file).getSubtitleReader()
|
||||||
return env.getReader(useFile)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun syncDialogs(input: List<Dialog>): List<Dialog> {
|
private fun syncDialogs(input: List<Dialog>): List<Dialog> {
|
||||||
return if (ConverterEnv.syncDialogs) Syncro().sync(input) else input
|
return if (ConverterEnv.syncDialogs) Syncro().sync(input) else input
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun canRead(): Boolean {
|
||||||
|
try {
|
||||||
|
val reader = getReader()
|
||||||
|
return reader != null
|
||||||
|
} catch (e: FileUnavailableException) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Throws(FileUnavailableException::class, FileIsNullOrEmpty::class)
|
@Throws(FileUnavailableException::class, FileIsNullOrEmpty::class)
|
||||||
override suspend fun convert(data: ConvertTask.Data) {
|
fun execute() {
|
||||||
val file = File(data.inputFile)
|
val file = File(data.inputFile)
|
||||||
listener.onStarted(file.absolutePath)
|
listener.onStarted(file.absolutePath)
|
||||||
try {
|
try {
|
||||||
Configuration.exportJson = true
|
Configuration.exportJson = true
|
||||||
val read = getSubtitleReader(file)?.read() ?: throw FileIsNullOrEmpty()
|
val read = getReader()?.read() ?: throw FileIsNullOrEmpty()
|
||||||
if (read.isEmpty())
|
if (read.isEmpty())
|
||||||
throw FileIsNullOrEmpty()
|
throw FileIsNullOrEmpty()
|
||||||
val filtered = read.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) }
|
val filtered = read.filter { !it.ignore && it.type !in listOf(DialogType.SIGN_SONG, DialogType.CAPTION) }
|
||||||
val syncOrNotSync = syncDialogs(filtered)
|
val syncOrNotSync = syncDialogs(filtered)
|
||||||
|
|
||||||
val exporter = env.createExporter(file, File(data.outputDirectory), data.outputFileName)
|
val exporter = Export(file, File(data.outputDirectory), data.outputFileName)
|
||||||
|
|
||||||
val outFiles = if (data.formats.isEmpty()) {
|
val outFiles = if (data.formats.isEmpty()) {
|
||||||
exporter.write(syncOrNotSync)
|
exporter.write(syncOrNotSync)
|
||||||
} else {
|
} else {
|
||||||
val exported = mutableListOf<File>()
|
val exported = mutableListOf<File>()
|
||||||
if (data.formats.contains(SubtitleFormat.SRT)) {
|
if (data.formats.contains(SubtitleFormats.SRT)) {
|
||||||
exported.add(exporter.writeSrt(syncOrNotSync))
|
exported.add(exporter.writeSrt(syncOrNotSync))
|
||||||
}
|
}
|
||||||
if (data.formats.contains(SubtitleFormat.SMI)) {
|
if (data.formats.contains(SubtitleFormats.SMI)) {
|
||||||
exported.add(exporter.writeSmi(syncOrNotSync))
|
exported.add(exporter.writeSmi(syncOrNotSync))
|
||||||
}
|
}
|
||||||
if (data.formats.contains(SubtitleFormat.VTT)) {
|
if (data.formats.contains(SubtitleFormats.VTT)) {
|
||||||
exported.add(exporter.writeVtt(syncOrNotSync))
|
exported.add(exporter.writeVtt(syncOrNotSync))
|
||||||
}
|
}
|
||||||
exported
|
exported
|
||||||
}
|
}
|
||||||
writtenUris = outFiles.map { it.absolutePath }
|
listener.onCompleted(file.absolutePath, outFiles.map { it.absolutePath })
|
||||||
listener.onCompleted(file.absolutePath, writtenUris!!)
|
|
||||||
} catch (e: Exception) {
|
} catch (e: Exception) {
|
||||||
listener.onError(file.absolutePath, e.message ?: e.localizedMessage)
|
listener.onError(file.absolutePath, e.message ?: e.localizedMessage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun getResult(): List<String> {
|
|
||||||
if (writtenUris == null) {
|
class FileIsNullOrEmpty(override val message: String? = "File read is null or empty"): RuntimeException()
|
||||||
throw IllegalStateException("Execute must be called before getting result")
|
class FileUnavailableException(override val message: String): RuntimeException()
|
||||||
}
|
|
||||||
return writtenUris!!
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@ -1,117 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter.listeners
|
|
||||||
|
|
||||||
import kotlinx.coroutines.Dispatchers
|
|
||||||
import kotlinx.coroutines.withContext
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.Task
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskListener
|
|
||||||
import no.iktdev.eventi.tasks.TaskType
|
|
||||||
import no.iktdev.library.subtitle.export.Export
|
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
|
||||||
import no.iktdev.library.subtitle.reader.Reader
|
|
||||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
|
||||||
import no.iktdev.mediaprocessing.converter.ExportAdapter
|
|
||||||
import no.iktdev.mediaprocessing.converter.Exporter
|
|
||||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
|
||||||
import no.iktdev.mediaprocessing.converter.convert.Converter
|
|
||||||
import no.iktdev.mediaprocessing.converter.convert.Converter2
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.io.File
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
open class ConvertTaskListener(): TaskListener(TaskType.CPU_INTENSIVE) {
|
|
||||||
|
|
||||||
override fun getWorkerId(): String {
|
|
||||||
return "${this::class.java.simpleName}-${TaskType.CPU_INTENSIVE}-${UUID.randomUUID()}"
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun supports(task: Task): Boolean {
|
|
||||||
return task is ConvertTask
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
override suspend fun onTask(task: Task): Event? {
|
|
||||||
if (task !is ConvertTask) {
|
|
||||||
throw IllegalArgumentException("Invalid task type: ${task::class.java.name}")
|
|
||||||
}
|
|
||||||
val converter = getConverter()
|
|
||||||
|
|
||||||
withHeartbeatRunner {
|
|
||||||
reporter?.updateLastSeen(task.taskId)
|
|
||||||
}
|
|
||||||
|
|
||||||
withContext(Dispatchers.Unconfined) {
|
|
||||||
converter.convert(task.data)
|
|
||||||
}
|
|
||||||
|
|
||||||
return try {
|
|
||||||
val result = converter.getResult()
|
|
||||||
val newEvent = ConvertTaskResultEvent(
|
|
||||||
data = ConvertTaskResultEvent.ConvertedData(
|
|
||||||
language = task.data.language,
|
|
||||||
outputFiles = result,
|
|
||||||
baseName = task.data.outputFileName
|
|
||||||
),
|
|
||||||
status = TaskStatus.Completed
|
|
||||||
).producedFrom(task)
|
|
||||||
newEvent
|
|
||||||
} catch (e: Exception) {
|
|
||||||
e.printStackTrace()
|
|
||||||
val newEvent = ConvertTaskResultEvent(
|
|
||||||
data = null,
|
|
||||||
status = TaskStatus.Failed
|
|
||||||
).producedFrom(task)
|
|
||||||
newEvent
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun createIncompleteStateTaskEvent(
|
|
||||||
task: Task,
|
|
||||||
status: TaskStatus,
|
|
||||||
exception: Exception?
|
|
||||||
): Event {
|
|
||||||
val message = when (status) {
|
|
||||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
|
||||||
TaskStatus.Cancelled -> "Canceled"
|
|
||||||
else -> ""
|
|
||||||
}
|
|
||||||
return ConvertTaskResultEvent(null, status, error = message)
|
|
||||||
}
|
|
||||||
|
|
||||||
open fun getConverter(): Converter {
|
|
||||||
return Converter2(getConverterEnvironment(), getListener())
|
|
||||||
}
|
|
||||||
|
|
||||||
open fun getListener(): ConvertListener {
|
|
||||||
return DefaultConvertListener()
|
|
||||||
}
|
|
||||||
|
|
||||||
class DefaultConvertListener: ConvertListener {
|
|
||||||
override fun onStarted(inputFile: String) {
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
open fun getConverterEnvironment(): ConverterEnvironment {
|
|
||||||
return DefaultConverterEnvironment()
|
|
||||||
}
|
|
||||||
|
|
||||||
class DefaultConverterEnvironment : ConverterEnvironment {
|
|
||||||
override fun canRead(file: File) = file.canRead()
|
|
||||||
|
|
||||||
override fun getReader(file: File): BaseReader? =
|
|
||||||
Reader(file).getSubtitleReader()
|
|
||||||
|
|
||||||
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
|
|
||||||
return ExportAdapter(Export(input, outputDir, name))
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -0,0 +1,133 @@
|
|||||||
|
package no.iktdev.mediaprocessing.converter.tasks
|
||||||
|
|
||||||
|
import kotlinx.coroutines.delay
|
||||||
|
import kotlinx.coroutines.runBlocking
|
||||||
|
import mu.KotlinLogging
|
||||||
|
import no.iktdev.eventi.data.EventMetadata
|
||||||
|
import no.iktdev.eventi.data.EventStatus
|
||||||
|
import no.iktdev.mediaprocessing.converter.*
|
||||||
|
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
||||||
|
import no.iktdev.mediaprocessing.converter.convert.Converter2
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.Status
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.services.TaskService
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.task.Task
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertData
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertWorkPerformed
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.ConvertedData
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.stereotype.Service
|
||||||
|
|
||||||
|
|
||||||
|
@Service
|
||||||
|
class ConvertService(
|
||||||
|
@Autowired var tasks: TaskCoordinator,
|
||||||
|
) : TaskService(), ConvertListener, TaskCoordinator.TaskEvents {
|
||||||
|
|
||||||
|
fun getProducerName(): String {
|
||||||
|
return this::class.java.simpleName
|
||||||
|
}
|
||||||
|
|
||||||
|
override val log = KotlinLogging.logger {}
|
||||||
|
override val logDir = ConverterEnv.logDirectory
|
||||||
|
|
||||||
|
override fun getServiceId(serviceName: String): String {
|
||||||
|
return super.getServiceId(this::class.java.simpleName)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
var worker: Converter2? = null
|
||||||
|
|
||||||
|
override fun onAttachListener() {
|
||||||
|
tasks.addConvertTaskListener(this)
|
||||||
|
tasks.addTaskEventListener(this)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
override fun isReadyToConsume(): Boolean {
|
||||||
|
return worker == null
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun isTaskClaimable(task: Task): Boolean {
|
||||||
|
return !taskManager.isTaskClaimed(referenceId = task.referenceId, eventId = task.eventId)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
override fun onTaskAssigned(task: Task) {
|
||||||
|
startConvert(task)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun startConvert(task: Task) {
|
||||||
|
val convert = task.data as ConvertData
|
||||||
|
worker = Converter2(convert, this)
|
||||||
|
worker?.execute()
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun onStarted(inputFile: String) {
|
||||||
|
val task = assignedTask ?: return
|
||||||
|
taskManager.markTaskAsClaimed(task.referenceId, task.eventId, serviceId)
|
||||||
|
log.info { "Convert started for ${task.referenceId}" }
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
||||||
|
val task = assignedTask ?: return
|
||||||
|
log.info { "Convert completed for ${task.referenceId}" }
|
||||||
|
val claimSuccessful = taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||||
|
|
||||||
|
runBlocking {
|
||||||
|
delay(1000)
|
||||||
|
if (!claimSuccessful) {
|
||||||
|
taskManager.markTaskAsCompleted(task.referenceId, task.eventId)
|
||||||
|
delay(1000)
|
||||||
|
}
|
||||||
|
var readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||||
|
while (!readbackIsSuccess) {
|
||||||
|
delay(1000)
|
||||||
|
readbackIsSuccess = taskManager.isTaskCompleted(task.referenceId, task.eventId)
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.onProduceEvent(ConvertWorkPerformed(
|
||||||
|
metadata = EventMetadata(
|
||||||
|
referenceId = task.referenceId,
|
||||||
|
derivedFromEventId = task.eventId,
|
||||||
|
status = EventStatus.Success,
|
||||||
|
source = getProducerName()
|
||||||
|
),
|
||||||
|
data = ConvertedData(
|
||||||
|
outputFiles = outputFiles
|
||||||
|
)
|
||||||
|
))
|
||||||
|
onClearTask()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun onError(inputFile: String, message: String) {
|
||||||
|
val task = assignedTask ?: return
|
||||||
|
super.onError(inputFile, message)
|
||||||
|
log.info { "Convert error for ${task.referenceId}\nmessage: $message" }
|
||||||
|
|
||||||
|
taskManager.markTaskAsCompleted(task.referenceId, task.eventId, Status.ERROR)
|
||||||
|
|
||||||
|
tasks.onProduceEvent(ConvertWorkPerformed(
|
||||||
|
metadata = EventMetadata(
|
||||||
|
referenceId = task.referenceId,
|
||||||
|
derivedFromEventId = task.eventId,
|
||||||
|
status = EventStatus.Failed,
|
||||||
|
source = getProducerName()
|
||||||
|
)
|
||||||
|
))
|
||||||
|
onClearTask()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
override fun onClearTask() {
|
||||||
|
super.onClearTask()
|
||||||
|
worker = null
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun onCancelOrStopProcess(eventId: String) {
|
||||||
|
TODO("Not yet implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
3
apps/converter/src/main/resources/application.properties
Normal file
3
apps/converter/src/main/resources/application.properties
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
spring.output.ansi.enabled=always
|
||||||
|
logging.level.org.apache.kafka=WARN
|
||||||
|
logging.level.root=INFO
|
||||||
@ -1,33 +0,0 @@
|
|||||||
spring:
|
|
||||||
output:
|
|
||||||
ansi:
|
|
||||||
enabled: always
|
|
||||||
flyway:
|
|
||||||
enabled: true
|
|
||||||
locations: classpath:flyway
|
|
||||||
baseline-on-migrate: false
|
|
||||||
|
|
||||||
management:
|
|
||||||
endpoints:
|
|
||||||
web:
|
|
||||||
exposure:
|
|
||||||
include: health
|
|
||||||
endpoint:
|
|
||||||
health:
|
|
||||||
show-details: always
|
|
||||||
|
|
||||||
logging:
|
|
||||||
level:
|
|
||||||
root: INFO
|
|
||||||
org.apache.kafka: INFO
|
|
||||||
Exposed: OFF
|
|
||||||
org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN
|
|
||||||
|
|
||||||
|
|
||||||
media:
|
|
||||||
cache: /src/cache
|
|
||||||
outgoing: /src/output
|
|
||||||
incoming: /src/input
|
|
||||||
|
|
||||||
streamit:
|
|
||||||
address: http://streamit.service
|
|
||||||
@ -1,82 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
|
||||||
|
|
||||||
import io.mockk.every
|
|
||||||
import io.mockk.mockk
|
|
||||||
import io.mockk.mockkObject
|
|
||||||
import io.mockk.verify
|
|
||||||
import no.iktdev.eventi.models.Task
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.TestBase
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.config.DatasourceConfiguration
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
|
|
||||||
import org.jetbrains.exposed.sql.Database
|
|
||||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
|
||||||
import org.junit.jupiter.api.Test
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith
|
|
||||||
import org.springframework.boot.builder.SpringApplicationBuilder
|
|
||||||
import org.springframework.boot.test.context.SpringBootTest
|
|
||||||
import org.springframework.test.context.TestPropertySource
|
|
||||||
import org.springframework.test.context.junit.jupiter.SpringExtension
|
|
||||||
import javax.sql.DataSource
|
|
||||||
|
|
||||||
@SpringBootTest(
|
|
||||||
classes = [ConverterApplication::class,
|
|
||||||
DatasourceConfiguration::class],
|
|
||||||
webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT
|
|
||||||
)
|
|
||||||
@TestPropertySource(properties = ["spring.flyway.enabled=true"])
|
|
||||||
@ExtendWith(SpringExtension::class)
|
|
||||||
class ConverterApplicationTest : TestBase() {
|
|
||||||
|
|
||||||
data class TestTask(
|
|
||||||
val success: Boolean
|
|
||||||
) : Task()
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
fun `context loads and common configuration is available`() {
|
|
||||||
// Hvis du har beans du vil verifisere, kan du autowire dem her
|
|
||||||
// @Autowired lateinit var database: Database
|
|
||||||
|
|
||||||
// Dummy assertion for å verifisere at konteksten starter
|
|
||||||
assertNotNull(Unit)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
fun `Verify that we can access TaskStore`() {
|
|
||||||
val tasks = TaskStore.getPendingTasks()
|
|
||||||
assertNotNull(tasks)
|
|
||||||
assert(tasks.isEmpty())
|
|
||||||
|
|
||||||
TaskStore.persist(TestTask(success = true).newReferenceId())
|
|
||||||
|
|
||||||
val tasksAfter = TaskStore.getPendingTasks()
|
|
||||||
assertNotNull(tasksAfter)
|
|
||||||
assert(tasksAfter.isNotEmpty())
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
fun `ExposedInitializer should connect to database`() {
|
|
||||||
mockkObject(Database)
|
|
||||||
|
|
||||||
every {
|
|
||||||
Database.connect(
|
|
||||||
any<DataSource>(),
|
|
||||||
any(),
|
|
||||||
any(),
|
|
||||||
any(),
|
|
||||||
any()
|
|
||||||
)
|
|
||||||
} returns mockk()
|
|
||||||
|
|
||||||
|
|
||||||
val context = SpringApplicationBuilder(ConverterApplication::class.java)
|
|
||||||
.properties("spring.main.web-application-type=none")
|
|
||||||
.run()
|
|
||||||
|
|
||||||
verify(exactly = 1) { Database.connect(any<DataSource>(), any(), any(), any(), any()) }
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,38 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
|
||||||
|
|
||||||
import kotlinx.coroutines.delay
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
|
||||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
|
||||||
import no.iktdev.mediaprocessing.converter.convert.Converter
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
class MockConverter(
|
|
||||||
val delayMillis: Long = 0,
|
|
||||||
private val simulatedResult: List<String>? = null,
|
|
||||||
private val taskResultStatus: TaskStatus = TaskStatus.Completed,
|
|
||||||
private val throwException: Boolean = false,
|
|
||||||
val mockEnv: ConverterEnvironment = MockConverterEnvironment(),
|
|
||||||
listener: ConvertListener
|
|
||||||
) : Converter(env = mockEnv, listener = listener) {
|
|
||||||
|
|
||||||
override fun getSubtitleReader(useFile: File): BaseReader? {
|
|
||||||
TODO("Not yet implemented")
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun convert(data: ConvertTask.Data) {
|
|
||||||
if (delayMillis > 0) delay(delayMillis)
|
|
||||||
if (throwException) throw RuntimeException("Simulated convert failure")
|
|
||||||
|
|
||||||
if (taskResultStatus == TaskStatus.Failed) {
|
|
||||||
listener.onError(data.inputFile, "Failed state desired")
|
|
||||||
} else {
|
|
||||||
listener.onCompleted(data.inputFile, simulatedResult!!)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun getResult(): List<String> {
|
|
||||||
return simulatedResult!!
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter
|
|
||||||
|
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
class MockConverterEnvironment(
|
|
||||||
var canReadValue: Boolean = true,
|
|
||||||
var reader: BaseReader? = null,
|
|
||||||
var exporter: Exporter? = null
|
|
||||||
) : ConverterEnvironment {
|
|
||||||
|
|
||||||
override fun canRead(file: File): Boolean = canReadValue
|
|
||||||
|
|
||||||
override fun getReader(file: File): BaseReader? = reader
|
|
||||||
|
|
||||||
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
|
|
||||||
return exporter ?: error("FakeEnv.exporter must be set before calling createExporter")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,249 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter.convert
|
|
||||||
|
|
||||||
import kotlinx.coroutines.test.runTest
|
|
||||||
import no.iktdev.library.subtitle.classes.Dialog
|
|
||||||
import no.iktdev.library.subtitle.classes.DialogType
|
|
||||||
import no.iktdev.library.subtitle.classes.Time
|
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
|
||||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
|
||||||
import no.iktdev.mediaprocessing.converter.Exporter
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
|
|
||||||
import org.junit.jupiter.api.Assertions.assertEquals
|
|
||||||
import org.junit.jupiter.api.DisplayName
|
|
||||||
import org.junit.jupiter.api.Test
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
class Converter2Test {
|
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
// Fake implementations
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class FakeReader(
|
|
||||||
private val dialogs: List<Dialog>,
|
|
||||||
private val shouldThrow: Boolean = false
|
|
||||||
) : BaseReader() {
|
|
||||||
override fun read(): List<Dialog> {
|
|
||||||
if (shouldThrow) throw RuntimeException("reader failed")
|
|
||||||
return dialogs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class FakeExporter(
|
|
||||||
private val srtFile: File? = null,
|
|
||||||
private val smiFile: File? = null,
|
|
||||||
private val vttFile: File? = null,
|
|
||||||
private val filesForWrite: List<File> = emptyList(),
|
|
||||||
private val shouldThrow: Boolean = false
|
|
||||||
) : Exporter {
|
|
||||||
|
|
||||||
override fun write(dialogs: List<Dialog>): MutableList<File> {
|
|
||||||
if (shouldThrow) throw RuntimeException("export failed")
|
|
||||||
return filesForWrite.toMutableList()
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun writeSrt(dialogs: List<Dialog>): File =
|
|
||||||
srtFile ?: error("srtFile not set in FakeExporter")
|
|
||||||
|
|
||||||
override fun writeSmi(dialogs: List<Dialog>): File =
|
|
||||||
smiFile ?: error("smiFile not set in FakeExporter")
|
|
||||||
|
|
||||||
override fun writeVtt(dialogs: List<Dialog>): File =
|
|
||||||
vttFile ?: error("vttFile not set in FakeExporter")
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class FakeListener : ConvertListener {
|
|
||||||
var started = 0
|
|
||||||
var completed = 0
|
|
||||||
var errors = 0
|
|
||||||
var lastError: String? = null
|
|
||||||
|
|
||||||
override fun onStarted(inputFile: String) {
|
|
||||||
started++
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
|
||||||
completed++
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun onError(inputFile: String, message: String) {
|
|
||||||
errors++
|
|
||||||
lastError = message
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class FakeEnv(
|
|
||||||
var canReadValue: Boolean = true,
|
|
||||||
var reader: BaseReader? = null,
|
|
||||||
var exporter: Exporter? = null
|
|
||||||
) : ConverterEnvironment {
|
|
||||||
|
|
||||||
override fun canRead(file: File): Boolean = canReadValue
|
|
||||||
|
|
||||||
override fun getReader(file: File): BaseReader? = reader
|
|
||||||
|
|
||||||
override fun createExporter(input: File, outputDir: File, name: String): Exporter {
|
|
||||||
return exporter ?: error("FakeEnv.exporter must be set before calling createExporter")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
// Helpers
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun makeTaskData(
|
|
||||||
input: String = "input.srt",
|
|
||||||
language: String = "no",
|
|
||||||
outDir: String = "out",
|
|
||||||
outName: String = "name",
|
|
||||||
formats: List<SubtitleFormat> = emptyList(),
|
|
||||||
allowOverwrite: Boolean = true
|
|
||||||
) = ConvertTask.Data(
|
|
||||||
inputFile = input,
|
|
||||||
language = language,
|
|
||||||
outputDirectory = outDir,
|
|
||||||
outputFileName = outName,
|
|
||||||
formats = formats,
|
|
||||||
allowOverwrite = allowOverwrite
|
|
||||||
)
|
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
// Tests
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName(
|
|
||||||
"""
|
|
||||||
Når execute() kjøres
|
|
||||||
Hvis reader returnerer tom liste
|
|
||||||
Så kalles onError
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
fun execute_emptyFile() = runTest {
|
|
||||||
val env = FakeEnv(
|
|
||||||
canReadValue = true,
|
|
||||||
reader = FakeReader(emptyList())
|
|
||||||
)
|
|
||||||
val listener = FakeListener()
|
|
||||||
|
|
||||||
val converter = Converter2(
|
|
||||||
env = env,
|
|
||||||
listener = listener
|
|
||||||
)
|
|
||||||
|
|
||||||
converter.convert(makeTaskData())
|
|
||||||
|
|
||||||
assertEquals(1, listener.errors)
|
|
||||||
assertEquals(0, listener.completed)
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName(
|
|
||||||
"""
|
|
||||||
Når execute() kjøres
|
|
||||||
Hvis reader returnerer dialoger
|
|
||||||
Og exporter.write lykkes
|
|
||||||
Så kalles onCompleted
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
fun execute_success() = runTest {
|
|
||||||
val dialogs = listOf(
|
|
||||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
|
||||||
)
|
|
||||||
|
|
||||||
val env = FakeEnv(
|
|
||||||
canReadValue = true,
|
|
||||||
reader = FakeReader(dialogs),
|
|
||||||
exporter = FakeExporter(srtFile = File("/fake/out.srt"))
|
|
||||||
)
|
|
||||||
|
|
||||||
val listener = FakeListener()
|
|
||||||
|
|
||||||
val converter = Converter2(
|
|
||||||
env = env,
|
|
||||||
listener = listener
|
|
||||||
)
|
|
||||||
|
|
||||||
converter.convert(
|
|
||||||
makeTaskData(
|
|
||||||
formats = listOf(SubtitleFormat.SRT)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assertEquals(1, listener.completed)
|
|
||||||
assertEquals(listOf("/fake/out.srt"), converter.getResult())
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName(
|
|
||||||
"""
|
|
||||||
Når execute() kjøres
|
|
||||||
Hvis exporter.write kaster exception
|
|
||||||
Så kalles onError
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
fun execute_exportFails() = runTest {
|
|
||||||
val dialogs = listOf(
|
|
||||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
|
||||||
)
|
|
||||||
|
|
||||||
val env = FakeEnv(
|
|
||||||
canReadValue = true,
|
|
||||||
reader = FakeReader(dialogs),
|
|
||||||
exporter = FakeExporter(srtFile = File("/fake/out.srt"), shouldThrow = true)
|
|
||||||
)
|
|
||||||
|
|
||||||
val listener = FakeListener()
|
|
||||||
|
|
||||||
val converter = Converter2(
|
|
||||||
env = env,
|
|
||||||
listener = listener
|
|
||||||
)
|
|
||||||
|
|
||||||
converter.convert(makeTaskData())
|
|
||||||
|
|
||||||
assertEquals(1, listener.errors)
|
|
||||||
assertEquals(0, listener.completed)
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName(
|
|
||||||
"""
|
|
||||||
Når execute() kjøres
|
|
||||||
Hvis formats inneholder SRT og VTT
|
|
||||||
Så brukes writeSrt og writeVtt
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
fun execute_multipleFormats() = runTest {
|
|
||||||
val dialogs = listOf(
|
|
||||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
val env = FakeEnv(
|
|
||||||
canReadValue = true,
|
|
||||||
reader = FakeReader(dialogs),
|
|
||||||
exporter = FakeExporter(srtFile = File("/fake/out.srt"), vttFile = File("/fake/out.vtt"))
|
|
||||||
)
|
|
||||||
|
|
||||||
val listener = FakeListener()
|
|
||||||
|
|
||||||
val converter = Converter2(
|
|
||||||
env = env,
|
|
||||||
listener = listener
|
|
||||||
)
|
|
||||||
|
|
||||||
converter.convert(
|
|
||||||
makeTaskData(
|
|
||||||
formats = listOf(SubtitleFormat.SRT, SubtitleFormat.VTT)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
assertEquals(1, listener.completed)
|
|
||||||
assertEquals(listOf("/fake/out.srt", "/fake/out.vtt"), converter.getResult())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,247 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.converter.listeners
|
|
||||||
|
|
||||||
import kotlinx.coroutines.test.runTest
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.Task
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskReporter
|
|
||||||
import no.iktdev.library.subtitle.classes.Dialog
|
|
||||||
import no.iktdev.library.subtitle.classes.DialogType
|
|
||||||
import no.iktdev.library.subtitle.classes.Time
|
|
||||||
import no.iktdev.library.subtitle.reader.BaseReader
|
|
||||||
import no.iktdev.mediaprocessing.converter.ConverterEnvironment
|
|
||||||
import no.iktdev.mediaprocessing.converter.Exporter
|
|
||||||
import no.iktdev.mediaprocessing.converter.MockConverter
|
|
||||||
import no.iktdev.mediaprocessing.converter.MockConverterEnvironment
|
|
||||||
import no.iktdev.mediaprocessing.converter.convert.ConvertListener
|
|
||||||
import no.iktdev.mediaprocessing.converter.convert.Converter
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleFormat
|
|
||||||
import org.junit.jupiter.api.Assertions.*
|
|
||||||
import org.junit.jupiter.api.DisplayName
|
|
||||||
import org.junit.jupiter.api.Test
|
|
||||||
import java.io.File
|
|
||||||
import java.util.*
|
|
||||||
import kotlin.system.measureTimeMillis
|
|
||||||
|
|
||||||
class ConvertTaskListenerTest {
|
|
||||||
|
|
||||||
class ConvertTaskListenerTestImplementation : ConvertTaskListener() {
|
|
||||||
fun getJob() = currentJob
|
|
||||||
|
|
||||||
|
|
||||||
var overrideEnv: ConverterEnvironment = DefaultConverterEnvironment()
|
|
||||||
override fun getConverterEnvironment(): ConverterEnvironment {
|
|
||||||
return overrideEnv
|
|
||||||
}
|
|
||||||
|
|
||||||
var overrideListener: ConvertListener? = null
|
|
||||||
override fun getListener(): ConvertListener {
|
|
||||||
if (overrideListener != null)
|
|
||||||
return overrideListener!!
|
|
||||||
else
|
|
||||||
return super.getListener()
|
|
||||||
}
|
|
||||||
|
|
||||||
var overrideConverter: Converter? = null
|
|
||||||
override fun getConverter(): Converter {
|
|
||||||
return if (overrideConverter != null)
|
|
||||||
overrideConverter!!
|
|
||||||
else
|
|
||||||
super.getConverter()
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
val listener = ConvertTaskListenerTestImplementation()
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
|
||||||
// Fake environment + fake converter
|
|
||||||
// ---------------------------------------------------------------------
|
|
||||||
|
|
||||||
class FakeListener : ConvertListener {
|
|
||||||
var started = 0
|
|
||||||
var completed = 0
|
|
||||||
var errors = 0
|
|
||||||
|
|
||||||
override fun onStarted(inputFile: String) {
|
|
||||||
started++
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun onCompleted(inputFile: String, outputFiles: List<String>) {
|
|
||||||
completed++
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun onError(inputFile: String, message: String) {
|
|
||||||
errors++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class FakeExporter(
|
|
||||||
private val files: List<File>,
|
|
||||||
private val shouldThrow: Boolean = false
|
|
||||||
) : Exporter {
|
|
||||||
|
|
||||||
override fun write(dialogs: List<Dialog>): MutableList<File> {
|
|
||||||
if (shouldThrow) throw RuntimeException("export failed")
|
|
||||||
return files.toMutableList()
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun writeSrt(dialogs: List<Dialog>) = files[0]
|
|
||||||
override fun writeSmi(dialogs: List<Dialog>) = files[0]
|
|
||||||
override fun writeVtt(dialogs: List<Dialog>) = files[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
class FakeReader(
|
|
||||||
private val dialogs: List<Dialog>,
|
|
||||||
private val shouldThrow: Boolean = false
|
|
||||||
) : BaseReader() {
|
|
||||||
override fun read(): List<Dialog> {
|
|
||||||
if (shouldThrow) throw RuntimeException("reader failed")
|
|
||||||
return dialogs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
val overrideReporter = object : TaskReporter {
|
|
||||||
override fun markClaimed(taskId: UUID, workerId: String) {}
|
|
||||||
override fun updateLastSeen(taskId: UUID) {}
|
|
||||||
override fun markCompleted(taskId: UUID) {}
|
|
||||||
override fun markFailed(referenceId: UUID, taskId: UUID) {}
|
|
||||||
override fun markCancelled(referenceId: UUID, taskId: UUID) {}
|
|
||||||
override fun updateProgress(taskId: UUID, progress: Int) {}
|
|
||||||
override fun log(taskId: UUID, message: String) {}
|
|
||||||
override fun publishEvent(event: Event) {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
|
||||||
// Helpers
|
|
||||||
// ---------------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun makeTask(
|
|
||||||
formats: List<SubtitleFormat> = emptyList()
|
|
||||||
): ConvertTask {
|
|
||||||
return ConvertTask(
|
|
||||||
ConvertTask.Data(
|
|
||||||
inputFile = "input.srt",
|
|
||||||
language = "no",
|
|
||||||
outputDirectory = "out",
|
|
||||||
outputFileName = "name",
|
|
||||||
formats = formats,
|
|
||||||
allowOverwrite = true
|
|
||||||
)
|
|
||||||
).apply { newReferenceId() }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
|
||||||
// Tests
|
|
||||||
// ---------------------------------------------------------------------
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("""
|
|
||||||
Når onTask kjøres og converterer
|
|
||||||
Hvis den bruker lengre tid
|
|
||||||
Så:
|
|
||||||
Skal koden vente til den er ferdig
|
|
||||||
""")
|
|
||||||
fun onTask_validate_delay() = runTest {
|
|
||||||
val delay = 1000L
|
|
||||||
val converter = MockConverter(
|
|
||||||
delay,
|
|
||||||
listOf("file:///potato.srt"),
|
|
||||||
listener = FakeListener()
|
|
||||||
)
|
|
||||||
listener.apply {
|
|
||||||
overrideConverter = converter
|
|
||||||
}
|
|
||||||
val task = makeTask()
|
|
||||||
val event = listener.onTask(task)
|
|
||||||
|
|
||||||
val time = measureTimeMillis {
|
|
||||||
val accepted = listener.accept(task, overrideReporter)
|
|
||||||
assertTrue(accepted, "Task listener did not accept the task.")
|
|
||||||
listener.getJob()?.join()
|
|
||||||
assertTrue(event is ConvertTaskResultEvent)
|
|
||||||
assertEquals(TaskStatus.Completed, (event as ConvertTaskResultEvent).status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName(
|
|
||||||
"""
|
|
||||||
Når onTask kjøres
|
|
||||||
Hvis converter lykkes
|
|
||||||
Så returneres Completed-event
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
fun onTask_success() = runTest {
|
|
||||||
val dialogs = listOf(
|
|
||||||
Dialog("0", Time(0, 0, 0, 0), Time(0, 0, 1000, 0), "Hello", DialogType.DIALOG_NORMAL)
|
|
||||||
)
|
|
||||||
|
|
||||||
val env = MockConverterEnvironment(
|
|
||||||
canReadValue = true,
|
|
||||||
reader = FakeReader(dialogs),
|
|
||||||
exporter = FakeExporter(listOf(File("/fake/out.srt")))
|
|
||||||
)
|
|
||||||
|
|
||||||
listener.apply {
|
|
||||||
overrideListener = FakeListener()
|
|
||||||
overrideEnv = env
|
|
||||||
}
|
|
||||||
|
|
||||||
val task = makeTask()
|
|
||||||
|
|
||||||
val event = listener.onTask(task) as ConvertTaskResultEvent
|
|
||||||
|
|
||||||
assertEquals(TaskStatus.Completed, event.status)
|
|
||||||
assertEquals(listOf("/fake/out.srt"), event.data!!.outputFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName(
|
|
||||||
"""
|
|
||||||
Når onTask kjøres
|
|
||||||
Hvis converter feiler
|
|
||||||
Så returneres Failed-event
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
fun onTask_failure() = runTest {
|
|
||||||
val env = MockConverterEnvironment(
|
|
||||||
canReadValue = true,
|
|
||||||
reader = FakeReader(emptyList()) // triggers FileIsNullOrEmpty
|
|
||||||
)
|
|
||||||
|
|
||||||
listener.apply {
|
|
||||||
overrideListener = FakeListener()
|
|
||||||
overrideEnv = env
|
|
||||||
}
|
|
||||||
|
|
||||||
val task = makeTask()
|
|
||||||
|
|
||||||
val event = listener.onTask(task) as ConvertTaskResultEvent
|
|
||||||
|
|
||||||
assertEquals(TaskStatus.Failed, event.status)
|
|
||||||
assertNull(event.data)
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName(
|
|
||||||
"""
|
|
||||||
Når supports() kalles
|
|
||||||
Så returnerer den true for ConvertTask og false for andre typer
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
fun supports_test() {
|
|
||||||
val listener = ConvertTaskListener()
|
|
||||||
|
|
||||||
assertTrue(listener.supports(makeTask()))
|
|
||||||
assertFalse(listener.supports(DummyTask()))
|
|
||||||
}
|
|
||||||
|
|
||||||
class DummyTask : Task()
|
|
||||||
}
|
|
||||||
@ -1,41 +0,0 @@
|
|||||||
spring:
|
|
||||||
main:
|
|
||||||
allow-bean-definition-overriding: true
|
|
||||||
flyway:
|
|
||||||
enabled: false
|
|
||||||
locations: classpath:flyway
|
|
||||||
autoconfigure:
|
|
||||||
exclude:
|
|
||||||
- org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
|
|
||||||
|
|
||||||
output:
|
|
||||||
ansi:
|
|
||||||
enabled: always
|
|
||||||
|
|
||||||
springdoc:
|
|
||||||
swagger-ui:
|
|
||||||
path: /open/swagger-ui
|
|
||||||
|
|
||||||
logging:
|
|
||||||
level:
|
|
||||||
org.springframework.web.socket.config.WebSocketMessageBrokerStats: WARN
|
|
||||||
org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping: DEBUG
|
|
||||||
|
|
||||||
management:
|
|
||||||
endpoints:
|
|
||||||
web:
|
|
||||||
exposure:
|
|
||||||
include:
|
|
||||||
- mappings
|
|
||||||
- health
|
|
||||||
endpoint:
|
|
||||||
health:
|
|
||||||
show-details: always
|
|
||||||
|
|
||||||
media:
|
|
||||||
cache: /src/cache
|
|
||||||
outgoing: /src/output
|
|
||||||
incoming: /src/input
|
|
||||||
|
|
||||||
streamit:
|
|
||||||
address: http://streamit.service
|
|
||||||
@ -22,7 +22,7 @@ Only one instance is supported, while multiple processer's can be run at any tim
|
|||||||
- Extracts info from filename
|
- Extracts info from filename
|
||||||
- Extracts info from file media streams
|
- Extracts info from file media streams
|
||||||
- Produces title and sanitized
|
- Produces title and sanitized
|
||||||
- py-metadata:
|
- pyMetadata:
|
||||||
- Picks up event
|
- Picks up event
|
||||||
- Searches with sources using title and sanitized
|
- Searches with sources using title and sanitized
|
||||||
- Produces result
|
- Produces result
|
||||||
|
|||||||
@ -1,10 +1,10 @@
|
|||||||
plugins {
|
plugins {
|
||||||
id("java")
|
id("java")
|
||||||
kotlin("jvm")
|
kotlin("jvm")
|
||||||
kotlin("plugin.spring")
|
kotlin("plugin.spring") version "1.5.31"
|
||||||
id("org.springframework.boot")
|
id("org.springframework.boot") version "2.5.5"
|
||||||
id("io.spring.dependency-management")
|
id("io.spring.dependency-management") version "1.0.11.RELEASE"
|
||||||
id("org.jetbrains.kotlin.plugin.serialization")
|
id("org.jetbrains.kotlin.plugin.serialization") version "1.5.0" // Legg til Kotlin Serialization-plugin
|
||||||
}
|
}
|
||||||
|
|
||||||
group = "no.iktdev.mediaprocessing"
|
group = "no.iktdev.mediaprocessing"
|
||||||
@ -22,45 +22,44 @@ repositories {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
val exposedVersion = "0.44.0"
|
||||||
dependencies {
|
dependencies {
|
||||||
|
|
||||||
/*Spring boot*/
|
/*Spring boot*/
|
||||||
implementation("org.springframework.boot:spring-boot-starter")
|
|
||||||
implementation("org.springframework.boot:spring-boot-starter-web")
|
implementation("org.springframework.boot:spring-boot-starter-web")
|
||||||
implementation("org.springframework.boot:spring-boot-starter-webflux")
|
implementation("org.springframework.boot:spring-boot-starter:3.2.0")
|
||||||
implementation("org.springframework.boot:spring-boot-starter-actuator")
|
// implementation("org.springframework.kafka:spring-kafka:3.0.1")
|
||||||
implementation("org.springframework.boot:spring-boot-starter-websocket")
|
implementation("org.springframework.kafka:spring-kafka:2.8.5")
|
||||||
implementation("org.springframework:spring-tx")
|
implementation("org.springframework.boot:spring-boot-starter-websocket:2.6.3")
|
||||||
implementation("org.springframework.boot:spring-boot-starter-validation")
|
|
||||||
|
|
||||||
|
|
||||||
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
|
implementation("io.github.microutils:kotlin-logging-jvm:2.0.11")
|
||||||
implementation("com.google.code.gson:gson:2.8.9")
|
implementation("com.google.code.gson:gson:2.8.9")
|
||||||
implementation("org.json:json:20210307")
|
implementation("org.json:json:20210307")
|
||||||
|
|
||||||
implementation(libs.exfl)
|
implementation("no.iktdev:exfl:0.0.16-SNAPSHOT")
|
||||||
implementation("no.iktdev.streamit.library:streamit-library-db:1.0.0-alpha14")
|
implementation("no.iktdev.streamit.library:streamit-library-db:1.0.0-alpha11")
|
||||||
implementation(libs.eventi)
|
|
||||||
|
|
||||||
|
|
||||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
|
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1")
|
||||||
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
|
implementation("com.github.vishna:watchservice-ktx:master-SNAPSHOT")
|
||||||
|
|
||||||
|
//implementation(project(mapOf("path" to ":shared")))
|
||||||
|
|
||||||
implementation(project(mapOf("path" to ":shared:ffmpeg")))
|
implementation(project(mapOf("path" to ":shared:eventi")))
|
||||||
implementation(project(mapOf("path" to ":shared:common")))
|
implementation(project(mapOf("path" to ":shared:common")))
|
||||||
implementation(project(mapOf("path" to ":shared:database")))
|
|
||||||
|
|
||||||
|
implementation("org.jetbrains.exposed:exposed-core:$exposedVersion")
|
||||||
|
implementation("org.jetbrains.exposed:exposed-dao:$exposedVersion")
|
||||||
|
implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion")
|
||||||
|
implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion")
|
||||||
|
implementation ("mysql:mysql-connector-java:8.0.29")
|
||||||
|
|
||||||
|
|
||||||
implementation("org.jetbrains.kotlin:kotlin-stdlib")
|
implementation("org.jetbrains.kotlin:kotlin-stdlib")
|
||||||
implementation(kotlin("stdlib-jdk8"))
|
implementation(kotlin("stdlib-jdk8"))
|
||||||
testImplementation("org.assertj:assertj-core:3.21.0")
|
testImplementation("org.assertj:assertj-core:3.21.0")
|
||||||
|
|
||||||
|
|
||||||
testImplementation("junit:junit:4.12")
|
testImplementation("junit:junit:4.12")
|
||||||
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2")
|
|
||||||
|
|
||||||
testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.0")
|
testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.0")
|
||||||
testImplementation("org.skyscreamer:jsonassert:1.5.0")
|
testImplementation("org.skyscreamer:jsonassert:1.5.0")
|
||||||
testImplementation("org.mockito:mockito-core:3.+")
|
testImplementation("org.mockito:mockito-core:3.+")
|
||||||
@ -70,35 +69,30 @@ dependencies {
|
|||||||
testImplementation("org.mockito:mockito-core:3.+")
|
testImplementation("org.mockito:mockito-core:3.+")
|
||||||
testImplementation("org.assertj:assertj-core:3.4.1")
|
testImplementation("org.assertj:assertj-core:3.4.1")
|
||||||
|
|
||||||
|
/*testImplementation("org.junit.vintage:junit-vintage-engine")
|
||||||
|
testImplementation("org.junit.jupiter:junit-jupiter:5.10.1")
|
||||||
|
testImplementation("org.junit.jupiter:junit-jupiter-params:5.8.1")
|
||||||
|
testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.1")
|
||||||
|
testRuntimeOnly ("org.junit.jupiter:junit-jupiter-engine:5.10.1")
|
||||||
|
testImplementation("org.mockito:mockito-core:5.8.0") // Oppdater versjonen hvis det er nyere tilgjengelig
|
||||||
|
testImplementation("org.mockito:mockito-junit-jupiter:5.8.0")
|
||||||
|
testImplementation(platform("org.junit:junit-bom:5.10.1"))
|
||||||
|
testImplementation("org.junit.platform:junit-platform-runner:1.10.1")*/
|
||||||
|
|
||||||
testImplementation(platform("org.junit:junit-bom:5.9.1"))
|
testImplementation(platform("org.junit:junit-bom:5.9.1"))
|
||||||
testImplementation("org.junit.jupiter:junit-jupiter")
|
testImplementation("org.junit.jupiter:junit-jupiter")
|
||||||
testImplementation("org.junit.jupiter:junit-jupiter-params")
|
|
||||||
testImplementation("junit:junit:4.13.2")
|
testImplementation("junit:junit:4.13.2")
|
||||||
testImplementation("org.mockito:mockito-core:3.+")
|
testImplementation("org.mockito:mockito-core:3.+")
|
||||||
testImplementation("org.assertj:assertj-core:3.4.1")
|
testImplementation("org.assertj:assertj-core:3.4.1")
|
||||||
testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0")
|
testImplementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.0")
|
||||||
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
|
|
||||||
testImplementation("io.mockk:mockk:1.13.9")
|
|
||||||
testImplementation("org.mockito:mockito-inline:5.2.0")
|
|
||||||
testImplementation("org.mockito.kotlin:mockito-kotlin:5.2.1")
|
|
||||||
testImplementation("org.mockito:mockito-junit-jupiter:5.11.0")
|
|
||||||
testImplementation(project(":shared:common", configuration = "testArtifacts"))
|
|
||||||
testImplementation(project(":shared:database", configuration = "testArtifacts"))
|
|
||||||
testImplementation("org.springframework.boot:spring-boot-starter-test")
|
|
||||||
val exposedVersion = "0.61.0"
|
|
||||||
testImplementation("org.jetbrains.exposed:exposed-core:${exposedVersion}")
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tasks.withType<Test> {
|
tasks.withType<Test> {
|
||||||
useJUnitPlatform()
|
useJUnitPlatform()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
kotlin {
|
kotlin {
|
||||||
jvmToolchain(21)
|
jvmToolchain(17)
|
||||||
}
|
}
|
||||||
|
|
||||||
tasks.bootJar {
|
tasks.bootJar {
|
||||||
|
|||||||
@ -1,33 +1,56 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
package no.iktdev.mediaprocessing.coordinator
|
||||||
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
import mu.KotlinLogging
|
||||||
import no.iktdev.eventi.events.EventTypeRegistry
|
|
||||||
import no.iktdev.eventi.tasks.TaskTypeRegistry
|
|
||||||
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
import no.iktdev.exfl.coroutines.CoroutinesDefault
|
||||||
import no.iktdev.exfl.coroutines.CoroutinesIO
|
import no.iktdev.exfl.coroutines.CoroutinesIO
|
||||||
import no.iktdev.exfl.observable.Observables
|
import no.iktdev.exfl.observable.Observables
|
||||||
import no.iktdev.mediaprocessing.coordinator.config.ExecutablesConfig
|
import no.iktdev.mediaprocessing.shared.common.*
|
||||||
import no.iktdev.mediaprocessing.coordinator.config.ProcesserClientProperties
|
import no.iktdev.eventi.database.MySqlDataSource
|
||||||
import no.iktdev.mediaprocessing.shared.common.configs.StreamItConfig
|
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.EventRegistry
|
import no.iktdev.mediaprocessing.shared.common.database.cal.RunnerManager
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.TaskRegistry
|
import no.iktdev.mediaprocessing.shared.common.database.cal.TasksManager
|
||||||
import no.iktdev.mediaprocessing.shared.common.getAppVersion
|
import no.iktdev.streamit.library.db.tables.*
|
||||||
import no.iktdev.mediaprocessing.shared.database.DatabaseApplication
|
import no.iktdev.streamit.library.db.tables.helper.cast_errors
|
||||||
import no.iktdev.mediaprocessing.shared.database.DatabasebasedMediaProcessingApp
|
import no.iktdev.streamit.library.db.tables.helper.data_audio
|
||||||
import org.springframework.boot.context.properties.EnableConfigurationProperties
|
import no.iktdev.streamit.library.db.tables.helper.data_video
|
||||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication
|
||||||
import org.springframework.boot.runApplication
|
import org.springframework.boot.runApplication
|
||||||
import org.springframework.context.annotation.Configuration
|
import org.springframework.context.annotation.Bean
|
||||||
|
|
||||||
|
val log = KotlinLogging.logger {}
|
||||||
|
private lateinit var eventDatabase: EventsDatabase
|
||||||
|
private lateinit var eventsManager: EventsManager
|
||||||
|
lateinit var runnerManager: RunnerManager
|
||||||
|
|
||||||
|
|
||||||
|
@SpringBootApplication
|
||||||
|
class CoordinatorApplication {
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
fun eventManager(): EventsManager {
|
||||||
|
return eventsManager
|
||||||
|
}
|
||||||
|
|
||||||
@DatabasebasedMediaProcessingApp
|
|
||||||
class CoordinatorApplication: DatabaseApplication() {
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private lateinit var storeDatabase: MySqlDataSource
|
||||||
|
|
||||||
val ioCoroutine = CoroutinesIO()
|
val ioCoroutine = CoroutinesIO()
|
||||||
val defaultCoroutine = CoroutinesDefault()
|
val defaultCoroutine = CoroutinesDefault()
|
||||||
|
|
||||||
private val log = KotlinLogging.logger {}
|
|
||||||
|
fun getStoreDatabase(): MySqlDataSource {
|
||||||
|
return storeDatabase
|
||||||
|
}
|
||||||
|
|
||||||
|
lateinit var taskManager: TasksManager
|
||||||
|
|
||||||
fun main(args: Array<String>) {
|
fun main(args: Array<String>) {
|
||||||
|
|
||||||
|
|
||||||
|
printSharedConfig()
|
||||||
|
|
||||||
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
ioCoroutine.addListener(listener = object: Observables.ObservableValue.ValueListener<Throwable> {
|
||||||
override fun onUpdated(value: Throwable) {
|
override fun onUpdated(value: Throwable) {
|
||||||
value.printStackTrace()
|
value.printStackTrace()
|
||||||
@ -38,31 +61,49 @@ fun main(args: Array<String>) {
|
|||||||
value.printStackTrace()
|
value.printStackTrace()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
eventDatabase = EventsDatabase().also {
|
||||||
|
eventsManager = EventsManager(it.database)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
storeDatabase = DatabaseEnvConfig.toStoredDatabase()
|
||||||
|
storeDatabase.createDatabase()
|
||||||
|
|
||||||
|
|
||||||
|
taskManager = TasksManager(eventDatabase.database)
|
||||||
|
|
||||||
|
|
||||||
|
val tables = arrayOf(
|
||||||
|
catalog,
|
||||||
|
genre,
|
||||||
|
movie,
|
||||||
|
serie,
|
||||||
|
subtitle,
|
||||||
|
summary,
|
||||||
|
users,
|
||||||
|
progress,
|
||||||
|
data_audio,
|
||||||
|
data_video,
|
||||||
|
cast_errors,
|
||||||
|
titles
|
||||||
|
)
|
||||||
|
storeDatabase.createTables(*tables)
|
||||||
|
|
||||||
|
runnerManager = RunnerManager(dataSource = eventDatabase.database, name = CoordinatorApplication::class.java.simpleName)
|
||||||
|
runnerManager.assignRunner()
|
||||||
|
|
||||||
runApplication<CoordinatorApplication>(*args)
|
runApplication<CoordinatorApplication>(*args)
|
||||||
log.info { "App Version: ${getAppVersion()}" }
|
log.info { "App Version: ${getAppVersion()}" }
|
||||||
}
|
}
|
||||||
//private val logger = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
@Configuration
|
fun printSharedConfig() {
|
||||||
open class ApplicationConfiguration() {
|
log.info { "File Input: ${SharedConfig.incomingContent}" }
|
||||||
init {
|
log.info { "File Output: ${SharedConfig.outgoingContent}" }
|
||||||
EventRegistry.getEvents().let {
|
log.info { "Ffprobe: ${SharedConfig.ffprobe}" }
|
||||||
EventTypeRegistry.register(it)
|
log.info { "Ffmpeg: ${SharedConfig.ffmpeg}" }
|
||||||
}
|
|
||||||
TaskRegistry.getTasks().let {
|
/*log.info { "Database: ${DatabaseConfig.database} @ ${DatabaseConfig.address}:${DatabaseConfig.port}" }
|
||||||
TaskTypeRegistry.register(it)
|
log.info { "Username: ${DatabaseConfig.username}" }
|
||||||
}
|
log.info { "Password: ${if (DatabaseConfig.password.isNullOrBlank()) "Is not set" else "Is set"}" }*/
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Configuration
|
|
||||||
@EnableConfigurationProperties(
|
|
||||||
value = [
|
|
||||||
ExecutablesConfig::class,
|
|
||||||
StreamItConfig::class,
|
|
||||||
ProcesserClientProperties::class
|
|
||||||
]
|
|
||||||
)
|
|
||||||
class CoordinatorConfig
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.config.ExecutablesConfig
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.configs.MediaPaths
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.configs.StreamItConfig
|
|
||||||
import org.springframework.stereotype.Service
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
@Service
|
|
||||||
class CoordinatorEnv(
|
|
||||||
val streamIt: StreamItConfig,
|
|
||||||
val exec: ExecutablesConfig,
|
|
||||||
val media: MediaPaths
|
|
||||||
) {
|
|
||||||
val streamitAddress = streamIt.address
|
|
||||||
val ffprobe = exec.ffprobe
|
|
||||||
|
|
||||||
val cachedContent = File(media.cache)
|
|
||||||
val outgoingContent = File(media.outgoing)
|
|
||||||
val incomingContent = File(media.incoming)
|
|
||||||
val preference: File = File("/data/config/preference.json")
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -0,0 +1,97 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator
|
||||||
|
|
||||||
|
import no.iktdev.eventi.data.EventMetadata
|
||||||
|
import no.iktdev.eventi.data.EventStatus
|
||||||
|
import no.iktdev.eventi.data.eventId
|
||||||
|
import no.iktdev.eventi.implementations.ActiveMode
|
||||||
|
import no.iktdev.eventi.implementations.EventCoordinator
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.Event
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.MediaProcessStartEvent
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.PermitWorkCreationEvent
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.data.StartEventData
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.dto.StartOperationEvents
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.context.ApplicationContext
|
||||||
|
import org.springframework.stereotype.Component
|
||||||
|
import java.io.File
|
||||||
|
import java.util.*
|
||||||
|
|
||||||
|
@Component
|
||||||
|
class Coordinator(
|
||||||
|
@Autowired
|
||||||
|
override var applicationContext: ApplicationContext,
|
||||||
|
@Autowired
|
||||||
|
override var eventManager: EventsManager
|
||||||
|
|
||||||
|
) : EventCoordinator<Event, EventsManager>() {
|
||||||
|
|
||||||
|
init {
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getProducerName(): String {
|
||||||
|
return this::class.java.simpleName
|
||||||
|
}
|
||||||
|
|
||||||
|
public fun startProcess(file: File, type: ProcessType) {
|
||||||
|
val operations: List<StartOperationEvents> = listOf(
|
||||||
|
StartOperationEvents.ENCODE,
|
||||||
|
StartOperationEvents.EXTRACT,
|
||||||
|
StartOperationEvents.CONVERT
|
||||||
|
)
|
||||||
|
startProcess(file, type, operations)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun startProcess(file: File, type: ProcessType, operations: List<StartOperationEvents>): UUID {
|
||||||
|
val referenceId: UUID = UUID.randomUUID()
|
||||||
|
val event = MediaProcessStartEvent(
|
||||||
|
metadata = EventMetadata(
|
||||||
|
referenceId = referenceId.toString(),
|
||||||
|
status = EventStatus.Success,
|
||||||
|
source = getProducerName()
|
||||||
|
),
|
||||||
|
data = StartEventData(
|
||||||
|
file = file.absolutePath,
|
||||||
|
type = type,
|
||||||
|
operations = operations
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
produceNewEvent(event)
|
||||||
|
return referenceId
|
||||||
|
}
|
||||||
|
|
||||||
|
fun permitWorkToProceedOn(referenceId: String, events: List<Event>, message: String) {
|
||||||
|
val defaultRequiredBy = listOf(Events.EventMediaParameterEncodeCreated, Events.EventMediaParameterExtractCreated)
|
||||||
|
val eventToAttachTo = if (events.any { it.eventType in defaultRequiredBy }) {
|
||||||
|
events.findLast { it.eventType in defaultRequiredBy }
|
||||||
|
} else events.find { it.eventType == Events.EventMediaProcessStarted }
|
||||||
|
if (eventToAttachTo == null) {
|
||||||
|
log.error { "No event to attach permit to" }
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
produceNewEvent(
|
||||||
|
PermitWorkCreationEvent(
|
||||||
|
metadata = EventMetadata(
|
||||||
|
referenceId = referenceId,
|
||||||
|
derivedFromEventId = eventToAttachTo.eventId(),
|
||||||
|
status = EventStatus.Success,
|
||||||
|
source = getProducerName()
|
||||||
|
),
|
||||||
|
data = message
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun getActiveTaskMode(): ActiveMode {
|
||||||
|
if (runnerManager.iAmSuperseded()) {
|
||||||
|
// This will let the application complete but not consume new
|
||||||
|
taskMode = ActiveMode.Passive
|
||||||
|
}
|
||||||
|
return taskMode
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator
|
||||||
|
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.Events
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.EventsListenerContract
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||||
|
|
||||||
|
abstract class CoordinatorEventListener(): EventsListenerContract<EventsManager, Coordinator>() {
|
||||||
|
abstract override val produceEvent: Events
|
||||||
|
abstract override val listensForEvents: List<Events>
|
||||||
|
abstract override var coordinator: Coordinator?
|
||||||
|
}
|
||||||
@ -1,21 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.ProgressUpdate
|
|
||||||
import org.springframework.stereotype.Service
|
|
||||||
import java.util.concurrent.ConcurrentHashMap
|
|
||||||
|
|
||||||
@Service
|
|
||||||
class CoordinatorService {
|
|
||||||
|
|
||||||
private val progressMap = ConcurrentHashMap<String, ProgressUpdate>()
|
|
||||||
|
|
||||||
fun updateProgress(update: ProgressUpdate) {
|
|
||||||
progressMap[update.taskId] = update
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getProgress(taskId: String): ProgressUpdate? =
|
|
||||||
progressMap[taskId]
|
|
||||||
|
|
||||||
fun getProgress(): List<ProgressUpdate> =
|
|
||||||
progressMap.values.toList()
|
|
||||||
}
|
|
||||||
@ -1,45 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
|
||||||
|
|
||||||
import kotlinx.coroutines.CoroutineScope
|
|
||||||
import kotlinx.coroutines.Dispatchers
|
|
||||||
import kotlinx.coroutines.Job
|
|
||||||
import kotlinx.coroutines.launch
|
|
||||||
import no.iktdev.eventi.events.EventDispatcher
|
|
||||||
import no.iktdev.eventi.events.EventPollerImplementation
|
|
||||||
import no.iktdev.eventi.events.SequenceDispatchQueue
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
|
||||||
import org.springframework.context.SmartLifecycle
|
|
||||||
import org.springframework.context.annotation.DependsOn
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class EventPollerAdministrator(
|
|
||||||
private val eventPoller: EventPoller
|
|
||||||
) : SmartLifecycle {
|
|
||||||
|
|
||||||
private var running = false
|
|
||||||
|
|
||||||
var job: Job? = null
|
|
||||||
override fun start() {
|
|
||||||
job = CoroutineScope(Dispatchers.Default).launch {
|
|
||||||
eventPoller.start()
|
|
||||||
}
|
|
||||||
running = true
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun stop() {
|
|
||||||
job?.cancel()
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun isRunning() = running
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
val sequenceDispatcher = SequenceDispatchQueue(8)
|
|
||||||
val dispatcher = EventDispatcher(eventStore = EventStore)
|
|
||||||
|
|
||||||
@Component
|
|
||||||
@DependsOn("ExposedInit")
|
|
||||||
class EventPoller: EventPollerImplementation(eventStore = EventStore, dispatchQueue = sequenceDispatcher, dispatcher = dispatcher) {
|
|
||||||
}
|
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator
|
||||||
|
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.DatabaseEnvConfig
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.tables.*
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.toEventsDatabase
|
||||||
|
|
||||||
|
class EventsDatabase() {
|
||||||
|
val database = DatabaseEnvConfig.toEventsDatabase()
|
||||||
|
val tables = listOf(
|
||||||
|
events, // For kafka
|
||||||
|
allEvents,
|
||||||
|
tasks,
|
||||||
|
runners,
|
||||||
|
processed
|
||||||
|
)
|
||||||
|
|
||||||
|
init {
|
||||||
|
database.createDatabase()
|
||||||
|
database.createTables(*tables.toTypedArray())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator
|
||||||
|
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.socket.SocketImplementation
|
||||||
|
import org.springframework.context.annotation.Configuration
|
||||||
|
import org.springframework.context.annotation.Import
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
class SocketLocalInit: SocketImplementation() {
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -1,190 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
|
||||||
|
|
||||||
import com.google.gson.Gson
|
|
||||||
import com.google.gson.JsonObject
|
|
||||||
import com.google.gson.JsonParser
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.AudioCodec
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.VideoCodec
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.io.File
|
|
||||||
import java.io.IOException
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// MODELLER
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
data class PeferenceConfig(
|
|
||||||
val processer: ProcesserPreference,
|
|
||||||
val language: LanguagePreference
|
|
||||||
)
|
|
||||||
|
|
||||||
data class LanguagePreference(
|
|
||||||
val preferredAudio: List<String>,
|
|
||||||
val preferredSubtitles: List<String>,
|
|
||||||
|
|
||||||
val preferOriginal: Boolean = true,
|
|
||||||
val avoidDub: Boolean = true,
|
|
||||||
|
|
||||||
// NEW: Prioritet for hvilket subtitle-format som skal brukes som master
|
|
||||||
val subtitleFormatPriority: List<String> = listOf("ass", "srt", "vtt", "smi"),
|
|
||||||
|
|
||||||
// NEW: Hvordan subtitles skal velges
|
|
||||||
val subtitleSelectionMode: SubtitleSelectionMode = SubtitleSelectionMode.DialogueOnly
|
|
||||||
) {
|
|
||||||
companion object {
|
|
||||||
fun default() = LanguagePreference(
|
|
||||||
preferredAudio = listOf("eng", "nor", "jpn"),
|
|
||||||
preferredSubtitles = listOf("eng", "nor"),
|
|
||||||
preferOriginal = true,
|
|
||||||
avoidDub = true,
|
|
||||||
subtitleFormatPriority = listOf("ass", "srt", "vtt", "smi"),
|
|
||||||
subtitleSelectionMode = SubtitleSelectionMode.DialogueOnly
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
enum class SubtitleSelectionMode {
|
|
||||||
DialogueOnly, // Kun dialog
|
|
||||||
DialogueAndForced, // Dialog + forced
|
|
||||||
All // Alle typer
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
data class ProcesserPreference(
|
|
||||||
val videoPreference: VideoPreference? = null,
|
|
||||||
val audioPreference: AudioPreference? = null
|
|
||||||
) {
|
|
||||||
companion object {
|
|
||||||
fun default(): ProcesserPreference {
|
|
||||||
return ProcesserPreference(
|
|
||||||
videoPreference = VideoPreference(VideoCodec.Hevc(), false),
|
|
||||||
audioPreference = AudioPreference(AudioCodec.Aac())
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
data class VideoPreference(
|
|
||||||
val codec: VideoCodec,
|
|
||||||
val enforceMkv: Boolean = false
|
|
||||||
)
|
|
||||||
|
|
||||||
data class AudioPreference(
|
|
||||||
val codec: AudioCodec
|
|
||||||
)
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// PREFERENCE COMPONENT
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class Preference(private val coordinatorEnv: CoordinatorEnv) {
|
|
||||||
|
|
||||||
private val gson = Gson()
|
|
||||||
private val lock = Any()
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Leser hele configen, men bevarer ukjente felter.
|
|
||||||
*/
|
|
||||||
fun getFullConfig(): PeferenceConfig {
|
|
||||||
val file = coordinatorEnv.preference
|
|
||||||
|
|
||||||
if (!file.exists()) {
|
|
||||||
val default = PeferenceConfig(
|
|
||||||
processer = ProcesserPreference.default(),
|
|
||||||
language = LanguagePreference.default()
|
|
||||||
)
|
|
||||||
writeJsonObject(JsonObject().apply {
|
|
||||||
add("processer", gson.toJsonTree(default.processer))
|
|
||||||
add("language", gson.toJsonTree(default.language))
|
|
||||||
}, file)
|
|
||||||
return default
|
|
||||||
}
|
|
||||||
|
|
||||||
val root = try {
|
|
||||||
JsonParser.parseString(file.readText()).asJsonObject
|
|
||||||
} catch (e: Exception) {
|
|
||||||
return PeferenceConfig(
|
|
||||||
processer = ProcesserPreference.default(),
|
|
||||||
language = LanguagePreference.default()
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
val processer = try {
|
|
||||||
gson.fromJson(root.get("processer"), ProcesserPreference::class.java)
|
|
||||||
?: ProcesserPreference.default()
|
|
||||||
} catch (_: Exception) {
|
|
||||||
ProcesserPreference.default()
|
|
||||||
}
|
|
||||||
|
|
||||||
val language = try {
|
|
||||||
gson.fromJson(root.get("language"), LanguagePreference::class.java)
|
|
||||||
?: LanguagePreference.default()
|
|
||||||
} catch (_: Exception) {
|
|
||||||
LanguagePreference.default()
|
|
||||||
}
|
|
||||||
|
|
||||||
return PeferenceConfig(processer, language)
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getProcesserPreference(): ProcesserPreference =
|
|
||||||
getFullConfig().processer
|
|
||||||
|
|
||||||
fun getLanguagePreference(): LanguagePreference =
|
|
||||||
getFullConfig().language
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Oppdaterer kun processer-delen og bevarer resten av JSON.
|
|
||||||
*/
|
|
||||||
fun saveProcesserPreference(pref: ProcesserPreference) {
|
|
||||||
val file = coordinatorEnv.preference
|
|
||||||
synchronized(lock) {
|
|
||||||
val root = readOrEmpty(file)
|
|
||||||
root.add("processer", gson.toJsonTree(pref))
|
|
||||||
writeJsonObject(root, file)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Oppdaterer kun language-delen og bevarer resten av JSON.
|
|
||||||
*/
|
|
||||||
fun saveLanguagePreference(pref: LanguagePreference) {
|
|
||||||
val file = coordinatorEnv.preference
|
|
||||||
synchronized(lock) {
|
|
||||||
val root = readOrEmpty(file)
|
|
||||||
root.add("language", gson.toJsonTree(pref))
|
|
||||||
writeJsonObject(root, file)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Overskriver hele configen (brukes hvis FE sender alt).
|
|
||||||
*/
|
|
||||||
fun saveFullConfig(cfg: PeferenceConfig) {
|
|
||||||
val file = coordinatorEnv.preference
|
|
||||||
synchronized(lock) {
|
|
||||||
val root = JsonObject()
|
|
||||||
root.add("processer", gson.toJsonTree(cfg.processer))
|
|
||||||
root.add("language", gson.toJsonTree(cfg.language))
|
|
||||||
writeJsonObject(root, file)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun readOrEmpty(file: File): JsonObject =
|
|
||||||
if (file.exists()) {
|
|
||||||
try {
|
|
||||||
JsonParser.parseString(file.readText()).asJsonObject
|
|
||||||
} catch (_: Exception) {
|
|
||||||
JsonObject()
|
|
||||||
}
|
|
||||||
} else JsonObject()
|
|
||||||
|
|
||||||
private fun writeJsonObject(obj: JsonObject, file: File) {
|
|
||||||
try {
|
|
||||||
file.parentFile?.mkdirs()
|
|
||||||
file.writeText(gson.toJson(obj))
|
|
||||||
} catch (e: IOException) {
|
|
||||||
throw RuntimeException("Failed to write preference file", e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import org.springframework.web.reactive.function.client.WebClient
|
|
||||||
import reactor.core.publisher.Mono
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class ProcesserClient(
|
|
||||||
private val processerWebClient: WebClient
|
|
||||||
) {
|
|
||||||
|
|
||||||
fun fetchLog(path: String): Mono<String> =
|
|
||||||
processerWebClient.get()
|
|
||||||
.uri { it.path("/state/log").queryParam("path", path).build() }
|
|
||||||
.retrieve()
|
|
||||||
.bodyToMono(String::class.java)
|
|
||||||
|
|
||||||
fun ping(): Mono<String> =
|
|
||||||
processerWebClient.get()
|
|
||||||
.uri("/actuator/health")
|
|
||||||
.retrieve()
|
|
||||||
.bodyToMono(String::class.java)
|
|
||||||
}
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
|
||||||
|
|
||||||
import org.springframework.boot.web.client.RestTemplateBuilder
|
|
||||||
import org.springframework.context.annotation.Bean
|
|
||||||
import org.springframework.context.annotation.Configuration
|
|
||||||
import org.springframework.web.client.RestTemplate
|
|
||||||
|
|
||||||
@Configuration
|
|
||||||
class RestTemplateConfig {
|
|
||||||
|
|
||||||
@Configuration
|
|
||||||
class RestTemplateConfig(
|
|
||||||
private val coordinatorEnv: CoordinatorEnv
|
|
||||||
) {
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
fun streamitRestTemplate(): RestTemplate {
|
|
||||||
return RestTemplateBuilder()
|
|
||||||
.rootUri(coordinatorEnv.streamitAddress)
|
|
||||||
.build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,74 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator
|
|
||||||
|
|
||||||
import kotlinx.coroutines.CoroutineScope
|
|
||||||
import kotlinx.coroutines.Dispatchers
|
|
||||||
import kotlinx.coroutines.launch
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskPollerImplementation
|
|
||||||
import no.iktdev.eventi.tasks.TaskReporter
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
import org.springframework.boot.ApplicationArguments
|
|
||||||
import org.springframework.boot.ApplicationRunner
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import org.springframework.stereotype.Service
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class TaskPollerAdministrator(
|
|
||||||
private val taskPoller: TaskPoller,
|
|
||||||
): ApplicationRunner {
|
|
||||||
override fun run(args: ApplicationArguments?) {
|
|
||||||
CoroutineScope(Dispatchers.Default).launch {
|
|
||||||
taskPoller.start()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Service
|
|
||||||
class TaskPoller(
|
|
||||||
private val reporter: TaskReporter,
|
|
||||||
) : TaskPollerImplementation(
|
|
||||||
taskStore = TaskStore,
|
|
||||||
reporterFactory = { reporter } // én reporter brukes for alle tasks
|
|
||||||
) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class DefaultTaskReporter() : TaskReporter {
|
|
||||||
override fun markClaimed(taskId: UUID, workerId: String) {
|
|
||||||
TaskStore.claim(taskId, workerId)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun updateLastSeen(taskId: UUID) {
|
|
||||||
TaskStore.heartbeat(taskId)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun markCompleted(taskId: UUID) {
|
|
||||||
TaskStore.markConsumed(taskId, TaskStatus.Completed)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun markFailed(referenceId: UUID, taskId: UUID) {
|
|
||||||
TaskStore.markConsumed(taskId, TaskStatus.Failed)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun markCancelled(referenceId: UUID, taskId: UUID) {
|
|
||||||
TaskStore.markConsumed(taskId, TaskStatus.Cancelled)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun updateProgress(taskId: UUID, progress: Int) {
|
|
||||||
// Not to be implemented for this application
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun log(taskId: UUID, message: String) {
|
|
||||||
// Not to be implemented for this application
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun publishEvent(event: Event) {
|
|
||||||
EventStore.persist(event)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.config
|
|
||||||
|
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties
|
|
||||||
|
|
||||||
@ConfigurationProperties(prefix = "executables")
|
|
||||||
data class ExecutablesConfig(
|
|
||||||
val ffprobe: String
|
|
||||||
)
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.config
|
|
||||||
|
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties
|
|
||||||
import org.springframework.context.annotation.Bean
|
|
||||||
import org.springframework.context.annotation.Configuration
|
|
||||||
import org.springframework.web.reactive.function.client.WebClient
|
|
||||||
|
|
||||||
@ConfigurationProperties(prefix = "processer")
|
|
||||||
data class ProcesserClientProperties(
|
|
||||||
val baseUrl: String
|
|
||||||
)
|
|
||||||
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.config
|
|
||||||
|
|
||||||
import org.springframework.context.annotation.Bean
|
|
||||||
import org.springframework.context.annotation.Configuration
|
|
||||||
import org.springframework.web.reactive.function.client.WebClient
|
|
||||||
|
|
||||||
@Configuration
|
|
||||||
class WebClients(
|
|
||||||
private val processerClientProperties: ProcesserClientProperties
|
|
||||||
|
|
||||||
) {
|
|
||||||
@Bean
|
|
||||||
fun webClient(): WebClient.Builder =
|
|
||||||
WebClient
|
|
||||||
.builder()
|
|
||||||
.codecs { it.defaultCodecs().maxInMemorySize(10 * 1024 * 1024) }
|
|
||||||
@Bean
|
|
||||||
fun processerWebClient(builder: WebClient.Builder): WebClient {
|
|
||||||
return builder.baseUrl(processerClientProperties.baseUrl).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,31 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator.controller
|
||||||
|
|
||||||
|
import com.google.gson.Gson
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.dto.RequestWorkProceed
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.database.cal.EventsManager
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.http.HttpStatus
|
||||||
|
import org.springframework.http.ResponseEntity
|
||||||
|
import org.springframework.stereotype.Controller
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping
|
||||||
|
|
||||||
|
@Controller
|
||||||
|
@RequestMapping(path = ["/action"])
|
||||||
|
class ActionEventController(@Autowired var coordinator: Coordinator, @Autowired var eventsManager: EventsManager) {
|
||||||
|
|
||||||
|
|
||||||
|
@RequestMapping("/flow/proceed")
|
||||||
|
fun permitRunOnSequence(@RequestBody data: RequestWorkProceed): ResponseEntity<String> {
|
||||||
|
|
||||||
|
val set = eventsManager.getEventsWith(data.referenceId)
|
||||||
|
if (set.isEmpty()) {
|
||||||
|
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(Gson().toJson(data))
|
||||||
|
}
|
||||||
|
coordinator.permitWorkToProceedOn(data.referenceId, set, "Requested by ${data.source}")
|
||||||
|
|
||||||
|
//EVENT_MEDIA_WORK_PROCEED_PERMITTED("event:media-work-proceed:permitted")
|
||||||
|
return ResponseEntity.ok(null)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,43 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.eventi.models.store.PersistedEvent
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.EventService
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.EventQuery
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.SequenceEvent
|
|
||||||
import org.springframework.web.bind.annotation.*
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/events")
|
|
||||||
class EventsController(
|
|
||||||
private val paging: EventService
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping()
|
|
||||||
fun getEvents(query: EventQuery): Paginated<PersistedEvent> {
|
|
||||||
return paging.getEvents(query)
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/sequence/{referenceId}")
|
|
||||||
fun getEventSequence(
|
|
||||||
@PathVariable referenceId: UUID,
|
|
||||||
@RequestParam(required = false) beforeEventId: UUID?,
|
|
||||||
@RequestParam(required = false) afterEventId: UUID?,
|
|
||||||
@RequestParam(defaultValue = "50") limit: Int
|
|
||||||
): List<SequenceEvent> {
|
|
||||||
return paging.getPagedEvents(
|
|
||||||
referenceId = referenceId,
|
|
||||||
beforeEventId = beforeEventId,
|
|
||||||
afterEventId = afterEventId,
|
|
||||||
limit = limit
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/history/{referenceId}/effective")
|
|
||||||
fun getEffectiveHistory(
|
|
||||||
@PathVariable referenceId: UUID,
|
|
||||||
): List<PersistedEvent> {
|
|
||||||
return paging.getEffectiveHistory(referenceId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.FileTableItem
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.queries.FilesTableQueries
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
|
||||||
import org.springframework.web.bind.annotation.RestController
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/files")
|
|
||||||
class FilesController {
|
|
||||||
|
|
||||||
@GetMapping()
|
|
||||||
fun getFilesInDatabase(): ResponseEntity<List<FileTableItem>?>? {
|
|
||||||
val files = FilesTableQueries().getFiles()
|
|
||||||
return ResponseEntity.ok(files)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.CoordinatorHealthService
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
|
||||||
import org.springframework.web.bind.annotation.RestController
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/health")
|
|
||||||
class HealthController(
|
|
||||||
private val healthService: CoordinatorHealthService
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping
|
|
||||||
fun getHealth(): CoordinatorHealth {
|
|
||||||
return healthService.getHealth()
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/events")
|
|
||||||
fun getEventRate(): EventRate = healthService.getEventRate()
|
|
||||||
|
|
||||||
@GetMapping("/storage")
|
|
||||||
fun getDiskStatus(): List<DiskInfo> = healthService.getDiskHealth()
|
|
||||||
}
|
|
||||||
@ -1,34 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorService
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.SseHub
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.ProgressUpdate
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.*
|
|
||||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/internal")
|
|
||||||
class InternalProcesserController(
|
|
||||||
private val coordinator: CoordinatorService,
|
|
||||||
private val hub: SseHub
|
|
||||||
) {
|
|
||||||
|
|
||||||
@PostMapping("/progress")
|
|
||||||
fun receiveProgress(@RequestBody update: ProgressUpdate): ResponseEntity<Void> {
|
|
||||||
coordinator.updateProgress(update)
|
|
||||||
|
|
||||||
hub.broadcast("progress", update)
|
|
||||||
return ResponseEntity.ok().build()
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/progress")
|
|
||||||
fun getAllProgress(): List<ProgressUpdate> {
|
|
||||||
return coordinator.getProgress()
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/sse")
|
|
||||||
fun stream(): SseEmitter {
|
|
||||||
return hub.createEmitter()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.ProcesserClient
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
|
||||||
import org.springframework.web.bind.annotation.RequestParam
|
|
||||||
import org.springframework.web.bind.annotation.RestController
|
|
||||||
import reactor.core.publisher.Mono
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/log")
|
|
||||||
class LogController(
|
|
||||||
private val processerClient: ProcesserClient
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping
|
|
||||||
fun getLog(@RequestParam path: String): Mono<String> {
|
|
||||||
return processerClient.fetchLog(path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.CommandService
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.requests.StartProcessRequest
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.PostMapping
|
|
||||||
import org.springframework.web.bind.annotation.RequestBody
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
|
||||||
import org.springframework.web.bind.annotation.RestController
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/operations")
|
|
||||||
class OperationsController(
|
|
||||||
private val commandService: CommandService
|
|
||||||
) {
|
|
||||||
|
|
||||||
@PostMapping("/start")
|
|
||||||
fun startProcess(@RequestBody req: StartProcessRequest): ResponseEntity<Map<String, String>> {
|
|
||||||
return when (val result = commandService.startProcess(req)) {
|
|
||||||
is CommandService.StartResult.Accepted -> ResponseEntity
|
|
||||||
.accepted()
|
|
||||||
.body(
|
|
||||||
mapOf(
|
|
||||||
"referenceId" to result.referenceId.toString(),
|
|
||||||
"status" to "accepted",
|
|
||||||
"message" to "Process accepted and StartedEvent created"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
is CommandService.StartResult.Rejected -> ResponseEntity
|
|
||||||
.badRequest()
|
|
||||||
.body(
|
|
||||||
mapOf(
|
|
||||||
"status" to "rejected",
|
|
||||||
"message" to result.reason
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -0,0 +1,18 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator.controller
|
||||||
|
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.stereotype.Controller
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping
|
||||||
|
|
||||||
|
@Controller
|
||||||
|
@RequestMapping(path = ["/polls"])
|
||||||
|
class PollController(@Autowired var coordinator: Coordinator) {
|
||||||
|
|
||||||
|
@GetMapping()
|
||||||
|
fun polls(): String {
|
||||||
|
val stat = coordinator.getActivePolls()
|
||||||
|
return "Active Polls ${stat.active}/${stat.total}"
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,34 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.PeferenceConfig
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.ProcesserPreference
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.*
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/preference")
|
|
||||||
class PreferenceController(
|
|
||||||
private val preference: Preference
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping
|
|
||||||
fun getFull(): ResponseEntity<PeferenceConfig> =
|
|
||||||
ResponseEntity.ok(preference.getFullConfig())
|
|
||||||
|
|
||||||
@PutMapping
|
|
||||||
fun putFull(@RequestBody body: PeferenceConfig): ResponseEntity<PeferenceConfig> {
|
|
||||||
preference.saveFullConfig(body)
|
|
||||||
return ResponseEntity.ok(preference.getFullConfig())
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/processer")
|
|
||||||
fun getProcesser(): ResponseEntity<ProcesserPreference> =
|
|
||||||
ResponseEntity.ok(preference.getProcesserPreference())
|
|
||||||
|
|
||||||
@PutMapping("/processer")
|
|
||||||
fun putProcesser(@RequestBody body: ProcesserPreference): ResponseEntity<ProcesserPreference> {
|
|
||||||
preference.saveProcesserPreference(body)
|
|
||||||
return ResponseEntity.ok(preference.getProcesserPreference())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,28 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import org.springframework.boot.actuate.health.HealthEndpoint
|
|
||||||
import org.springframework.boot.actuate.health.Status
|
|
||||||
import org.springframework.http.HttpStatus
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
|
||||||
import org.springframework.web.bind.annotation.RestController
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/system")
|
|
||||||
class ReadinessController(
|
|
||||||
private val healthEndpoint: HealthEndpoint
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping("/ready")
|
|
||||||
fun ready(): ResponseEntity<String> {
|
|
||||||
val health = healthEndpoint.health()
|
|
||||||
|
|
||||||
return if (health.status == Status.UP) {
|
|
||||||
ResponseEntity.ok("READY")
|
|
||||||
} else {
|
|
||||||
ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
|
|
||||||
.body("NOT_READY: ${health.status}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator.controller
|
||||||
|
|
||||||
|
import com.google.gson.Gson
|
||||||
|
import no.iktdev.mediaprocessing.coordinator.Coordinator
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.ProcessType
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.dto.EventRequest
|
||||||
|
import no.iktdev.mediaprocessing.shared.common.contract.dto.StartOperationEvents
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.http.HttpStatus
|
||||||
|
import org.springframework.http.ResponseEntity
|
||||||
|
import org.springframework.stereotype.Controller
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping
|
||||||
|
import org.springframework.web.bind.annotation.ResponseStatus
|
||||||
|
import java.io.File
|
||||||
|
|
||||||
|
@Controller
|
||||||
|
@RequestMapping(path = ["/request"])
|
||||||
|
class RequestEventController(@Autowired var coordinator: Coordinator) {
|
||||||
|
|
||||||
|
@PostMapping("/convert")
|
||||||
|
@ResponseStatus(HttpStatus.OK)
|
||||||
|
fun requestConvert(@RequestBody payload: String): ResponseEntity<String> {
|
||||||
|
var convert: EventRequest? = null
|
||||||
|
var referenceId: String?
|
||||||
|
try {
|
||||||
|
convert = Gson().fromJson(payload, EventRequest::class.java)
|
||||||
|
val file = File(convert.file)
|
||||||
|
if (!file.exists()) {
|
||||||
|
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(convert.file)
|
||||||
|
}
|
||||||
|
referenceId = coordinator.startProcess(file, ProcessType.FLOW, listOf(StartOperationEvents.CONVERT)).toString()
|
||||||
|
|
||||||
|
} catch (e: Exception) {
|
||||||
|
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(Gson().toJson(convert))
|
||||||
|
}
|
||||||
|
return ResponseEntity.ok(referenceId)
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/extract")
|
||||||
|
@ResponseStatus(HttpStatus.OK)
|
||||||
|
fun requestExtract(@RequestBody payload: String): ResponseEntity<String> {
|
||||||
|
var request: EventRequest? = null
|
||||||
|
var referenceId: String?
|
||||||
|
try {
|
||||||
|
request = Gson().fromJson(payload, EventRequest::class.java)
|
||||||
|
val file = File(request.file)
|
||||||
|
if (!file.exists()) {
|
||||||
|
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload)
|
||||||
|
}
|
||||||
|
referenceId = coordinator.startProcess(file, ProcessType.MANUAL, listOf(StartOperationEvents.EXTRACT)).toString()
|
||||||
|
|
||||||
|
} catch (e: Exception) {
|
||||||
|
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(payload)
|
||||||
|
}
|
||||||
|
return ResponseEntity.ok(referenceId)
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/all")
|
||||||
|
@ResponseStatus(HttpStatus.OK)
|
||||||
|
fun requestAll(@RequestBody payload: String): ResponseEntity<String> {
|
||||||
|
var request: EventRequest? = null
|
||||||
|
var referenceId: String?
|
||||||
|
try {
|
||||||
|
request = Gson().fromJson(payload, EventRequest::class.java)
|
||||||
|
val file = File(request.file)
|
||||||
|
if (!file.exists()) {
|
||||||
|
return ResponseEntity.status(HttpStatus.NO_CONTENT).body(payload)
|
||||||
|
}
|
||||||
|
referenceId = coordinator.startProcess(file, type = ProcessType.MANUAL).toString()
|
||||||
|
|
||||||
|
} catch (e: Exception) {
|
||||||
|
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(payload)
|
||||||
|
}
|
||||||
|
return ResponseEntity.ok(referenceId)
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,84 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.translate.ApiResponse
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.SequenceAggregatorService
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.SequenceSummary
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
|
||||||
import org.springframework.http.HttpStatus
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.*
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/sequences")
|
|
||||||
class SequenceController(
|
|
||||||
private val aggregator: SequenceAggregatorService
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping("/active")
|
|
||||||
fun getActive(): List<SequenceSummary> {
|
|
||||||
return aggregator.getActiveSequences()
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/recent")
|
|
||||||
fun getRecent(
|
|
||||||
@RequestParam(defaultValue = "15") limit: Int
|
|
||||||
): List<SequenceSummary> {
|
|
||||||
return aggregator.getRecentSequences(limit)
|
|
||||||
}
|
|
||||||
|
|
||||||
@PostMapping("/{referenceId}/continue")
|
|
||||||
fun continueSequence(
|
|
||||||
@PathVariable referenceId: UUID
|
|
||||||
): ResponseEntity<ApiResponse> {
|
|
||||||
return try {
|
|
||||||
|
|
||||||
val id = EventStore.createManuallyContinueEvent(referenceId)
|
|
||||||
|
|
||||||
ResponseEntity.ok(
|
|
||||||
ApiResponse(
|
|
||||||
ok = true,
|
|
||||||
message = "Sequence continued, event $id created!"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
} catch (ex: Exception) {
|
|
||||||
ResponseEntity
|
|
||||||
.status(HttpStatus.INTERNAL_SERVER_ERROR)
|
|
||||||
.body(
|
|
||||||
ApiResponse(
|
|
||||||
ok = false,
|
|
||||||
message = ex.message ?: "Unknown error"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@PostMapping("/{referenceId}/delete")
|
|
||||||
fun deleteSequences(
|
|
||||||
@PathVariable referenceId: UUID
|
|
||||||
): ResponseEntity<ApiResponse> {
|
|
||||||
return try {
|
|
||||||
|
|
||||||
val id = EventStore.deleteSequence(referenceId)
|
|
||||||
|
|
||||||
ResponseEntity.ok(
|
|
||||||
ApiResponse(
|
|
||||||
ok = true,
|
|
||||||
message = "Sequence deleted, Event id for deletion marking is $id"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
} catch (ex: Exception) {
|
|
||||||
ResponseEntity
|
|
||||||
.status(HttpStatus.INTERNAL_SERVER_ERROR)
|
|
||||||
.body(
|
|
||||||
ApiResponse(
|
|
||||||
ok = false,
|
|
||||||
message = ex.message ?: "Unknown error"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,91 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.controller
|
|
||||||
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.translate.CoordinatorTaskTransferDto
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.translate.toCoordinatorTransferDto
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.EventService
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.services.TaskService
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.util.UtcNow
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.Paginated
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.ResetTaskResponse
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.TaskQuery
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.map
|
|
||||||
import org.springframework.http.HttpStatus
|
|
||||||
import org.springframework.http.ResponseEntity
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping
|
|
||||||
import org.springframework.web.bind.annotation.PathVariable
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping
|
|
||||||
import org.springframework.web.bind.annotation.RestController
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/tasks")
|
|
||||||
class TaskController(
|
|
||||||
private val taskService: TaskService,
|
|
||||||
private val eventService: EventService
|
|
||||||
) {
|
|
||||||
|
|
||||||
@GetMapping("/active")
|
|
||||||
fun getActiveTasks(): List<CoordinatorTaskTransferDto> {
|
|
||||||
val tasks = taskService.getActiveTasks()
|
|
||||||
val logEvents = eventService.getTaskEventResultsWithLogs(tasks.map { it.referenceId }.toSet())
|
|
||||||
return tasks.map { it.toCoordinatorTransferDto(logEvents) }
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping
|
|
||||||
fun getPagedTasks(query: TaskQuery): Paginated<CoordinatorTaskTransferDto> {
|
|
||||||
val paginatedTasks = taskService.getPagedTasks(query)
|
|
||||||
val logEvents = eventService.getTaskEventResultsWithLogs(paginatedTasks.items.map { it.referenceId }.toSet())
|
|
||||||
|
|
||||||
return paginatedTasks.map { it.toCoordinatorTransferDto(logEvents) }
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@GetMapping("/{id}")
|
|
||||||
fun getTask(@PathVariable id: UUID): CoordinatorTaskTransferDto? {
|
|
||||||
val tasks = taskService.getTaskById(id) ?: return null
|
|
||||||
val logEvents = eventService.getTaskEventResultsWithLogs(setOf(tasks.referenceId))
|
|
||||||
return tasks.toCoordinatorTransferDto(logEvents)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@GetMapping("/{taskId}/reset")
|
|
||||||
fun resetTask(@PathVariable taskId: UUID, forced: Boolean = false): ResponseEntity<ResetTaskResponse> {
|
|
||||||
val task = taskService.getTaskById(taskId)
|
|
||||||
?: return ResponseEntity.notFound().build()
|
|
||||||
|
|
||||||
val referenceId = task.referenceId
|
|
||||||
if (eventService.isSequenceDeleted(referenceId)) {
|
|
||||||
return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build()
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Opprett DeleteEvent
|
|
||||||
val deletedId = eventService.deleteTaskFailureForReset(referenceId, taskId)
|
|
||||||
if (deletedId == null) {
|
|
||||||
if (forced) {
|
|
||||||
eventService.createForcedTaskResetAuditEvent(referenceId, taskId)
|
|
||||||
} else {
|
|
||||||
return ResponseEntity.status(HttpStatus.CONFLICT).build()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Reset task
|
|
||||||
val success = taskService.resetFailedTask(taskId)
|
|
||||||
|
|
||||||
return ResponseEntity.ok(
|
|
||||||
ResetTaskResponse(
|
|
||||||
taskId = taskId,
|
|
||||||
referenceId = referenceId,
|
|
||||||
reset = success,
|
|
||||||
deletedEventId = deletedId,
|
|
||||||
resetAt = UtcNow()
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@GetMapping("/{taskId}/reset/force")
|
|
||||||
fun resetTaskForce(@PathVariable taskId: UUID): ResponseEntity<ResetTaskResponse> {
|
|
||||||
return resetTask(taskId, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
package no.iktdev.mediaprocessing.coordinator.coordination
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class to handle messages from websockets, produced by Processer instances.
|
||||||
|
* This is due to keep a overview of progress by processer
|
||||||
|
*/
|
||||||
|
class ProcesserSocketMessageListener {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@ -1,9 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto
|
|
||||||
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
data class LogAssociatedIds(
|
|
||||||
val referenceId: UUID,
|
|
||||||
val ids: Set<UUID>,
|
|
||||||
val logFile: String
|
|
||||||
)
|
|
||||||
@ -1,26 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto.health
|
|
||||||
|
|
||||||
import java.time.Instant
|
|
||||||
|
|
||||||
data class CoordinatorHealth(
|
|
||||||
val status: CoordinatorHealthStatus,
|
|
||||||
val abandonedTasks: Int,
|
|
||||||
val stalledTasks: Int,
|
|
||||||
val activeTasks: Int,
|
|
||||||
val queuedTasks: Int,
|
|
||||||
val failedTasks: Int,
|
|
||||||
val sequencesOnHold: Int,
|
|
||||||
val lastActivity: Instant?,
|
|
||||||
|
|
||||||
// IDs for UI linking
|
|
||||||
val abandonedTaskIds: List<String>,
|
|
||||||
val stalledTaskIds: List<String>,
|
|
||||||
val sequencesOnHoldIds: List<String>,
|
|
||||||
val overdueSequenceIds: List<String>,
|
|
||||||
|
|
||||||
|
|
||||||
// Detailed sequence info
|
|
||||||
val overdueSequences: List<SequenceHealth>,
|
|
||||||
|
|
||||||
val details: Map<String, Any?>
|
|
||||||
)
|
|
||||||
@ -1,7 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto.health
|
|
||||||
|
|
||||||
enum class CoordinatorHealthStatus {
|
|
||||||
HEALTHY,
|
|
||||||
DEGRADED,
|
|
||||||
UNHEALTHY
|
|
||||||
}
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto.health
|
|
||||||
|
|
||||||
import java.time.Duration
|
|
||||||
import java.time.Instant
|
|
||||||
|
|
||||||
data class SequenceHealth(
|
|
||||||
val referenceId: String,
|
|
||||||
|
|
||||||
val age: Duration,
|
|
||||||
val expected: Duration,
|
|
||||||
val lastEventAt: Instant,
|
|
||||||
val eventCount: Int,
|
|
||||||
|
|
||||||
// nye felter
|
|
||||||
val startTime: Instant,
|
|
||||||
val expectedFinishTime: Instant,
|
|
||||||
val overdueDuration: Duration,
|
|
||||||
val isOverdue: Boolean
|
|
||||||
)
|
|
||||||
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto.rate
|
|
||||||
|
|
||||||
data class EventRate(
|
|
||||||
val lastMinute: Long,
|
|
||||||
val lastFiveMinutes: Long
|
|
||||||
)
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto.translate
|
|
||||||
|
|
||||||
data class ApiResponse(
|
|
||||||
val ok: Boolean,
|
|
||||||
val message: String
|
|
||||||
)
|
|
||||||
@ -1,46 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.dto.translate
|
|
||||||
|
|
||||||
import no.iktdev.eventi.models.store.PersistedTask
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.LogAssociatedIds
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
|
||||||
import java.time.Instant
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
data class CoordinatorTaskTransferDto(
|
|
||||||
val id: Long,
|
|
||||||
val referenceId: UUID,
|
|
||||||
val status: String,
|
|
||||||
val taskId: UUID,
|
|
||||||
val task: String,
|
|
||||||
val data: String,
|
|
||||||
val claimed: Boolean,
|
|
||||||
val claimedBy: String?,
|
|
||||||
val consumed: Boolean,
|
|
||||||
val lastCheckIn: Instant?,
|
|
||||||
val persistedAt: Instant,
|
|
||||||
val logs: List<String> = emptyList(),
|
|
||||||
val abandoned: Boolean,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
fun PersistedTask.toCoordinatorTransferDto(logs: List<LogAssociatedIds>): CoordinatorTaskTransferDto {
|
|
||||||
val matchingLogs = logs
|
|
||||||
.filter { log -> log.ids.contains(taskId) }
|
|
||||||
.map { it.logFile }
|
|
||||||
|
|
||||||
return CoordinatorTaskTransferDto(
|
|
||||||
id = id,
|
|
||||||
referenceId = referenceId,
|
|
||||||
status = status.name,
|
|
||||||
taskId = taskId,
|
|
||||||
task = task,
|
|
||||||
data = data,
|
|
||||||
claimed = claimed,
|
|
||||||
claimedBy = claimedBy,
|
|
||||||
consumed = consumed,
|
|
||||||
lastCheckIn = lastCheckIn,
|
|
||||||
persistedAt = persistedAt,
|
|
||||||
logs = matchingLogs,
|
|
||||||
abandoned = TaskLifecycleRules.isAbandoned(consumed, persistedAt, lastCheckIn)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@ -1,33 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.listeners.SummaryEventListener
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class CollectEventsListener(eventStore: no.iktdev.eventi.stores.EventStore = EventStore) : SummaryEventListener(eventStore) {
|
|
||||||
|
|
||||||
private val log = KotlinLogging.logger {}
|
|
||||||
override fun shouldSummarize(fullHistory: List<Event>): Boolean {
|
|
||||||
val projection = CollectProjection(fullHistory)
|
|
||||||
if (projection.startedWith == null) return false
|
|
||||||
if (!projection.isWorkflowComplete()) return false
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun produceSummary(fullHistory: List<Event>): Event {
|
|
||||||
// Must have all relevant tasks completed
|
|
||||||
val eventIds = fullHistory.map { it.eventId }.toSet()
|
|
||||||
|
|
||||||
return CollectedEvent(eventIds).derivedOf(fullHistory.last())
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun summaryAlreadyExists(fullHistory: List<Event>): Boolean {
|
|
||||||
return fullHistory.any { it is CollectedEvent }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,31 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CompletedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.getName
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class CompletedListener: EventListener() {
|
|
||||||
val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
if (event !is StoreContentAndMetadataTaskResultEvent)
|
|
||||||
return null
|
|
||||||
val useEvent = event as StoreContentAndMetadataTaskResultEvent
|
|
||||||
if (useEvent.status != TaskStatus.Completed) {
|
|
||||||
log.info { "${useEvent.referenceId} - ${MigrateContentToStoreTaskResultEvent::class.getName()} is failed, thus no task will be created" }
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
return CompletedEvent().derivedOf(event)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,60 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ConvertTaskCreatedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.OperationType
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ProcesserExtractResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ConvertTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.io.File
|
|
||||||
import java.nio.file.Files
|
|
||||||
import java.nio.file.Path
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaCreateConvertTaskListener: EventListener() {
|
|
||||||
private val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
fun allowOverwrite(): Boolean {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
|
|
||||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
|
||||||
if (startedEvent.data.operation.isNotEmpty()) {
|
|
||||||
if (!startedEvent.data.operation.contains(OperationType.ConvertSubtitles))
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
val selectedEvent = event as? ProcesserExtractResultEvent ?: return null
|
|
||||||
if (selectedEvent.status != TaskStatus.Completed)
|
|
||||||
return null
|
|
||||||
|
|
||||||
|
|
||||||
val result = selectedEvent.data ?: return null
|
|
||||||
if (!Files.exists(Path.of(result.cachedOutputFile)))
|
|
||||||
return null
|
|
||||||
val useFile = File(result.cachedOutputFile)
|
|
||||||
|
|
||||||
val convertTask = ConvertTask(
|
|
||||||
data = ConvertTask.Data(
|
|
||||||
inputFile = result.cachedOutputFile,
|
|
||||||
language = result.language,
|
|
||||||
allowOverwrite = allowOverwrite(),
|
|
||||||
outputDirectory = useFile.parentFile.absolutePath,
|
|
||||||
outputFileName = useFile.nameWithoutExtension,
|
|
||||||
)
|
|
||||||
).derivedOf(event)
|
|
||||||
TaskStore.persist(convertTask)
|
|
||||||
|
|
||||||
return ConvertTaskCreatedEvent(convertTask.taskId).derivedOf(event)
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,49 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadTaskCreatedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MetadataSearchResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaCreateCoverDownloadTaskListener: EventListener() {
|
|
||||||
private val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
val useEvent = event as? MetadataSearchResultEvent ?: return null
|
|
||||||
if (useEvent.status != TaskStatus.Completed) {
|
|
||||||
log.warn("MetadataResult on ${event.referenceId} did not complete successfully")
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
val downloadData = useEvent.results.map {
|
|
||||||
val data = it.metadata
|
|
||||||
val outputFileName = "${data.title}-${data.source}"
|
|
||||||
CoverDownloadTask.CoverDownloadData(
|
|
||||||
url = it.metadata.cover,
|
|
||||||
source = it.metadata.source,
|
|
||||||
outputFileName = outputFileName
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
val downloadTasks = downloadData.map {
|
|
||||||
CoverDownloadTask(it)
|
|
||||||
.derivedOf(useEvent)
|
|
||||||
}
|
|
||||||
|
|
||||||
downloadTasks.forEach { TaskStore.persist(it) }
|
|
||||||
|
|
||||||
return CoverDownloadTaskCreatedEvent(
|
|
||||||
downloadTasks.map { it.taskId }
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,73 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.*
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeData
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.EncodeTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaCreateEncodeTaskListener(
|
|
||||||
private val preference: Preference
|
|
||||||
) : EventListener() {
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
val preference = preference.getProcesserPreference()
|
|
||||||
|
|
||||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
|
||||||
if (startedEvent.data.operation.isNotEmpty()) {
|
|
||||||
if (!startedEvent.data.operation.contains(OperationType.Encode))
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
val selectedEvent = event as? MediaTracksEncodeSelectedEvent ?: return null
|
|
||||||
val streams = history.filterIsInstance<MediaStreamParsedEvent>().firstOrNull()?.data ?: return null
|
|
||||||
|
|
||||||
val videoPreference = preference.videoPreference?.codec ?: VideoCodec.Hevc()
|
|
||||||
val audioPreference = preference.audioPreference?.codec ?: AudioCodec.Aac(channels = 2)
|
|
||||||
|
|
||||||
val audioTargets = mutableListOf<AudioTarget>(
|
|
||||||
AudioTarget(
|
|
||||||
index = selectedEvent.selectedAudioTrack,
|
|
||||||
codec = audioPreference
|
|
||||||
)
|
|
||||||
)
|
|
||||||
selectedEvent.selectedAudioExtendedTrack?.let {
|
|
||||||
audioTargets.add(AudioTarget(
|
|
||||||
index = it,
|
|
||||||
codec = audioPreference
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
val plan = MediaPlan(
|
|
||||||
videoTrack = VideoTarget(index = selectedEvent.selectedVideoTrack, codec = videoPreference),
|
|
||||||
audioTracks = audioTargets
|
|
||||||
)
|
|
||||||
val args = plan.toFfmpegArgs(streams.videoStream, streams.audioStream)
|
|
||||||
val filename = startedEvent.data.fileUri.let { File(it) }.nameWithoutExtension
|
|
||||||
val extension = plan.toContainer()
|
|
||||||
|
|
||||||
val task = EncodeTask(
|
|
||||||
data = EncodeData(
|
|
||||||
arguments = args,
|
|
||||||
outputFileName = "$filename.$extension",
|
|
||||||
inputFile = startedEvent.data.fileUri
|
|
||||||
)
|
|
||||||
).derivedOf(event)
|
|
||||||
|
|
||||||
|
|
||||||
TaskStore.persist(task)
|
|
||||||
return ProcesserEncodeTaskCreatedEvent(
|
|
||||||
taskCreated = task.taskId
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,79 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.dsl.SubtitleCodec
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleData
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.ExtractSubtitleTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.io.File
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaCreateExtractTaskListener(): EventListener() {
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
|
|
||||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
|
||||||
if (startedEvent.data.operation.isNotEmpty()) {
|
|
||||||
if (!startedEvent.data.operation.contains(OperationType.ExtractSubtitles))
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
val selectedEvent = event as? MediaTracksExtractSelectedEvent ?: return null
|
|
||||||
val streams = history.filterIsInstance<MediaStreamParsedEvent>().firstOrNull()?.data ?: return null
|
|
||||||
|
|
||||||
val selectedStreams: Map<Int, SubtitleStream> =
|
|
||||||
selectedEvent.selectedSubtitleTracks.mapNotNull { streamIndex ->
|
|
||||||
val stream = streams.subtitleStream.firstOrNull { it.index == streamIndex }
|
|
||||||
stream?.let { streams.subtitleStream.indexOf(it) to it }
|
|
||||||
}.toMap()
|
|
||||||
|
|
||||||
|
|
||||||
val entries = selectedStreams.mapNotNull { (idx, stream )->
|
|
||||||
toSubtitleArgumentData(idx, startedEvent.data.fileUri.let { File(it) }, stream)
|
|
||||||
}
|
|
||||||
|
|
||||||
val createdTaskIds: MutableList<UUID> = mutableListOf()
|
|
||||||
entries.forEach { entry ->
|
|
||||||
ExtractSubtitleTask(data = entry).derivedOf(event).also {
|
|
||||||
TaskStore.persist(it)
|
|
||||||
createdTaskIds.add(it.taskId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return ProcesserExtractTaskCreatedEvent(
|
|
||||||
tasksCreated = createdTaskIds
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
|
|
||||||
fun toSubtitleArgumentData(index: Int, inputFile: File, stream: SubtitleStream): ExtractSubtitleData? {
|
|
||||||
val codec = SubtitleCodec.getCodec(stream.codec_name) ?: return null
|
|
||||||
val extension = codec.getExtension()
|
|
||||||
|
|
||||||
// ffmpeg-args for å mappe og copy akkurat dette subtitle-sporet
|
|
||||||
val args = mutableListOf<String>()
|
|
||||||
args += listOf("-map", "0:s:$index")
|
|
||||||
args += codec.buildFfmpegArgs(stream)
|
|
||||||
|
|
||||||
val language = stream.tags.language?: return null
|
|
||||||
|
|
||||||
// outputfilnavn basert på index og extension
|
|
||||||
val outputFileName = "${inputFile.nameWithoutExtension}-${language}.${extension}"
|
|
||||||
|
|
||||||
return ExtractSubtitleData(
|
|
||||||
inputFile = inputFile.absolutePath,
|
|
||||||
arguments = args,
|
|
||||||
outputFileName = outputFileName,
|
|
||||||
language = language
|
|
||||||
)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,84 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.ListenerOrder
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MetadataSearchTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
import org.jetbrains.annotations.VisibleForTesting
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.util.*
|
|
||||||
import java.util.concurrent.ConcurrentHashMap
|
|
||||||
import java.util.concurrent.Executors
|
|
||||||
import java.util.concurrent.ScheduledFuture
|
|
||||||
import java.util.concurrent.TimeUnit
|
|
||||||
|
|
||||||
@Component
|
|
||||||
@ListenerOrder(5)
|
|
||||||
class MediaCreateMetadataSearchTaskListener: EventListener() {
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
internal val scheduledExpiries = ConcurrentHashMap<UUID, ScheduledFuture<*>>()
|
|
||||||
private val scheduler = Executors.newScheduledThreadPool(1)
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
|
|
||||||
val startedEvent = history.filterIsInstance<StartProcessingEvent>().firstOrNull() ?: return null
|
|
||||||
if (startedEvent.data.operation.isNotEmpty()) {
|
|
||||||
if (!startedEvent.data.operation.contains(OperationType.MetadataSearch))
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
// For replay
|
|
||||||
if (event is MetadataSearchTaskCreatedEvent) {
|
|
||||||
val hasResult = history.filter { it is MetadataSearchResultEvent }
|
|
||||||
.any { it.metadata.derivedFromId?.contains(event.taskId) == true }
|
|
||||||
|
|
||||||
if (!hasResult) {
|
|
||||||
scheduleTaskExpiry(event.taskId, event.eventId, event.referenceId)
|
|
||||||
}
|
|
||||||
} else if (event is MetadataSearchResultEvent) {
|
|
||||||
val cancelKeys = event.metadata.derivedFromId ?: emptySet()
|
|
||||||
scheduledExpiries.filter { it -> it.key in cancelKeys }.keys.forEach { key ->
|
|
||||||
scheduledExpiries.remove(key)?.cancel(true)
|
|
||||||
}
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
val useEvent = event as? MediaParsedInfoEvent ?: return null
|
|
||||||
|
|
||||||
val task = MetadataSearchTask(
|
|
||||||
MetadataSearchTask.SearchData(
|
|
||||||
searchTitles = useEvent.data.parsedSearchTitles,
|
|
||||||
collection = useEvent.data.parsedCollection
|
|
||||||
)
|
|
||||||
).derivedOf(useEvent)
|
|
||||||
TaskStore.persist(task)
|
|
||||||
val finalResult = MetadataSearchTaskCreatedEvent(task.taskId).derivedOf(useEvent)
|
|
||||||
scheduleTaskExpiry(task.taskId, finalResult.eventId, task.referenceId)
|
|
||||||
return finalResult
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun scheduleTaskExpiry(taskId: UUID, eventId: UUID, referenceId: UUID) {
|
|
||||||
if (scheduledExpiries.containsKey(taskId)) return
|
|
||||||
|
|
||||||
val future = scheduler.schedule({
|
|
||||||
// Hvis tasken fortsatt ikke har result/failed → marker som failed
|
|
||||||
TaskStore.claim(taskId, "Coordinator-MetadataSearchTaskListener-TimeoutScheduler")
|
|
||||||
TaskStore.markConsumed(taskId, TaskStatus.Failed)
|
|
||||||
val failureEvent = MetadataSearchResultEvent(
|
|
||||||
status = TaskStatus.Failed,
|
|
||||||
).apply { setFailed(listOf(taskId)) }
|
|
||||||
//publishEvent(MetadataSearchFailedEvent(taskId, "Timeout").derivedOf(referenceId))
|
|
||||||
scheduledExpiries.remove(taskId)
|
|
||||||
}, 10, TimeUnit.MINUTES)
|
|
||||||
|
|
||||||
scheduledExpiries[taskId] = future
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,76 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleItem
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleType
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaDetermineSubtitleTrackTypeListener: EventListener() {
|
|
||||||
fun ignoreSHD(): Boolean = true
|
|
||||||
fun ignoreCC(): Boolean = true
|
|
||||||
fun ignoreSongs(): Boolean = true
|
|
||||||
fun ignoreCommentary(): Boolean = true
|
|
||||||
|
|
||||||
val supportedCodecs = setOf(
|
|
||||||
"ass", "subrip", "webvtt", "vtt", "smi"
|
|
||||||
)
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
val useEvent = event as? MediaStreamParsedEvent ?: return null
|
|
||||||
|
|
||||||
val collected = useEvent.data.subtitleStream
|
|
||||||
.mapToType()
|
|
||||||
.excludeTypes()
|
|
||||||
.onlySupportedCodecs()
|
|
||||||
|
|
||||||
|
|
||||||
return MediaTracksDetermineSubtitleTypeEvent(
|
|
||||||
subtitleTrackItems = collected
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
fun getCommentaryFilters(): Set<String> = setOf("commentary", "comentary", "kommentar", "kommentaar")
|
|
||||||
fun getSongFilters(): Set<String> = setOf("song", "sign", "lyrics")
|
|
||||||
fun getClosedCaptionFilters(): Set<String> = setOf("closed caption", "cc", "close caption", "closed-caption", "cc.")
|
|
||||||
fun getSHDFilters(): Set<String> = setOf("shd", "hh", "hard of hearing", "hard-of-hearing")
|
|
||||||
|
|
||||||
private fun List<SubtitleItem>.excludeTypes(): List<SubtitleItem> {
|
|
||||||
return this.filter {
|
|
||||||
when (it.type) {
|
|
||||||
SubtitleType.Song -> !ignoreSongs()
|
|
||||||
SubtitleType.Commentary -> !ignoreCommentary()
|
|
||||||
SubtitleType.ClosedCaption -> !ignoreCC()
|
|
||||||
SubtitleType.SHD -> !ignoreSHD()
|
|
||||||
SubtitleType.Dialogue -> true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun List<SubtitleStream>.mapToType(): List<SubtitleItem> {
|
|
||||||
return this.map {
|
|
||||||
val title = it.tags.title?.lowercase() ?: ""
|
|
||||||
val type = when {
|
|
||||||
getCommentaryFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.Commentary
|
|
||||||
getSongFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.Song
|
|
||||||
getClosedCaptionFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.ClosedCaption
|
|
||||||
getSHDFilters().any { keyword -> title.contains(keyword) } -> SubtitleType.SHD
|
|
||||||
else -> SubtitleType.Dialogue
|
|
||||||
}
|
|
||||||
SubtitleItem(it, type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun List<SubtitleItem>.onlySupportedCodecs(): List<SubtitleItem> {
|
|
||||||
return this.filter { it.stream.codec_name in supportedCodecs }
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,82 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import com.google.gson.Gson
|
|
||||||
import com.google.gson.JsonObject
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.ListenerOrder
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.ParsedMediaStreams
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@ListenerOrder(4)
|
|
||||||
@Component
|
|
||||||
class MediaParseStreamsListener: EventListener() {
|
|
||||||
val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
if (event !is CoordinatorReadStreamsResultEvent) return null
|
|
||||||
if (event.status != TaskStatus.Completed)
|
|
||||||
return null
|
|
||||||
if (event.data == null) {
|
|
||||||
log.error { "No data to parse in CoordinatorReadStreamsResultEvent" }
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
val streams = parseStreams(event.data)
|
|
||||||
return MediaStreamParsedEvent(
|
|
||||||
data = streams
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
fun parseStreams(data: JsonObject?): ParsedMediaStreams {
|
|
||||||
val ignoreCodecs = listOf("png", "mjpeg")
|
|
||||||
val gson = Gson()
|
|
||||||
return try {
|
|
||||||
val jStreams = data!!.getAsJsonArray("streams")
|
|
||||||
|
|
||||||
val videoStreams = mutableListOf<VideoStream>()
|
|
||||||
val audioStreams = mutableListOf<AudioStream>()
|
|
||||||
val subtitleStreams = mutableListOf<SubtitleStream>()
|
|
||||||
|
|
||||||
for (streamJson in jStreams) {
|
|
||||||
val streamObject = streamJson.asJsonObject
|
|
||||||
if (!streamObject.has("codec_name")) continue
|
|
||||||
val codecName = streamObject.get("codec_name").asString
|
|
||||||
val codecType = streamObject.get("codec_type").asString
|
|
||||||
|
|
||||||
if (codecName in ignoreCodecs) continue
|
|
||||||
|
|
||||||
when (codecType) {
|
|
||||||
"video" -> videoStreams.add(gson.fromJson(streamObject, VideoStream::class.java))
|
|
||||||
"audio" -> audioStreams.add(gson.fromJson(streamObject, AudioStream::class.java))
|
|
||||||
"subtitle" -> subtitleStreams.add(gson.fromJson(streamObject, SubtitleStream::class.java))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
val parsedStreams = ParsedMediaStreams(
|
|
||||||
videoStream = videoStreams,
|
|
||||||
audioStream = audioStreams,
|
|
||||||
subtitleStream = subtitleStreams
|
|
||||||
)
|
|
||||||
parsedStreams
|
|
||||||
|
|
||||||
} catch (e: Exception) {
|
|
||||||
"Failed to parse data, its either not a valid json structure or expected and required fields are not present.".also {
|
|
||||||
log.error { it }
|
|
||||||
}
|
|
||||||
throw e
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,271 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.ListenerOrder
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.MediaType
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
@ListenerOrder(2)
|
|
||||||
@Component
|
|
||||||
class MediaParsedInfoListener : EventListener() {
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
val started = event as? StartProcessingEvent ?: return null
|
|
||||||
val file = File(started.data.fileUri)
|
|
||||||
|
|
||||||
val filename = file.guessDesiredFileName()
|
|
||||||
val collection = file.getDesiredCollection()
|
|
||||||
val searchTitles = file.guessSearchableTitle()
|
|
||||||
val mediaType = file.guessMovieOrSeries()
|
|
||||||
|
|
||||||
val episodeInfo = if (mediaType == MediaType.Serie) {
|
|
||||||
val serieInfo = file.guessSerieInfo()
|
|
||||||
MediaParsedInfoEvent.ParsedData.EpisodeInfo(
|
|
||||||
episodeNumber = serieInfo.episodeNumber,
|
|
||||||
seasonNumber = serieInfo.seasonNumber,
|
|
||||||
episodeTitle = serieInfo.episodeTitle,
|
|
||||||
)
|
|
||||||
} else null
|
|
||||||
|
|
||||||
return MediaParsedInfoEvent(
|
|
||||||
MediaParsedInfoEvent.ParsedData(
|
|
||||||
parsedFileName = filename,
|
|
||||||
parsedCollection = collection,
|
|
||||||
parsedSearchTitles = searchTitles,
|
|
||||||
mediaType = mediaType,
|
|
||||||
episodeInfo = episodeInfo,
|
|
||||||
)
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
|
|
||||||
fun String.getCleanedTitle(): String {
|
|
||||||
return this
|
|
||||||
.noBrackets()
|
|
||||||
.noParens()
|
|
||||||
.noResolutionAndAfter()
|
|
||||||
.noSourceTags()
|
|
||||||
.noYear()
|
|
||||||
.noDots()
|
|
||||||
.noTrailingOrLeading()
|
|
||||||
.noUnderscores()
|
|
||||||
.noExtraSpaces()
|
|
||||||
.trim()
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
fun String.noBrackets() = Regex("\\[.*?]").replace(this, " ")
|
|
||||||
fun String.noParens() = Regex("\\(.*?\\)").replace(this, " ")
|
|
||||||
fun String.noTrailingOrLeading() = Regex("^[^a-zA-Z0-9!,]+|[^a-zA-Z0-9!~,]+\$").replace(this, " ")
|
|
||||||
fun String.noResolutionAndAfter() = Regex("[0-9]+[pk].*", RegexOption.IGNORE_CASE).replace(this, "")
|
|
||||||
fun String.noSourceTags() =
|
|
||||||
Regex("(?i)(bluray|laserdisc|dvd|web|uhd|hd|htds|imax).*", RegexOption.IGNORE_CASE).replace(this, " ")
|
|
||||||
|
|
||||||
fun String.noUnderscores() = this.replace("_", " ")
|
|
||||||
fun String.noYear() = Regex("\\b\\d{4}\\b").replace(this.takeIf { !it.matches(Regex("^\\d{4}")) } ?: this, "")
|
|
||||||
fun String.noDots() = Regex("(?<!\\b(?:Dr|Mr|Ms|Mrs|Lt|Capt|Prof|St|Ave))\\.").replace(this, " ")
|
|
||||||
fun String.noExtraSpaces() = Regex("\\s{2,}").replace(this, " ")
|
|
||||||
fun String.fullTrim() = this.trim('.', ',', ' ', '_', '-')
|
|
||||||
|
|
||||||
|
|
||||||
fun File.guessMovieOrSeries(): MediaType {
|
|
||||||
val name = this.nameWithoutExtension.lowercase()
|
|
||||||
|
|
||||||
// Serie-mønstre: dekker alle vanlige shorthand og varianter
|
|
||||||
val seriesPatterns = listOf(
|
|
||||||
Regex("s\\d{1,2}e\\d{1,2}"), // S01E03, s1e5
|
|
||||||
Regex("\\d{1,2}x\\d{1,2}"), // 1x03, 2x10
|
|
||||||
Regex("season\\s*\\d+"), // Season 2
|
|
||||||
Regex("episode\\s*\\d+"), // Episode 5
|
|
||||||
Regex("ep\\s*\\d+"), // Ep05, Ep 5
|
|
||||||
Regex("s\\d{1,2}\\s*[- ]\\s*e\\d{1,2}"), // S1 - E5, S01 - E05
|
|
||||||
Regex("s\\d{1,2}\\s*ep\\s*\\d{1,2}"), // S1 Ep05
|
|
||||||
Regex("series\\s*\\d+"), // Series 2 (britisk stil)
|
|
||||||
Regex("s\\d{1,2}[. ]e\\d{1,2}") // S01.E02 eller S01 E02
|
|
||||||
)
|
|
||||||
|
|
||||||
if (seriesPatterns.any { it.containsMatchIn(name) }) {
|
|
||||||
return MediaType.Serie
|
|
||||||
}
|
|
||||||
|
|
||||||
// Film-mønstre: årstall (1900–2099) etter tittel
|
|
||||||
val moviePattern = Regex("\\b(19|20)\\d{2}\\b")
|
|
||||||
if (moviePattern.containsMatchIn(name)) {
|
|
||||||
return MediaType.Movie
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: hvis ingen mønstre passer, anta film
|
|
||||||
return MediaType.Movie
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
fun File.guessDesiredFileName(): String {
|
|
||||||
val type = this.guessMovieOrSeries()
|
|
||||||
return when (type) {
|
|
||||||
MediaType.Movie -> this.guessDesiredMovieTitle()
|
|
||||||
MediaType.Serie -> this.guessDesiredSerieTitle()
|
|
||||||
MediaType.Subtitle -> {
|
|
||||||
this.nameWithoutExtension
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fun File.getDesiredCollection(): String {
|
|
||||||
val collection = when (this.guessMovieOrSeries()) {
|
|
||||||
MediaType.Movie -> this.guessDesiredMovieTitle()
|
|
||||||
MediaType.Serie -> this.guessDesiredSerieTitle()
|
|
||||||
MediaType.Subtitle -> {
|
|
||||||
this.parentFile.parentFile.nameWithoutExtension
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return collection.noParens().noYear().split(" - ").first().trim()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return A fully cleaned title suitable to use for collection
|
|
||||||
*/
|
|
||||||
fun File.guessDesiredMovieTitle(): String {
|
|
||||||
val cleaned = this.nameWithoutExtension.getCleanedTitle()
|
|
||||||
val yearRegex = Regex("\\b(19|20)\\d{2}\\b")
|
|
||||||
val yearMatch = yearRegex.find(cleaned)
|
|
||||||
|
|
||||||
return if (yearMatch != null) {
|
|
||||||
val title = cleaned.replace(yearRegex, "").trim()
|
|
||||||
"${title} (${yearMatch.value})"
|
|
||||||
} else {
|
|
||||||
cleaned
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return A fully cleaned title including season and episode with possible episode title
|
|
||||||
*/
|
|
||||||
fun File.guessDesiredSerieTitle(): String {
|
|
||||||
val parsedSerieInfo = this.guessSerieInfo()
|
|
||||||
|
|
||||||
val tag = buildString {
|
|
||||||
append("S${(parsedSerieInfo.seasonNumber ?: 1).toString().padStart(2, '0')}")
|
|
||||||
append("E${(parsedSerieInfo.episodeNumber ?: 1).toString().padStart(2, '0')}")
|
|
||||||
if (parsedSerieInfo.revision != null) append(" (v$parsedSerieInfo.revision)")
|
|
||||||
}
|
|
||||||
|
|
||||||
return buildString {
|
|
||||||
append(parsedSerieInfo.serieTitle)
|
|
||||||
append(" - ")
|
|
||||||
append(tag)
|
|
||||||
if (parsedSerieInfo.episodeTitle.isNotEmpty()) {
|
|
||||||
append(" - ")
|
|
||||||
append(parsedSerieInfo.episodeTitle)
|
|
||||||
}
|
|
||||||
}.trim()
|
|
||||||
}
|
|
||||||
|
|
||||||
fun File.guessSerieInfo(): ParsedSerieInfo {
|
|
||||||
val raw = this.nameWithoutExtension
|
|
||||||
|
|
||||||
val seasonRegex = Regex("""(?i)(?:S|Season|Series)\s*(\d{1,2})""")
|
|
||||||
val episodeRegex = Regex("""(?i)(?:E|Episode|Ep)\s*(\d{1,3})""")
|
|
||||||
val revisionRegex = Regex("""(?i)\bv(\d+)\b""")
|
|
||||||
val seasonEpisodeRegex = Regex("""(?i)(\d{1,2})x(\d{1,2})(?:[vV](\d+))?""")
|
|
||||||
|
|
||||||
var season: Int? = null
|
|
||||||
var episode: Int? = null
|
|
||||||
var revision: Int? = null
|
|
||||||
var baseTitle = raw.getCleanedTitle()
|
|
||||||
var episodeTitle = ""
|
|
||||||
|
|
||||||
val seMatch = seasonEpisodeRegex.find(raw)
|
|
||||||
if (seMatch != null) {
|
|
||||||
season = seMatch.groupValues[1].toIntOrNull()
|
|
||||||
episode = seMatch.groupValues[2].toIntOrNull()
|
|
||||||
revision = seMatch.groupValues.getOrNull(3)?.toIntOrNull()
|
|
||||||
baseTitle = raw.substring(0, seMatch.range.first).getCleanedTitle()
|
|
||||||
episodeTitle = raw.substring(seMatch.range.last + 1).getCleanedTitle()
|
|
||||||
} else {
|
|
||||||
val seasonMatch = seasonRegex.find(raw)
|
|
||||||
val episodeMatch = episodeRegex.find(raw)
|
|
||||||
val revisionMatch = revisionRegex.find(raw)
|
|
||||||
|
|
||||||
season = seasonMatch?.groupValues?.get(1)?.toIntOrNull()
|
|
||||||
episode = episodeMatch?.groupValues?.get(1)?.toIntOrNull()
|
|
||||||
revision = revisionMatch?.groupValues?.get(1)?.toIntOrNull()
|
|
||||||
|
|
||||||
baseTitle = if (seasonMatch != null) {
|
|
||||||
raw.substring(0, seasonMatch.range.first).getCleanedTitle()
|
|
||||||
} else raw.getCleanedTitle()
|
|
||||||
|
|
||||||
episodeTitle = if (episodeMatch != null) {
|
|
||||||
raw.substring(episodeMatch.range.last + 1).getCleanedTitle()
|
|
||||||
} else ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: hvis baseTitle er tom eller bare inneholder S/E, bruk parent‑mappe
|
|
||||||
if (baseTitle.isBlank() || baseTitle.matches(Regex("""(?i)^s?\d+e?\d+$"""))) {
|
|
||||||
baseTitle = this.parentFile?.name?.getCleanedTitle() ?: "Dumb ways to die"
|
|
||||||
}
|
|
||||||
|
|
||||||
return ParsedSerieInfo(
|
|
||||||
serieTitle = baseTitle,
|
|
||||||
episodeNumber = episode ?: 1,
|
|
||||||
seasonNumber = season ?: 1,
|
|
||||||
revision = revision,
|
|
||||||
episodeTitle
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
fun File.guessSearchableTitle(): List<String> {
|
|
||||||
val cleaned = this.guessDesiredFileName()
|
|
||||||
.noResolutionAndAfter()
|
|
||||||
.noSourceTags()
|
|
||||||
.noDots()
|
|
||||||
.noExtraSpaces()
|
|
||||||
.fullTrim()
|
|
||||||
|
|
||||||
val titles = mutableListOf<String>()
|
|
||||||
|
|
||||||
val yearRegex = Regex("""\b(19|20)\d{2}\b""")
|
|
||||||
val hasYear = yearRegex.containsMatchIn(cleaned)
|
|
||||||
|
|
||||||
// 1. Hvis årstall finnes, legg hele cleaned først
|
|
||||||
if (hasYear) {
|
|
||||||
titles.add(cleaned)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Første del før bindestrek
|
|
||||||
val firstPart = cleaned.split(" - ").firstOrNull()?.trim() ?: cleaned
|
|
||||||
titles.add(firstPart)
|
|
||||||
|
|
||||||
// 3. Hele cleaned (hvis ikke allerede lagt inn først)
|
|
||||||
if (!hasYear) {
|
|
||||||
titles.add(cleaned)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Variant uten årstall
|
|
||||||
val noYear = yearRegex.replace(cleaned, "")
|
|
||||||
.noParens().trim()
|
|
||||||
if (noYear.isNotEmpty() && noYear != cleaned) {
|
|
||||||
titles.add(noYear)
|
|
||||||
}
|
|
||||||
|
|
||||||
return titles.distinct()
|
|
||||||
}
|
|
||||||
|
|
||||||
data class ParsedSerieInfo(
|
|
||||||
val serieTitle: String,
|
|
||||||
val episodeNumber: Int,
|
|
||||||
val seasonNumber: Int,
|
|
||||||
val revision: Int? = null,
|
|
||||||
val episodeTitle: String,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,33 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.ListenerOrder
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsTaskCreatedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaParsedInfoEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@ListenerOrder(3)
|
|
||||||
@Component
|
|
||||||
class MediaReadStreamsTaskCreatedListener: EventListener() {
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
if (event !is MediaParsedInfoEvent) return null
|
|
||||||
val startEvent = history.lastOrNull { it is StartProcessingEvent } as? StartProcessingEvent
|
|
||||||
?: return null
|
|
||||||
|
|
||||||
|
|
||||||
val readTask = MediaReadTask(
|
|
||||||
fileUri = startEvent.data.fileUri
|
|
||||||
).derivedOf(event)
|
|
||||||
|
|
||||||
TaskStore.persist(readTask)
|
|
||||||
return CoordinatorReadStreamsTaskCreatedEvent(readTask.taskId).derivedOf(event) // Create task instead of event
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,158 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.AudioStream
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.VideoStream
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaStreamParsedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksEncodeSelectedEvent
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaSelectEncodeTracksListener(
|
|
||||||
private val preference: Preference
|
|
||||||
) : EventListener() {
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
val useEvent = event as? MediaStreamParsedEvent ?: return null
|
|
||||||
|
|
||||||
val videoTrackIndex = getVideoTrackToUse(useEvent.data.videoStream)
|
|
||||||
val audioDefaultTrack = getAudioDefaultTrackToUse(useEvent.data.audioStream)
|
|
||||||
val audioExtendedTrack = getAudioExtendedTrackToUse(
|
|
||||||
useEvent.data.audioStream,
|
|
||||||
selectedDefaultTrack = audioDefaultTrack
|
|
||||||
)
|
|
||||||
|
|
||||||
return MediaTracksEncodeSelectedEvent(
|
|
||||||
selectedVideoTrack = videoTrackIndex,
|
|
||||||
selectedAudioTrack = audioDefaultTrack,
|
|
||||||
selectedAudioExtendedTrack = audioExtendedTrack
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// AUDIO SELECTION
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private fun getAudioDefaultTrackToUse(audioStreams: List<AudioStream>): Int {
|
|
||||||
val pref = preference.getLanguagePreference()
|
|
||||||
|
|
||||||
val selected = selectBestAudioStream(
|
|
||||||
streams = audioStreams,
|
|
||||||
preferredLanguages = pref.preferredAudio,
|
|
||||||
preferOriginal = pref.preferOriginal,
|
|
||||||
avoidDub = pref.avoidDub,
|
|
||||||
mode = AudioSelectMode.DEFAULT
|
|
||||||
) ?: audioStreams.firstOrNull()
|
|
||||||
|
|
||||||
return audioStreams.indexOf(selected)
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun getAudioExtendedTrackToUse(
|
|
||||||
audioStreams: List<AudioStream>,
|
|
||||||
selectedDefaultTrack: Int
|
|
||||||
): Int? {
|
|
||||||
val pref = preference.getLanguagePreference()
|
|
||||||
|
|
||||||
val candidates = audioStreams
|
|
||||||
.filter { it.index != selectedDefaultTrack }
|
|
||||||
.filter { (it.duration_ts ?: 0) > 0 }
|
|
||||||
.filter { it.channels > 2 }
|
|
||||||
|
|
||||||
val selected = selectBestAudioStream(
|
|
||||||
streams = candidates,
|
|
||||||
preferredLanguages = pref.preferredAudio,
|
|
||||||
preferOriginal = pref.preferOriginal,
|
|
||||||
avoidDub = pref.avoidDub,
|
|
||||||
mode = AudioSelectMode.EXTENDED
|
|
||||||
) ?: return null
|
|
||||||
|
|
||||||
return audioStreams.indexOf(selected)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// CORE AUDIO SELECTION LOGIC
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
private enum class AudioSelectMode { DEFAULT, EXTENDED }
|
|
||||||
|
|
||||||
private fun selectBestAudioStream(
|
|
||||||
streams: List<AudioStream>,
|
|
||||||
preferredLanguages: List<String>,
|
|
||||||
preferOriginal: Boolean,
|
|
||||||
avoidDub: Boolean,
|
|
||||||
mode: AudioSelectMode
|
|
||||||
): AudioStream? {
|
|
||||||
if (streams.isEmpty()) return null
|
|
||||||
|
|
||||||
// 1. Originalspråk
|
|
||||||
if (preferOriginal) {
|
|
||||||
val originals = streams.filter { it.disposition.original == 1 }
|
|
||||||
if (originals.isNotEmpty()) {
|
|
||||||
return when (mode) {
|
|
||||||
AudioSelectMode.DEFAULT -> originals.minByOrNull { it.channels }
|
|
||||||
AudioSelectMode.EXTENDED -> originals.maxByOrNull { it.channels }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Filtrer bort dub
|
|
||||||
val filtered = if (avoidDub) {
|
|
||||||
streams.filter { it.disposition.dub != 1 }
|
|
||||||
} else streams
|
|
||||||
|
|
||||||
// 3. Foretrukne språk
|
|
||||||
for (lang in preferredLanguages) {
|
|
||||||
val match = filtered.filter {
|
|
||||||
it.tags.language?.equals(lang, ignoreCase = true) == true
|
|
||||||
}
|
|
||||||
if (match.isNotEmpty()) {
|
|
||||||
return when (mode) {
|
|
||||||
AudioSelectMode.DEFAULT -> match.minByOrNull { it.channels }
|
|
||||||
AudioSelectMode.EXTENDED -> match.maxByOrNull { it.channels }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Default-flagget
|
|
||||||
val default = filtered.firstOrNull { it.disposition.default == 1 }
|
|
||||||
if (default != null) return default
|
|
||||||
|
|
||||||
// 5. Fallback
|
|
||||||
return filtered.firstOrNull()
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// QUALITY SCORE (no bitrate)
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun qualityScore(s: AudioStream): Int {
|
|
||||||
val channelsScore = s.channels * 10
|
|
||||||
val bitsScore = s.bits_per_sample
|
|
||||||
val sampleRateScore = s.sample_rate.toIntOrNull()?.div(1000) ?: 0
|
|
||||||
|
|
||||||
return channelsScore + bitsScore + sampleRateScore
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// VIDEO SELECTION
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun getVideoTrackToUse(streams: List<VideoStream>): Int {
|
|
||||||
val selectStream = streams
|
|
||||||
.filter { (it.duration_ts ?: 0) > 0 }
|
|
||||||
.maxByOrNull { it.duration_ts ?: 0 }
|
|
||||||
?: streams.minByOrNull { it.index }
|
|
||||||
?: throw Exception("No video streams found")
|
|
||||||
|
|
||||||
return streams.indexOf(selectStream)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,130 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.Preference
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.SubtitleSelectionMode
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.data.SubtitleStream
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksDetermineSubtitleTypeEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MediaTracksExtractSelectedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleItem
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.SubtitleType
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaSelectExtractTracksListener(
|
|
||||||
private val preference: Preference
|
|
||||||
) : EventListener() {
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
val useEvent = event as? MediaTracksDetermineSubtitleTypeEvent ?: return null
|
|
||||||
|
|
||||||
val pref = preference.getLanguagePreference()
|
|
||||||
|
|
||||||
// 1. Filter by subtitle type (dialogue, forced, etc.)
|
|
||||||
val filteredByType = filterBySelectionMode(
|
|
||||||
items = useEvent.subtitleTrackItems,
|
|
||||||
mode = pref.subtitleSelectionMode
|
|
||||||
)
|
|
||||||
|
|
||||||
// 2. Extract streams
|
|
||||||
val streams = filteredByType.map { it.stream }
|
|
||||||
|
|
||||||
// 3. Select subtitles based on language + format priority
|
|
||||||
val selected = selectSubtitleStreams(
|
|
||||||
streams = streams,
|
|
||||||
preferredLanguages = pref.preferredSubtitles,
|
|
||||||
preferOriginal = pref.preferOriginal,
|
|
||||||
formatPriority = pref.subtitleFormatPriority
|
|
||||||
)
|
|
||||||
|
|
||||||
return MediaTracksExtractSelectedEvent(
|
|
||||||
selectedSubtitleTracks = selected.map { it.index }
|
|
||||||
).derivedOf(event)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// TYPE FILTERING
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun filterBySelectionMode(
|
|
||||||
items: List<SubtitleItem>,
|
|
||||||
mode: SubtitleSelectionMode
|
|
||||||
): List<SubtitleItem> {
|
|
||||||
return when (mode) {
|
|
||||||
SubtitleSelectionMode.DialogueOnly ->
|
|
||||||
items.filter { it.type == SubtitleType.Dialogue }
|
|
||||||
|
|
||||||
SubtitleSelectionMode.DialogueAndForced ->
|
|
||||||
items.filter {
|
|
||||||
it.type == SubtitleType.Dialogue || it.stream.disposition?.forced == 1
|
|
||||||
}
|
|
||||||
|
|
||||||
SubtitleSelectionMode.All ->
|
|
||||||
items
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// SUBTITLE SELECTION LOGIC
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun selectSubtitleStreams(
|
|
||||||
streams: List<SubtitleStream>,
|
|
||||||
preferredLanguages: List<String>,
|
|
||||||
preferOriginal: Boolean,
|
|
||||||
formatPriority: List<String>
|
|
||||||
): List<SubtitleStream> {
|
|
||||||
if (streams.isEmpty()) return emptyList()
|
|
||||||
|
|
||||||
// 1. Originalspråk
|
|
||||||
if (preferOriginal) {
|
|
||||||
val originals = streams.filter { it.disposition?.original == 1 }
|
|
||||||
if (originals.isNotEmpty()) {
|
|
||||||
return originals.uniquePerLanguageBestFormat(formatPriority)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Preferred languages
|
|
||||||
for (lang in preferredLanguages) {
|
|
||||||
val match = streams.filter {
|
|
||||||
it.tags.language?.equals(lang, ignoreCase = true) == true ||
|
|
||||||
it.subtitle_tags.language?.equals(lang, ignoreCase = true) == true
|
|
||||||
}
|
|
||||||
if (match.isNotEmpty()) {
|
|
||||||
return match.uniquePerLanguageBestFormat(formatPriority)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Default subtitles
|
|
||||||
val defaults = streams.filter { it.disposition?.default == 1 }
|
|
||||||
if (defaults.isNotEmpty()) {
|
|
||||||
return defaults.uniquePerLanguageBestFormat(formatPriority)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Fallback: all subtitles
|
|
||||||
return streams.uniquePerLanguageBestFormat(formatPriority)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
// UNIQUE PER LANGUAGE + FORMAT PRIORITY
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun List<SubtitleStream>.uniquePerLanguageBestFormat(
|
|
||||||
formatPriority: List<String>
|
|
||||||
): List<SubtitleStream> {
|
|
||||||
return this
|
|
||||||
.groupBy { it.tags.language ?: it.subtitle_tags.language ?: "unknown" }
|
|
||||||
.mapNotNull { (_, langGroup) ->
|
|
||||||
langGroup
|
|
||||||
.sortedBy { s ->
|
|
||||||
val idx = formatPriority.indexOf(s.codec_name.lowercase())
|
|
||||||
if (idx == -1) Int.MAX_VALUE else idx
|
|
||||||
}
|
|
||||||
.firstOrNull()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,79 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ManualAllowCompletionEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskCreatedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.projection.MigrateContentProject
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MigrateCreateStoreTaskListener(
|
|
||||||
private val coordinatorEnv: CoordinatorEnv,
|
|
||||||
): EventListener() {
|
|
||||||
private val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
|
|
||||||
if (event !is CollectedEvent && event !is ManualAllowCompletionEvent)
|
|
||||||
return null
|
|
||||||
|
|
||||||
val useEvent = if (event is ManualAllowCompletionEvent) {
|
|
||||||
history.lastOrNull { it is CollectedEvent } as? CollectedEvent
|
|
||||||
?: return null
|
|
||||||
} else {
|
|
||||||
event as CollectedEvent
|
|
||||||
}
|
|
||||||
|
|
||||||
val useHistory = history.filter { useEvent.eventIds.contains(it.eventId) }
|
|
||||||
|
|
||||||
val collectProjection = CollectProjection(useHistory)
|
|
||||||
log.info { collectProjection.prettyPrint() }
|
|
||||||
|
|
||||||
val statusAcceptable = collectProjection.getTaskStatus().none { it == CollectProjection.TaskStatus.Failed }
|
|
||||||
if (!statusAcceptable) {
|
|
||||||
log.warn { "One or more tasks have failed in ${event.referenceId}" }
|
|
||||||
}
|
|
||||||
|
|
||||||
val migrateContentProjection = MigrateContentProject(useHistory, coordinatorEnv.outgoingContent)
|
|
||||||
|
|
||||||
val collection = migrateContentProjection.useStore?.name ?:
|
|
||||||
throw RuntimeException("No content store configured for migration in ${event.referenceId}")
|
|
||||||
|
|
||||||
val videContent = migrateContentProjection.getVideoStoreFile()?.let { MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath) }
|
|
||||||
val subtitleContent = migrateContentProjection.getSubtitleStoreFiles()?.map {
|
|
||||||
MigrateToContentStoreTask.Data.SingleSubtitle(it.language, it.cts.cachedFile.absolutePath, it.cts.storeFile.absolutePath, )
|
|
||||||
}
|
|
||||||
val coverContent = migrateContentProjection.getCoverStoreFiles()?.map {
|
|
||||||
MigrateToContentStoreTask.Data.SingleContent(it.cachedFile.absolutePath, it.storeFile.absolutePath)
|
|
||||||
}
|
|
||||||
val storeTask = MigrateToContentStoreTask(
|
|
||||||
MigrateToContentStoreTask.Data(
|
|
||||||
collection = collection,
|
|
||||||
videoContent = videContent,
|
|
||||||
subtitleContent = subtitleContent,
|
|
||||||
coverContent = coverContent
|
|
||||||
)
|
|
||||||
).derivedOf(event)
|
|
||||||
|
|
||||||
|
|
||||||
if (!CollectProjection(history).isStorePermitted()) {
|
|
||||||
log.info { "\uD83D\uDED1 Not storing content and metadata automatically for collection: $collection @ ${useEvent.referenceId}" }
|
|
||||||
log.info { "A manual allow completion event is required to proceed." }
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
TaskStore.persist(storeTask)
|
|
||||||
|
|
||||||
return MigrateContentToStoreTaskCreatedEvent(storeTask.taskId).derivedOf(useEvent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,31 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import no.iktdev.eventi.ListenerOrder
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.*
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@ListenerOrder(1)
|
|
||||||
@Component
|
|
||||||
class StartedListener : EventListener() {
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
val useEvent = event as? FileReadyEvent ?: return null
|
|
||||||
|
|
||||||
return StartProcessingEvent(
|
|
||||||
data = StartData(
|
|
||||||
flow = StartFlow.Auto,
|
|
||||||
fileUri = useEvent.data.fileUri,
|
|
||||||
operation = setOf(
|
|
||||||
OperationType.ExtractSubtitles,
|
|
||||||
OperationType.ConvertSubtitles,
|
|
||||||
OperationType.Encode,
|
|
||||||
OperationType.MetadataSearch
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,78 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.events
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.events.EventListener
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CollectedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.ManualAllowCompletionEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskCreatedEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.getName
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.ContentExport
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.projection.CollectProjection
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.projection.StoreProjection
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class StoreContentAndMetadataListener: EventListener() {
|
|
||||||
val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
override fun onEvent(
|
|
||||||
event: Event,
|
|
||||||
history: List<Event>
|
|
||||||
): Event? {
|
|
||||||
if (event !is MigrateContentToStoreTaskResultEvent && event !is ManualAllowCompletionEvent)
|
|
||||||
return null
|
|
||||||
|
|
||||||
val useEvent = if (event is ManualAllowCompletionEvent) {
|
|
||||||
history.lastOrNull { it is MigrateContentToStoreTaskResultEvent } as? MigrateContentToStoreTaskResultEvent
|
|
||||||
?: return null
|
|
||||||
} else {
|
|
||||||
event as MigrateContentToStoreTaskResultEvent
|
|
||||||
}
|
|
||||||
|
|
||||||
val collectionEvent = history.lastOrNull { it is CollectedEvent } as? CollectedEvent
|
|
||||||
?: return null
|
|
||||||
|
|
||||||
val useHistory = (history.filter { collectionEvent.eventIds.contains(it.eventId) }) + listOf(useEvent)
|
|
||||||
val projection = StoreProjection(useHistory)
|
|
||||||
|
|
||||||
val collection = projection.getCollection()
|
|
||||||
if (collection.isNullOrBlank()) {
|
|
||||||
log.error { "Collection is null @ ${useEvent.referenceId}" }
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
val metadata = projection.projectMetadata()
|
|
||||||
if (metadata == null) {
|
|
||||||
log.error { "Metadata is null @ ${useEvent.referenceId}"}
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
if (useEvent.status != TaskStatus.Completed) {
|
|
||||||
log.info { "${useEvent.referenceId} - ${MigrateContentToStoreTaskResultEvent::class.getName()} is failed, thus no task will be created" }
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!CollectProjection(history).isStorePermitted()) {
|
|
||||||
log.info { "\uD83D\uDED1 Not storing content and metadata automatically for collection: $collection @ ${useEvent.referenceId}" }
|
|
||||||
log.info { "A manual allow completion event is required to proceed." }
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
val exportInfo = ContentExport(
|
|
||||||
collection = collection,
|
|
||||||
media = projection.projectMediaFiles(),
|
|
||||||
episodeInfo = projection.projectEpisodeInfo(),
|
|
||||||
metadata = metadata
|
|
||||||
)
|
|
||||||
|
|
||||||
val task = StoreContentAndMetadataTask(exportInfo).derivedOf(useEvent)
|
|
||||||
TaskStore.persist(task)
|
|
||||||
|
|
||||||
return StoreContentAndMetadataTaskCreatedEvent(task.taskId).derivedOf(useEvent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,90 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.Task
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskListener
|
|
||||||
import no.iktdev.eventi.tasks.TaskType
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.DownloadClient
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoverDownloadResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.CoverDownloadTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.notExist
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class DownloadCoverTaskListener(
|
|
||||||
private val coordinatorEnv: CoordinatorEnv,
|
|
||||||
): TaskListener(TaskType.MIXED) {
|
|
||||||
val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
override fun getWorkerId(): String {
|
|
||||||
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun supports(task: Task): Boolean {
|
|
||||||
return task is CoverDownloadTask
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun onTask(task: Task): Event? {
|
|
||||||
val pickedTask = task as? CoverDownloadTask ?: return null
|
|
||||||
log.info { "Downloading cover from ${pickedTask.data.url}" }
|
|
||||||
val taskData = pickedTask.data
|
|
||||||
|
|
||||||
val downloadClient = getDownloadClient()
|
|
||||||
val downloadResult = try {
|
|
||||||
downloadClient.download(taskData.url, taskData.outputFileName)
|
|
||||||
} catch (e: Exception) {
|
|
||||||
return CoverDownloadResultEvent(status = TaskStatus.Failed)
|
|
||||||
}
|
|
||||||
val downloadedFile = downloadResult.result
|
|
||||||
|
|
||||||
if (downloadResult.success && downloadedFile != null) {
|
|
||||||
log.info { "Downloaded cover to ${downloadedFile.absolutePath}" }
|
|
||||||
return CoverDownloadResultEvent(
|
|
||||||
status = TaskStatus.Completed,
|
|
||||||
data = CoverDownloadResultEvent.CoverDownloadedData(
|
|
||||||
source = taskData.source,
|
|
||||||
outputFile = downloadedFile.absolutePath
|
|
||||||
)
|
|
||||||
).producedFrom(pickedTask)
|
|
||||||
} else {
|
|
||||||
log.error { "Failed to download cover from ${taskData.url}" }
|
|
||||||
return CoverDownloadResultEvent(
|
|
||||||
status = TaskStatus.Failed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun createIncompleteStateTaskEvent(
|
|
||||||
task: Task,
|
|
||||||
status: TaskStatus,
|
|
||||||
exception: Exception?
|
|
||||||
): Event {
|
|
||||||
val message = when (status) {
|
|
||||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
|
||||||
TaskStatus.Cancelled -> "Canceled"
|
|
||||||
else -> ""
|
|
||||||
}
|
|
||||||
return CoverDownloadResultEvent(null, status, error = message)
|
|
||||||
}
|
|
||||||
|
|
||||||
open fun getDownloadClient(): DownloadClient {
|
|
||||||
return DefaultDownloadClient(coordinatorEnv)
|
|
||||||
}
|
|
||||||
|
|
||||||
class DefaultDownloadClient(private val coordinatorEnv: CoordinatorEnv) : DownloadClient(
|
|
||||||
outDir = coordinatorEnv.cachedContent,
|
|
||||||
connectionFactory = DefaultConnectionFactory(),) {
|
|
||||||
override fun onCreate() {
|
|
||||||
super.onCreate()
|
|
||||||
if (outDir.notExist()) {
|
|
||||||
outDir.mkdirs()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
|
||||||
|
|
||||||
import no.iktdev.eventi.tasks.TaskListener
|
|
||||||
import no.iktdev.eventi.tasks.TaskType
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.FFprobe
|
|
||||||
|
|
||||||
abstract class FfprobeTaskListener(taskType: TaskType): TaskListener(taskType) {
|
|
||||||
abstract fun getFfprobe(): FFprobe
|
|
||||||
}
|
|
||||||
@ -1,71 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.Task
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskType
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
|
||||||
import no.iktdev.mediaprocessing.ffmpeg.FFprobe
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.CoordinatorReadStreamsResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MediaReadTask
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MediaStreamReadTaskListener(
|
|
||||||
private val coordinatorEnv: CoordinatorEnv
|
|
||||||
): FfprobeTaskListener(TaskType.CPU_INTENSIVE) {
|
|
||||||
val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
override fun getWorkerId(): String {
|
|
||||||
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun supports(task: Task): Boolean {
|
|
||||||
return task is MediaReadTask
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun onTask(task: Task): Event? {
|
|
||||||
val pickedTask = task as? MediaReadTask ?: return null
|
|
||||||
try {
|
|
||||||
val probeResult = getFfprobe()
|
|
||||||
.readJsonStreams(pickedTask.fileUri)
|
|
||||||
|
|
||||||
val result =
|
|
||||||
probeResult.data ?: throw RuntimeException("No data returned from ffprobe for ${pickedTask.fileUri}")
|
|
||||||
|
|
||||||
return CoordinatorReadStreamsResultEvent(
|
|
||||||
status = TaskStatus.Completed,
|
|
||||||
data = result
|
|
||||||
).producedFrom(task)
|
|
||||||
|
|
||||||
} catch (e: Exception) {
|
|
||||||
log.error(e) { "Error reading media streams for ${pickedTask.fileUri}" }
|
|
||||||
return CoordinatorReadStreamsResultEvent(
|
|
||||||
status = TaskStatus.Failed,
|
|
||||||
data = null
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun createIncompleteStateTaskEvent(
|
|
||||||
task: Task,
|
|
||||||
status: TaskStatus,
|
|
||||||
exception: Exception?
|
|
||||||
): Event {
|
|
||||||
val message = when (status) {
|
|
||||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
|
||||||
TaskStatus.Cancelled -> "Canceled"
|
|
||||||
else -> ""
|
|
||||||
}
|
|
||||||
return CoordinatorReadStreamsResultEvent(null, status, error = message)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun getFfprobe(): FFprobe {
|
|
||||||
return JsonFfinfo(coordinatorEnv.ffprobe)
|
|
||||||
}
|
|
||||||
|
|
||||||
class JsonFfinfo(executable: String): FFprobe(executable) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,197 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
|
||||||
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.Task
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskListener
|
|
||||||
import no.iktdev.eventi.tasks.TaskType
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.util.FileServiceException
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.util.FileSystemService
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.MigrateContentToStoreTaskResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.MigrateToContentStoreTask
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.model.MigrateStatus
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.silentTry
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import java.io.File
|
|
||||||
import java.nio.file.Files
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class MigrateContentToStoreTaskListener : TaskListener(TaskType.IO_INTENSIVE) {
|
|
||||||
|
|
||||||
override fun getWorkerId(): String =
|
|
||||||
"${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
|
||||||
|
|
||||||
override fun supports(task: Task): Boolean =
|
|
||||||
task is MigrateToContentStoreTask
|
|
||||||
|
|
||||||
override suspend fun onTask(task: Task): Event? {
|
|
||||||
val picked = task as? MigrateToContentStoreTask ?: return null
|
|
||||||
val fs = getFileSystemService()
|
|
||||||
|
|
||||||
val video = migrateVideo(fs, picked.data.videoContent)
|
|
||||||
val subs = migrateSubtitle(fs, picked.data.subtitleContent ?: emptyList())
|
|
||||||
val covers = migrateCover(fs, picked.data.coverContent ?: emptyList())
|
|
||||||
|
|
||||||
deleteCache(fs, picked)
|
|
||||||
|
|
||||||
return MigrateContentToStoreTaskResultEvent(
|
|
||||||
status = TaskStatus.Completed,
|
|
||||||
migrateData = MigrateContentToStoreTaskResultEvent.MigrateData(
|
|
||||||
collection = picked.data.collection,
|
|
||||||
videoMigrate = video,
|
|
||||||
subtitleMigrate = subs,
|
|
||||||
coverMigrate = covers
|
|
||||||
)
|
|
||||||
).producedFrom(task)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun createIncompleteStateTaskEvent(
|
|
||||||
task: Task,
|
|
||||||
status: TaskStatus,
|
|
||||||
exception: Exception?
|
|
||||||
): Event {
|
|
||||||
val message = when (status) {
|
|
||||||
TaskStatus.Failed -> exception?.message ?: "Unknown error"
|
|
||||||
TaskStatus.Cancelled -> "Canceled"
|
|
||||||
else -> ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return MigrateContentToStoreTaskResultEvent(
|
|
||||||
migrateData = null,
|
|
||||||
status = status,
|
|
||||||
error = message
|
|
||||||
).producedFrom(task)
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun deleteCache(fs: FileSystemService, task: MigrateToContentStoreTask) {
|
|
||||||
task.data.videoContent?.cachedUri?.let { silentTry { fs.delete(File(it)) } }
|
|
||||||
task.data.subtitleContent?.forEach { silentTry { fs.delete(File(it.cachedUri)) } }
|
|
||||||
task.data.coverContent?.forEach { silentTry { fs.delete(File(it.cachedUri)) } }
|
|
||||||
}
|
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
// MIGRATION HELPERS
|
|
||||||
// -------------------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun migrateFile(fs: FileSystemService, source: File, destination: File) {
|
|
||||||
if (destination.exists()) {
|
|
||||||
try {
|
|
||||||
fs.verifyIdentical(source, destination)
|
|
||||||
return
|
|
||||||
} catch (e: FileServiceException.VerificationFailed) {
|
|
||||||
throw FileServiceException.DestinationExistsButDifferent(source, destination)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fs.copy(source, destination)
|
|
||||||
fs.verifyIdentical(source, destination)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
internal fun migrateVideo(
|
|
||||||
fs: FileSystemService,
|
|
||||||
content: MigrateToContentStoreTask.Data.SingleContent?
|
|
||||||
): MigrateContentToStoreTaskResultEvent.FileMigration {
|
|
||||||
|
|
||||||
if (content == null) {
|
|
||||||
return MigrateContentToStoreTaskResultEvent.FileMigration(null, MigrateStatus.NotPresent)
|
|
||||||
}
|
|
||||||
|
|
||||||
val source = File(content.cachedUri)
|
|
||||||
val dest = File(content.storeUri)
|
|
||||||
|
|
||||||
migrateFile(fs, source, dest)
|
|
||||||
|
|
||||||
return MigrateContentToStoreTaskResultEvent.FileMigration(
|
|
||||||
storedUri = dest.absolutePath,
|
|
||||||
status = MigrateStatus.Completed
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
internal fun migrateSubtitle(
|
|
||||||
fs: FileSystemService,
|
|
||||||
subs: List<MigrateToContentStoreTask.Data.SingleSubtitle>
|
|
||||||
): List<MigrateContentToStoreTaskResultEvent.SubtitleMigration> {
|
|
||||||
|
|
||||||
if (subs.isEmpty()) {
|
|
||||||
return listOf(
|
|
||||||
MigrateContentToStoreTaskResultEvent.SubtitleMigration(
|
|
||||||
language = null,
|
|
||||||
storedUri = null,
|
|
||||||
status = MigrateStatus.NotPresent
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return subs.map { sub ->
|
|
||||||
val source = File(sub.cachedUri)
|
|
||||||
val dest = File(sub.storeUri)
|
|
||||||
|
|
||||||
migrateFile(fs, source, dest)
|
|
||||||
|
|
||||||
MigrateContentToStoreTaskResultEvent.SubtitleMigration(
|
|
||||||
language = sub.language,
|
|
||||||
storedUri = dest.absolutePath,
|
|
||||||
status = MigrateStatus.Completed
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
internal fun migrateCover(
|
|
||||||
fs: FileSystemService,
|
|
||||||
covers: List<MigrateToContentStoreTask.Data.SingleContent>
|
|
||||||
): List<MigrateContentToStoreTaskResultEvent.FileMigration> {
|
|
||||||
|
|
||||||
if (covers.isEmpty()) {
|
|
||||||
return listOf(
|
|
||||||
MigrateContentToStoreTaskResultEvent.FileMigration(
|
|
||||||
storedUri = null,
|
|
||||||
status = MigrateStatus.NotPresent
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return covers.map { cover ->
|
|
||||||
val source = File(cover.cachedUri)
|
|
||||||
val dest = File(cover.storeUri)
|
|
||||||
|
|
||||||
migrateFile(fs, source, dest)
|
|
||||||
|
|
||||||
MigrateContentToStoreTaskResultEvent.FileMigration(
|
|
||||||
storedUri = dest.absolutePath,
|
|
||||||
status = MigrateStatus.Completed
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
open fun getFileSystemService(): FileSystemService =
|
|
||||||
DefaultFileSystemService()
|
|
||||||
|
|
||||||
class DefaultFileSystemService : FileSystemService {
|
|
||||||
|
|
||||||
override fun copy(source: File, destination: File) {
|
|
||||||
if (!source.exists()) {
|
|
||||||
throw FileServiceException.SourceMissing(source)
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
source.copyTo(destination, overwrite = true)
|
|
||||||
} catch (e: Exception) {
|
|
||||||
throw FileServiceException.CopyFailed(source, destination, e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun verifyIdentical(source: File, destination: File) {
|
|
||||||
val mismatch = Files.mismatch(source.toPath(), destination.toPath())
|
|
||||||
if (mismatch != -1L) {
|
|
||||||
throw FileServiceException.VerificationFailed(source, destination)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun delete(file: File) {
|
|
||||||
file.delete()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,73 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.listeners.tasks
|
|
||||||
|
|
||||||
import no.iktdev.eventi.models.Event
|
|
||||||
import no.iktdev.eventi.models.Task
|
|
||||||
import no.iktdev.eventi.models.store.TaskStatus
|
|
||||||
import no.iktdev.eventi.tasks.TaskListener
|
|
||||||
import no.iktdev.eventi.tasks.TaskType
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StoreContentAndMetadataTaskResultEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.tasks.StoreContentAndMetadataTask
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired
|
|
||||||
import org.springframework.http.HttpEntity
|
|
||||||
import org.springframework.http.HttpHeaders
|
|
||||||
import org.springframework.http.HttpMethod
|
|
||||||
import org.springframework.http.MediaType
|
|
||||||
import org.springframework.stereotype.Component
|
|
||||||
import org.springframework.web.client.RestTemplate
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Component
|
|
||||||
class StoreContentAndMetadataTaskListener : TaskListener(TaskType.MIXED) {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
lateinit var streamitRestTemplate: RestTemplate
|
|
||||||
|
|
||||||
override fun getWorkerId(): String {
|
|
||||||
return "${this::class.java.simpleName}-${taskType}-${UUID.randomUUID()}"
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun supports(task: Task): Boolean {
|
|
||||||
return task is StoreContentAndMetadataTask
|
|
||||||
}
|
|
||||||
|
|
||||||
override suspend fun onTask(task: Task): Event? {
|
|
||||||
val pickedTask = task as? StoreContentAndMetadataTask ?: return null
|
|
||||||
|
|
||||||
val headers = HttpHeaders().apply { contentType = MediaType.APPLICATION_JSON }
|
|
||||||
val entity = HttpEntity(pickedTask.data, headers)
|
|
||||||
|
|
||||||
// ❗ Ikke fang exceptions — la TaskListener håndtere dem
|
|
||||||
val response = streamitRestTemplate.exchange(
|
|
||||||
"/api/mediaprocesser/import",
|
|
||||||
HttpMethod.POST,
|
|
||||||
entity,
|
|
||||||
Void::class.java,
|
|
||||||
)
|
|
||||||
|
|
||||||
if (!response.statusCode.is2xxSuccessful) {
|
|
||||||
throw IllegalStateException("StreamIt returned ${response.statusCode}")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hvis vi kommer hit → alt OK
|
|
||||||
return StoreContentAndMetadataTaskResultEvent(
|
|
||||||
status = TaskStatus.Completed
|
|
||||||
).producedFrom(task)
|
|
||||||
}
|
|
||||||
|
|
||||||
override fun createIncompleteStateTaskEvent(
|
|
||||||
task: Task,
|
|
||||||
status: TaskStatus,
|
|
||||||
exception: Exception?
|
|
||||||
): Event {
|
|
||||||
val message = when (status) {
|
|
||||||
TaskStatus.Failed -> exception?.message ?: "Unknown error, see log"
|
|
||||||
TaskStatus.Cancelled -> "Canceled"
|
|
||||||
else -> ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return StoreContentAndMetadataTaskResultEvent(
|
|
||||||
status = status,
|
|
||||||
error = message
|
|
||||||
).producedFrom(task)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,49 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.services
|
|
||||||
|
|
||||||
import mu.KotlinLogging
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.requests.StartProcessRequest
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartData
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartFlow
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.event_task_contract.events.StartProcessingEvent
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.notExist
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.EventStore
|
|
||||||
import org.springframework.stereotype.Service
|
|
||||||
import java.io.File
|
|
||||||
import java.util.*
|
|
||||||
|
|
||||||
@Service
|
|
||||||
class CommandService {
|
|
||||||
val log = KotlinLogging.logger {}
|
|
||||||
|
|
||||||
fun startProcess(request: StartProcessRequest): StartResult {
|
|
||||||
return try {
|
|
||||||
val file = File(request.fileUri)
|
|
||||||
if (file.notExist()) {
|
|
||||||
throw IllegalArgumentException("File does not exists at ${request.fileUri}")
|
|
||||||
}
|
|
||||||
if (!file.canRead()) {
|
|
||||||
throw IllegalStateException("File is not readable ${request.fileUri}")
|
|
||||||
}
|
|
||||||
|
|
||||||
val startProcessingEvent = StartProcessingEvent(
|
|
||||||
data = StartData(
|
|
||||||
fileUri = request.fileUri,
|
|
||||||
operation = request.operationTypes,
|
|
||||||
flow = StartFlow.Manual
|
|
||||||
)
|
|
||||||
).newReferenceId()
|
|
||||||
EventStore.persist(startProcessingEvent)
|
|
||||||
StartResult.Accepted(startProcessingEvent.referenceId)
|
|
||||||
} catch (e: Exception) {
|
|
||||||
log.error { e }
|
|
||||||
StartResult.Rejected("Failed to start process for file ${request.fileUri}, with the following reason: ${e.message}")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
sealed class StartResult {
|
|
||||||
data class Accepted(val referenceId: UUID) : StartResult()
|
|
||||||
data class Rejected(val reason: String) : StartResult()
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,132 +0,0 @@
|
|||||||
package no.iktdev.mediaprocessing.coordinator.services
|
|
||||||
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.CoordinatorEnv
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealth
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.health.CoordinatorHealthStatus
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.health.SequenceHealth
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.dto.rate.EventRate
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.util.DiskInfo
|
|
||||||
import no.iktdev.mediaprocessing.coordinator.util.getDiskInfoFor
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.dto.CurrentState
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.rules.EventLifecycleRules
|
|
||||||
import no.iktdev.mediaprocessing.shared.common.rules.TaskLifecycleRules
|
|
||||||
import no.iktdev.mediaprocessing.shared.database.stores.TaskStore
|
|
||||||
import org.springframework.stereotype.Service
|
|
||||||
import java.time.Duration
|
|
||||||
import java.time.Instant
|
|
||||||
|
|
||||||
@Service
|
|
||||||
class CoordinatorHealthService(
|
|
||||||
private val taskService: TaskService,
|
|
||||||
private val eventService: EventService,
|
|
||||||
private val aggregator: SequenceAggregatorService,
|
|
||||||
private val coordinatorEnv: CoordinatorEnv
|
|
||||||
) {
|
|
||||||
|
|
||||||
fun getHealth(): CoordinatorHealth {
|
|
||||||
val tasks = taskService.getActiveTasks()
|
|
||||||
val incompleteSequences = eventService.getIncompleteSequences()
|
|
||||||
.groupBy { it.referenceId }
|
|
||||||
.values
|
|
||||||
|
|
||||||
// --- TASK HEALTH ---
|
|
||||||
val abandonedTaskIds = tasks
|
|
||||||
.filter { TaskLifecycleRules.isAbandoned(it.consumed, it.persistedAt, it.lastCheckIn) }
|
|
||||||
.map { it.taskId }
|
|
||||||
|
|
||||||
val stalledTaskIds = tasks
|
|
||||||
.filter { TaskLifecycleRules.isStalled(it) }
|
|
||||||
.map { it.taskId }
|
|
||||||
|
|
||||||
val failedTasks = taskService.getFailedTasks()
|
|
||||||
val sequencesOnHold = aggregator.getActiveSequences().filter { it.currentState == CurrentState.OnHold }
|
|
||||||
|
|
||||||
// --- SEQUENCE HEALTH ---
|
|
||||||
val overdueSequences = incompleteSequences
|
|
||||||
.filter { EventLifecycleRules.isOverdue(it) }
|
|
||||||
.map { seq ->
|
|
||||||
val refId = seq.first().referenceId
|
|
||||||
val firstEventAt = seq.minOf { it.persistedAt }
|
|
||||||
val lastEventAt = seq.maxOf { it.persistedAt }
|
|
||||||
|
|
||||||
val expectedWindow = EventLifecycleRules.expectedCompletionTimeWindow(seq)
|
|
||||||
val actualAge = Duration.between(firstEventAt, Instant.now())
|
|
||||||
|
|
||||||
// Operasjonelle verdier frontend trenger
|
|
||||||
val expectedFinish = firstEventAt.plus(expectedWindow)
|
|
||||||
val overdueDuration = actualAge.minus(expectedWindow).coerceAtLeast(Duration.ZERO)
|
|
||||||
val isOverdue = overdueDuration > Duration.ZERO
|
|
||||||
|
|
||||||
SequenceHealth(
|
|
||||||
referenceId = refId.toString(),
|
|
||||||
|
|
||||||
// eksisterende felter
|
|
||||||
age = actualAge,
|
|
||||||
expected = expectedWindow,
|
|
||||||
lastEventAt = lastEventAt,
|
|
||||||
eventCount = seq.size,
|
|
||||||
|
|
||||||
// nye operasjonelle felter
|
|
||||||
startTime = firstEventAt,
|
|
||||||
expectedFinishTime = expectedFinish,
|
|
||||||
overdueDuration = overdueDuration,
|
|
||||||
isOverdue = isOverdue
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
val overdueSequenceIds = overdueSequences.map { it.referenceId }
|
|
||||||
|
|
||||||
// --- AGGREGATED STATUS ---
|
|
||||||
val status = when {
|
|
||||||
abandonedTaskIds.isNotEmpty() ||
|
|
||||||
stalledTaskIds.isNotEmpty() ||
|
|
||||||
overdueSequenceIds.isNotEmpty() -> CoordinatorHealthStatus.DEGRADED
|
|
||||||
|
|
||||||
else -> CoordinatorHealthStatus.HEALTHY
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
val lastActivityCandidates = listOfNotNull(
|
|
||||||
tasks.maxOfOrNull { it.persistedAt },
|
|
||||||
eventService.getLastEventTimestamp()
|
|
||||||
)
|
|
||||||
|
|
||||||
return CoordinatorHealth(
|
|
||||||
status = status,
|
|
||||||
abandonedTasks = abandonedTaskIds.size,
|
|
||||||
stalledTasks = stalledTaskIds.size,
|
|
||||||
activeTasks = tasks.count { !it.consumed },
|
|
||||||
failedTasks = failedTasks.count(),
|
|
||||||
queuedTasks = TaskStore.getPendingTasks().size,
|
|
||||||
lastActivity = lastActivityCandidates.maxOrNull(),
|
|
||||||
|
|
||||||
abandonedTaskIds = abandonedTaskIds.map { it.toString() },
|
|
||||||
stalledTaskIds = stalledTaskIds.map { it.toString() },
|
|
||||||
sequencesOnHold = sequencesOnHold.size,
|
|
||||||
sequencesOnHoldIds = sequencesOnHold.map { it.referenceId },
|
|
||||||
overdueSequenceIds = overdueSequenceIds,
|
|
||||||
overdueSequences = overdueSequences,
|
|
||||||
|
|
||||||
details = mapOf(
|
|
||||||
"oldestActiveTaskAgeMinutes" to tasks.minOfOrNull {
|
|
||||||
Duration.between(it.persistedAt, Instant.now()).toMinutes()
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getEventRate(): EventRate {
|
|
||||||
return EventRate(
|
|
||||||
lastMinute = eventService.getEventsLast(1),
|
|
||||||
lastFiveMinutes = eventService.getEventsLast(5)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getDiskHealth(): List<DiskInfo> {
|
|
||||||
val paths = listOf(coordinatorEnv.incomingContent,
|
|
||||||
coordinatorEnv.cachedContent,
|
|
||||||
coordinatorEnv.outgoingContent)
|
|
||||||
.map { it -> it.absolutePath }
|
|
||||||
return getDiskInfoFor(paths)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user