From 858b66a883b9fdba2acd672eb0d45dc377c1c8cb Mon Sep 17 00:00:00 2001 From: Brage Date: Fri, 12 Jan 2024 20:46:45 +0100 Subject: [PATCH] v2 --- .github/workflows/v2.yml | 253 ++++++++++++++++++ .idea/gradle.xml | 1 - .../converter/tasks/ConvertService.kt | 19 +- apps/coordinator/build.gradle.kts | 2 + .../coordinator/Coordinator.kt | 7 +- .../coordinator/CoordinatorApplication.kt | 44 ++- .../coordinator/mapping/MetadataMapping.kt | 47 +++- .../coordinator/mapping/OutputFilesMapping.kt | 33 +++ .../coordinator/mapping/ProcessMapping.kt | 41 +-- .../coordinator/mapping/VideoDetailsMapper.kt | 27 ++ .../tasks/event/BaseInfoFromFile.kt | 2 +- .../tasks/event/CollectAndStoreTask.kt | 144 ++++++++++ .../coordinator/tasks/event/CompleteTask.kt | 56 ++++ .../tasks/event/DownloadAndStoreCoverTask.kt | 57 ++++ .../event/MetadataAndBaseInfoToCoverTask.kt | 2 +- .../event/MetadataAndBaseInfoToFileOut.kt | 2 +- .../tasks/event/ParseVideoFileStreams.kt | 2 +- .../tasks/event/ReadVideoFileStreams.kt | 2 +- .../event/ffmpeg/EncodeArgumentCreatorTask.kt | 2 +- .../ffmpeg/ExtractArgumentCreatorTask.kt | 2 +- .../input/watcher/InputDirectoryWatcher.kt | 2 + .../reader/BaseInfoFromFileTest.kt | 2 +- .../reader/ParseVideoFileStreamsTest.kt | 2 +- .../processer/services/EncodeService.kt | 10 +- .../processer/services/ExtractService.kt | 12 +- apps/pyMetadata/app.py | 7 +- apps/pyMetadata/sources/anii.py | 15 +- apps/pyMetadata/sources/imdb.py | 13 +- apps/pyMetadata/sources/mal.py | 15 +- apps/pyMetadata/sources/result.py | 11 +- apps/pyMetadata/sources/select.py | 6 +- settings.gradle.kts | 25 +- .../shared/common/DownloadClient.kt | 15 +- .../datasource/TableDefaultOperations.kt | 26 +- .../common/persistance/PersistentDataStore.kt | 2 +- .../shared/common/persistance/events.kt | 1 + .../contract/reader/MediaProcessedDto.kt | 2 + .../shared/contract/reader/MetadataDto.kt | 16 +- .../shared/contract/reader/OutputFilesDto.kt | 10 +- .../shared/contract/reader/VideoDetails.kt | 14 + .../kafka/core/DeserializingRegistry.kt | 14 +- .../shared/kafka/core/KafkaEvents.kt | 3 +- .../shared/kafka/dto/Message.kt | 1 - .../shared/kafka/dto/MessageDataWrapper.kt | 2 - .../shared/kafka/dto/Status.kt | 4 +- .../dto/events_result/BaseInfoPerformed.kt | 2 +- .../dto/events_result/ConvertWorkPerformed.kt | 4 +- .../dto/events_result/ConvertWorkerRequest.kt | 4 +- ...tInfo.kt => CoverDownloadWorkPerformed.kt} | 12 +- .../dto/events_result/CoverInfoPerformed.kt | 5 +- .../events_result/FfmpegWorkRequestCreated.kt | 4 +- .../FfmpegWorkerArgumentsCreated.kt | 2 +- .../dto/events_result/MediaEncodeInfo.kt | 13 - .../dto/events_result/MediaExtractInfo.kt | 12 - .../MediaStreamsParsePerformed.kt | 2 +- .../dto/events_result/MetadataPerformed.kt | 9 +- .../dto/events_result/ProcessCompleted.kt | 6 +- .../kafka/dto/events_result/ProcessStarted.kt | 2 +- .../dto/events_result/ReaderPerformed.kt | 2 +- .../dto/events_result/VideoInfoPerformed.kt | 2 +- .../work/ProcesserEncodeWorkPerformed.kt | 18 ++ .../ProcesserExtractWorkPerformed.kt} | 11 +- .../shared/kafka/SerializationTest.kt | 2 +- 63 files changed, 908 insertions(+), 179 deletions(-) create mode 100644 .github/workflows/v2.yml create mode 100644 apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/OutputFilesMapping.kt create mode 100644 apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/mapping/VideoDetailsMapper.kt create mode 100644 apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/CollectAndStoreTask.kt create mode 100644 apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/CompleteTask.kt create mode 100644 apps/coordinator/src/main/kotlin/no/iktdev/mediaprocessing/coordinator/tasks/event/DownloadAndStoreCoverTask.kt create mode 100644 shared/contract/src/main/kotlin/no/iktdev/mediaprocessing/shared/contract/reader/VideoDetails.kt rename shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/{MediaConvertInfo.kt => CoverDownloadWorkPerformed.kt} (52%) delete mode 100644 shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaEncodeInfo.kt delete mode 100644 shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/MediaExtractInfo.kt create mode 100644 shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/work/ProcesserEncodeWorkPerformed.kt rename shared/kafka/src/main/kotlin/no/iktdev/mediaprocessing/shared/kafka/dto/events_result/{FfmpegWorkPerformed.kt => work/ProcesserExtractWorkPerformed.kt} (70%) diff --git a/.github/workflows/v2.yml b/.github/workflows/v2.yml new file mode 100644 index 00000000..367f5f6b --- /dev/null +++ b/.github/workflows/v2.yml @@ -0,0 +1,253 @@ +name: Build V2 + +on: + push: + branches: + - v2 + pull_request: + branches: + - v2 + workflow_dispatch: + + +jobs: + pre-check: + runs-on: ubuntu-latest + outputs: + pyMetadata: ${{ steps.filter.outputs.pyMetadata }} + commonCode: ${{ steps.filter.outputs.commonCode }} + reader: ${{ steps.filter.outputs.reader }} + encode: ${{ steps.filter.outputs.encode }} + convert: ${{ steps.filter.outputs.convert }} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + apps/pyMetadata: + - 'pyMetadata/**' + apps/coordinator: + - 'coordinator/**' + apps/processer: + - 'processer/**' + apps/converter: + - 'converter/**' + + shared: + - 'shared/**' + # Step to print the outputs from "pre-check" job + - name: Print Outputs from pre-check job + run: | + echo "Apps" + echo "app:pyMetadata: ${{ needs.pre-check.outputs.pyMetadata }}" + echo "app:coordinator: ${{ needs.pre-check.outputs.coordinator }}" + echo "app:processer: ${{ needs.pre-check.outputs.processer }}" + echo "app:converter: ${{ needs.pre-check.outputs.converter }}" + + echo "Shared" + echo "shared: ${{ needs.pre-check.outputs.shared }}" + + build-shared: + runs-on: ubuntu-latest + needs: pre-check + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache Shared code Gradle dependencies + id: cache-gradle + uses: actions/cache@v2 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }} + + - name: Build Shared code + if: steps.cache-gradle.outputs.cache-hit != 'true' || needs.pre-check.outputs.commonCode == 'true' || github.event_name == 'workflow_dispatch' + run: | + cd CommonCode + chmod +x ./gradlew + ./gradlew build + + + build-processer: + needs: build-shared + if: ${{ needs.pre-check.outputs.processer == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }} + runs-on: ubuntu-latest + #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache Shared Gradle dependencies + id: cache-gradle + uses: actions/cache@v2 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }} + + - name: Build Processer module + id: build-processer + run: | + cd apps/processer + chmod +x ./gradlew + ./gradlew build + echo "Build completed" + + + - name: Generate Docker image tag + id: docker-tag + run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" + + - name: Docker login + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_HUB_NAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: ./apps/processer + push: true + tags: | + bskjon/mediaprocessing-processer:v2-latest + bskjon/mediaprocessing-processer:v2-${{ github.sha }} + bskjon/mediaprocessing-processer:v2-${{ steps.docker-tag.outputs.tag }} + + build-converter: + needs: build-shared + if: ${{ needs.pre-check.outputs.converter == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }} + runs-on: ubuntu-latest + #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache Shared Gradle dependencies + id: cache-gradle + uses: actions/cache@v2 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }} + + - name: Build Converter module + id: build-converter + run: | + cd apps/converter + chmod +x ./gradlew + ./gradlew build + echo "Build completed" + + + - name: Generate Docker image tag + id: docker-tag + run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" + + - name: Docker login + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_HUB_NAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: ./apps/converter + push: true + tags: | + bskjon/mediaprocessing-converter:v2-latest + bskjon/mediaprocessing-converter:v2-${{ github.sha }} + bskjon/mediaprocessing-converter:v2-${{ steps.docker-tag.outputs.tag }} + + build-coordinator: + needs: build-shared + if: ${{ needs.pre-check.outputs.coordinator == 'true' || github.event_name == 'workflow_dispatch' || needs.pre-check.outputs.shared == 'true' }} + runs-on: ubuntu-latest + #if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache Shared Gradle dependencies + id: cache-gradle + uses: actions/cache@v2 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('shared/build.gradle.kts') }} + + - name: Build Coordinator module + id: build-coordinator + run: | + cd apps/coordinator + chmod +x ./gradlew + ./gradlew build + echo "Build completed" + + + - name: Generate Docker image tag + id: docker-tag + run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" + + - name: Docker login + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_HUB_NAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: ./apps/coordinator + push: true + tags: | + bskjon/mediaprocessing-coordinator:v2-latest + bskjon/mediaprocessing-coordinator:v2-${{ github.sha }} + bskjon/mediaprocessing-coordinator:v2-${{ steps.docker-tag.outputs.tag }} + + build-pymetadata: + needs: pre-check + if: ${{ needs.pre-check.outputs.pyMetadata == 'true' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Build pyMetadata module + id: build-pymetadata + run: | + if [[ "${{ steps.check-pymetadata.outputs.changed }}" == "true" || "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + cd apps/pyMetadata + # Add the necessary build steps for your Python module here + echo "Build completed" + else + echo "pyMetadata has not changed. Skipping pyMetadata module build." + echo "::set-output name=job_skipped::true" + fi + + - name: Generate Docker image tag + id: docker-tag + run: echo "::set-output name=tag::$(date -u +'%Y.%m.%d')-$(uuidgen | cut -c 1-8)" + + - name: Docker login + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_HUB_NAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: ./apps/pyMetadata + push: true + tags: | + bskjon/mediaprocessing-pymetadata:v2-latest + bskjon/mediaprocessing-pymetadata:v2-${{ github.sha }} + bskjon/mediaprocessing-pymetadata:v2-${{ steps.docker-tag.outputs.tag }} \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml index 64cf64d5..b74e6df4 100644 --- a/.idea/gradle.xml +++ b/.idea/gradle.xml @@ -11,7 +11,6 @@