From 0f685e62f6470c14fa80b3b3645f7929c0b03e75 Mon Sep 17 00:00:00 2001 From: TianZerL Date: Sat, 16 May 2020 16:51:25 +0800 Subject: [PATCH] Add video processing supports for Android --- .../tianzerl/anime4kcpp/Anime4KCPP.java | 12 +- .../tianzerl/anime4kcpp/Anime4KCPPGPU.java | 8 +- .../tianzerl/anime4kcpp/MainActivity.java | 32 +++-- .../anime4kcpp/VideoAudioProcessor.java | 112 ++++++++++++++++++ .../res/values/ic_launcher_background.xml | 2 +- Android/app/src/main/res/values/strings.xml | 4 +- Anime4KCore/include/Anime4K.h | 4 +- Anime4KCore/src/Anime4K.cpp | 2 +- cmake/ThirdPartyForCLI.cmake | 2 +- cmake/ThirdPartyForGUI.cmake | 2 +- cmake/ThirdPartyForVS.cmake | 2 +- 11 files changed, 154 insertions(+), 28 deletions(-) create mode 100644 Android/app/src/main/java/github/tianzerl/anime4kcpp/VideoAudioProcessor.java diff --git a/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPP.java b/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPP.java index 5e98bd86..f1822b14 100644 --- a/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPP.java +++ b/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPP.java @@ -70,15 +70,15 @@ public void setVideoMode(boolean flag) { setVideoModeAnime4KCPP(anime4k, flag); } - public void loadImage(final String src) { + public void loadImage(final String src) throws Exception { loadImageAnime4KCPP(anime4k, src); } - public void loadVideo(final String src) { + public void loadVideo(final String src) throws Exception { loadVideoAnime4KCPP(anime4k, src); } - public void setVideoSaveInfo(final String dst) { + public void setVideoSaveInfo(final String dst) throws Exception { setVideoSaveInfoAnime4KCPP(anime4k, dst); } @@ -130,9 +130,9 @@ protected native void setArgumentsAnime4KCPP(long ptr, byte postFilters); protected native void setVideoModeAnime4KCPP(long ptr, boolean flag); - protected native void loadImageAnime4KCPP(long ptr, final String src); - protected native void loadVideoAnime4KCPP(long ptr, final String src); - protected native void setVideoSaveInfoAnime4KCPP(long ptr, final String dst); + protected native void loadImageAnime4KCPP(long ptr, final String src) throws Exception; + protected native void loadVideoAnime4KCPP(long ptr, final String src) throws Exception; + protected native void setVideoSaveInfoAnime4KCPP(long ptr, final String dst) throws Exception; protected native void processAnime4KCPP(long ptr); protected native void saveImageAnime4KCPP(long ptr, final String dst); protected native void saveVideoAnime4KCPP(long ptr); diff --git a/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPPGPU.java b/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPPGPU.java index 892de645..7cbe5c88 100644 --- a/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPPGPU.java +++ b/Android/app/src/main/java/github/tianzerl/anime4kcpp/Anime4KCPPGPU.java @@ -6,7 +6,7 @@ public static boolean checkGPUSupport() { return checkGPUSupportAnime4KCPPGPU(); } - public Anime4KCPPGPU() { + public Anime4KCPPGPU() throws Exception { anime4k = createAnime4KCPPGPU(); } @@ -21,7 +21,7 @@ public Anime4KCPPGPU( boolean preprocessing, boolean postprocessing, byte preFilters, - byte postFilters) { + byte postFilters) throws Exception { anime4k = createAnime4KCPPGPUByArgs( passes, pushColorCount, @@ -43,7 +43,7 @@ protected void finalize() throws Throwable { releaseAnime4KCPPGPU(anime4k); } - private native long createAnime4KCPPGPU(); + private native long createAnime4KCPPGPU() throws Exception; private native long createAnime4KCPPGPUByArgs(int passes, int pushColorCount, double strengthColor, @@ -54,7 +54,7 @@ private native long createAnime4KCPPGPUByArgs(int passes, boolean preprocessing, boolean postProcessing, byte preFilters, - byte postFilters); + byte postFilters) throws Exception; private native void releaseAnime4KCPPGPU(long ptr); protected native static boolean checkGPUSupportAnime4KCPPGPU(); } diff --git a/Android/app/src/main/java/github/tianzerl/anime4kcpp/MainActivity.java b/Android/app/src/main/java/github/tianzerl/anime4kcpp/MainActivity.java index ad18b2c0..c11d8271 100644 --- a/Android/app/src/main/java/github/tianzerl/anime4kcpp/MainActivity.java +++ b/Android/app/src/main/java/github/tianzerl/anime4kcpp/MainActivity.java @@ -10,6 +10,7 @@ import android.view.Menu; import android.view.MenuItem; import android.view.View; +import android.view.WindowManager; import android.widget.ArrayAdapter; import android.widget.Button; import android.widget.CompoundButton; @@ -37,7 +38,7 @@ public class MainActivity extends AppCompatActivity { private enum Error { - Anime4KCPPError, FailedToCreateFolders + Anime4KCPPError, FailedToCreateFolders, FailedToDeleteTmpFile } private enum GPUState { @@ -54,9 +55,7 @@ private enum FileType { ArrayList newArrayList = new ArrayList<>(); for (FileItemBeanImpl file: arrayList) { - //disable video processing for now - /*if(file.isDir() || getFileType(file.getFilePath()) != FileType.Unknown)*/ - if(file.isDir() || getFileType(file.getFilePath()) == FileType.Image) + if(file.isDir() || getFileType(file.getFilePath()) != FileType.Unknown) newArrayList.add(file); } return newArrayList; @@ -109,9 +108,8 @@ protected void onCreate(Bundle savedInstanceState) { setSwitches(); - //disable video suffix - EditText editTextVideoSuffix = findViewById(R.id.editTextSuffixVideo); - editTextVideoSuffix.setEnabled(false); + //Keep screen on + this.getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); } @Override @@ -191,7 +189,6 @@ protected void onActivityResult(int requestCode, int resultCode, @Nullable Inten } } - private FileType getFileType(@NonNull String src) { String imageSuffix = ((EditText)findViewById(R.id.editTextSuffixImage)).getText().toString(); String VideoSuffix = ((EditText)findViewById(R.id.editTextSuffixVideo)).getText().toString(); @@ -266,6 +263,13 @@ private void errorHandler(@NonNull Error errorType,@Nullable Exception exp) { .setPositiveButton("OK",null) .show(); break; + case FailedToDeleteTmpFile: + new AlertDialog.Builder(MainActivity.this) + .setTitle("ERROR") + .setMessage("Failed to delete Temporary output video file, delete it manually.") + .setPositiveButton("OK",null) + .show(); + break; } } @@ -444,9 +448,11 @@ else if(fileType == FileType.Video) anime4KCPP.setVideoMode(true); for (Pair video: videos) { File srcFile = new File(video.first); + String tmpOutputPath = dst+"/" + "tmpOutput" + videoCount +".mp4"; + String OutputPath = dst+"/" + prefix+srcFile.getName() + ".mp4"; anime4KCPP.loadVideo(srcFile.getPath()); - anime4KCPP.setVideoSaveInfo(dst+"/"+prefix+srcFile.getName()); + anime4KCPP.setVideoSaveInfo(tmpOutputPath); long start = System.currentTimeMillis(); anime4KCPP.process(); @@ -454,6 +460,14 @@ else if(fileType == FileType.Video) anime4KCPP.saveVideo(); + new VideoAudioProcessor(srcFile.getPath(), tmpOutputPath, OutputPath).merge(); + + if (!(new File(tmpOutputPath).delete())) + { + Message message = new Message(); + message.obj = Error.FailedToDeleteTmpFile; + otherErrorHandler.sendMessage(message); + } totalTime += end - start; publishProgress((++videoCount + imageCount) * 100 / taskCount, video.second); } diff --git a/Android/app/src/main/java/github/tianzerl/anime4kcpp/VideoAudioProcessor.java b/Android/app/src/main/java/github/tianzerl/anime4kcpp/VideoAudioProcessor.java new file mode 100644 index 00000000..447bdc35 --- /dev/null +++ b/Android/app/src/main/java/github/tianzerl/anime4kcpp/VideoAudioProcessor.java @@ -0,0 +1,112 @@ +package github.tianzerl.anime4kcpp; + +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.media.MediaMuxer; +import android.util.Log; + +import androidx.annotation.NonNull; + +import java.io.IOException; +import java.nio.ByteBuffer; + +class VideoAudioProcessor { + private String src; + private String tmp; + private String dst; + + VideoAudioProcessor(@NonNull String srcPath, @NonNull String tmpPath, @NonNull String dstPath) { + src = srcPath; + tmp = tmpPath; + dst = dstPath; + } + + void merge() throws IOException { + MediaExtractor srcVideoExtractor = new MediaExtractor(); + MediaExtractor tmpVideoExtractor = new MediaExtractor(); + MediaMuxer dstVideoMuxer = new MediaMuxer(dst,MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + srcVideoExtractor.setDataSource(src); + tmpVideoExtractor.setDataSource(tmp); + int srcTrackCount = srcVideoExtractor.getTrackCount(); + int srcAudioTrackIndex = -1; + int dstAudioTrackIndex = -1; + int dstVideoTrackIndex; + int dstAudioMaxInputSize = -1; + int tmpVideoMaxInputSize; + //Find src audio track; + for (int i = 0; i < srcTrackCount; i++) + { + MediaFormat mimeFormat = srcVideoExtractor.getTrackFormat(i); + String mimeType = mimeFormat.getString(MediaFormat.KEY_MIME); + assert mimeType != null; + Log.d("MediaType",mimeType); + if(mimeType.startsWith(MediaFormat.MIMETYPE_AUDIO_AAC)) + { + srcAudioTrackIndex = i; + dstAudioTrackIndex = dstVideoMuxer.addTrack(mimeFormat); + dstAudioMaxInputSize = mimeFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE); + } + } + + if(srcAudioTrackIndex == -1) + { + throw new IOException("Only supports to merge aac audio track"); + } + + //tmp video track; + { + MediaFormat mimeFormat = tmpVideoExtractor.getTrackFormat(0); + dstVideoTrackIndex = dstVideoMuxer.addTrack(mimeFormat); + tmpVideoMaxInputSize = mimeFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE); + } + + dstVideoMuxer.start(); + + srcVideoExtractor.selectTrack(srcAudioTrackIndex); + MediaCodec.BufferInfo srcAudioBufferInfo = new MediaCodec.BufferInfo(); + ByteBuffer srcAudioBuffer = ByteBuffer.allocate(dstAudioMaxInputSize); + while (true) + { + int readSampleSize = srcVideoExtractor.readSampleData(srcAudioBuffer,0); + if (readSampleSize < 0) + { + srcVideoExtractor.unselectTrack(srcAudioTrackIndex); + break; + } + + srcAudioBufferInfo.size = readSampleSize; + srcAudioBufferInfo.offset = 0; + srcAudioBufferInfo.flags = srcVideoExtractor.getSampleFlags(); + srcAudioBufferInfo.presentationTimeUs = srcVideoExtractor.getSampleTime(); + + dstVideoMuxer.writeSampleData(dstAudioTrackIndex,srcAudioBuffer,srcAudioBufferInfo); + + srcVideoExtractor.advance(); + } + + tmpVideoExtractor.selectTrack(0); + MediaCodec.BufferInfo tmpVideoBufferInfo = new MediaCodec.BufferInfo(); + ByteBuffer tmpVideoBuffer = ByteBuffer.allocate(tmpVideoMaxInputSize); + while (true) + { + int readSampleSize = tmpVideoExtractor.readSampleData(tmpVideoBuffer,0); + if (readSampleSize < 0) + { + tmpVideoExtractor.unselectTrack(0); + break; + } + + tmpVideoBufferInfo.size = readSampleSize; + tmpVideoBufferInfo.offset = 0; + tmpVideoBufferInfo.flags = tmpVideoExtractor.getSampleFlags(); + tmpVideoBufferInfo.presentationTimeUs = tmpVideoExtractor.getSampleTime(); + + dstVideoMuxer.writeSampleData(dstVideoTrackIndex, tmpVideoBuffer,tmpVideoBufferInfo); + tmpVideoExtractor.advance(); + } + + dstVideoMuxer.stop(); + dstVideoMuxer.release(); + } +} diff --git a/Android/app/src/main/res/values/ic_launcher_background.xml b/Android/app/src/main/res/values/ic_launcher_background.xml index f42ada65..c5d5899f 100644 --- a/Android/app/src/main/res/values/ic_launcher_background.xml +++ b/Android/app/src/main/res/values/ic_launcher_background.xml @@ -1,4 +1,4 @@ #FFFFFF - + \ No newline at end of file diff --git a/Android/app/src/main/res/values/strings.xml b/Android/app/src/main/res/values/strings.xml index 0bb8ca0a..75ad6529 100644 --- a/Android/app/src/main/res/values/strings.xml +++ b/Android/app/src/main/res/values/strings.xml @@ -6,8 +6,8 @@ CLEAR START Settings - Waitting - Processing + Waitting… + Processing… GPUMode FastMode Fast diff --git a/Anime4KCore/include/Anime4K.h b/Anime4KCore/include/Anime4K.h index aa04fe97..1ff6aeca 100644 --- a/Anime4KCore/include/Anime4K.h +++ b/Anime4KCore/include/Anime4K.h @@ -29,7 +29,7 @@ #endif #endif -#define ANIME4KCPP_CORE_VERSION "1.7.1" +#define ANIME4KCPP_CORE_VERSION "1.8.0" #define MAX3(a, b, c) std::max({a, b, c}) #define MIN3(a, b, c) std::min({a, b, c}) @@ -82,7 +82,7 @@ class DLL Anime4K void setVideoMode(const bool flag); void loadVideo(const std::string& srcFile); void loadImage(const std::string& srcFile); - void loadImage(const cv::InputArray srcImage); + void loadImage(cv::InputArray srcImage); void loadImage(int rows, int cols, unsigned char* data, size_t bytesPerLine = 0ULL); void loadImage(int rows, int cols, unsigned char* r, unsigned char* g, unsigned char* b); void setVideoSaveInfo(const std::string& dstFile,const CODEC codec = MP4V); diff --git a/Anime4KCore/src/Anime4K.cpp b/Anime4KCore/src/Anime4K.cpp index f32fb594..614bec71 100644 --- a/Anime4KCore/src/Anime4K.cpp +++ b/Anime4KCore/src/Anime4K.cpp @@ -96,7 +96,7 @@ void Anime4K::loadImage(const std::string& srcFile) W = zf * orgW; } -void Anime4K::loadImage(const cv::InputArray srcImage) +void Anime4K::loadImage(cv::InputArray srcImage) { dstImg = orgImg = srcImage.getMat(); if (orgImg.empty()||orgImg.type()!=CV_8UC3) diff --git a/cmake/ThirdPartyForCLI.cmake b/cmake/ThirdPartyForCLI.cmake index 38d4281c..a5256afa 100644 --- a/cmake/ThirdPartyForCLI.cmake +++ b/cmake/ThirdPartyForCLI.cmake @@ -2,4 +2,4 @@ find_package(OpenCL REQUIRED) include_directories(../ThirdParty/include ${OpenCL_INCLUDE_DIRS}) -target_link_libraries(${PROJECT_NAME} Anime4KCPPCore) \ No newline at end of file +target_link_libraries(${PROJECT_NAME} Anime4KCPPCore) diff --git a/cmake/ThirdPartyForGUI.cmake b/cmake/ThirdPartyForGUI.cmake index 98698b3e..a5428afa 100644 --- a/cmake/ThirdPartyForGUI.cmake +++ b/cmake/ThirdPartyForGUI.cmake @@ -6,4 +6,4 @@ include_directories(${OpenCV_INCLUDE_DIRS} ${OpenCL_INCLUDE_DIRS}) target_link_libraries(Anime4KCPP_GUI PRIVATE Qt5::Widgets) target_link_libraries(${PROJECT_NAME} PRIVATE ${OpenCV_LIBS}) -target_link_libraries(${PROJECT_NAME} PRIVATE Anime4KCPPCore) \ No newline at end of file +target_link_libraries(${PROJECT_NAME} PRIVATE Anime4KCPPCore) diff --git a/cmake/ThirdPartyForVS.cmake b/cmake/ThirdPartyForVS.cmake index db7c15c9..2abe6aee 100644 --- a/cmake/ThirdPartyForVS.cmake +++ b/cmake/ThirdPartyForVS.cmake @@ -4,4 +4,4 @@ target_link_directories(${PROJECT_NAME} PRIVATE ${VapourSynth_SDK_PATH}/lib64) target_link_libraries(${PROJECT_NAME} VapourSynth) -include(../cmake/ThirdPartyForCore.cmake) \ No newline at end of file +include(../cmake/ThirdPartyForCore.cmake)