修复音视频合并的问题

gadio/media/frame.py

@@ -1,11 +1,7 @@
-
 import os
 
-import cv2
+from PIL import Image, ImageDraw, ImageFont, ImageFilter
 import numpy as np
-from cv2 import VideoWriter_fourcc
-from PIL import Image, ImageDraw, ImageFont
-
 from gadio.configs.config import config
 from gadio.models.radio import Radio
 from gadio.text.wrapper import Wrapper
@@ -28,17 +24,15 @@ def __init__(self, *args, **kwargs):
     @staticmethod
     def create_cover(radio: Radio):
         """create a cover page for start of video. No text pasted on this page.
-
         Arguments:
             radio {Radio} -- Radio
-
         Returns:
             image -- a cv2 frame.
         """
         cover_dir = os.sep.join(
             ['cache', str(radio.radio_id), radio.cover.local_name])
         print('Creating cover page')
-        image = cv2.imread(cover_dir)
+        image = Image.open(cover_dir)
         image = Frame.expand_frame(image, Frame.width, Frame.height)
         return image
 
@@ -50,52 +44,49 @@ def create_page(page: Page, radio: Radio):
         2. Convert opencv image to Pillow image
         3. Draw text on Pillow image
         4. Convert back to opencv image for opencv VideoWriter
-
         Beware that Pillow image and opencv channel orders are different.
         Arguments:
             page {Page} -- Gadio page
-
         Keyword Arguments:
             radio {Radio} -- radio
-
         Returns:
             np.array -- An numpy array representing cv2 image.
         """
         image_suffix = page.image.suffix
-        if (image_suffix == "" or image_suffix.lower() == '.gif'):
-            # If image is not found or image is gif, load cover as background
+        if (image_suffix == ""):
             image_dir = os.sep.join(['cache', str(radio.radio_id), radio.cover.local_name])
+            image = Image.open(image_dir)
+        elif (image_suffix.lower() == '.gif'):
+            image_dir = os.sep.join(['cache', str(radio.radio_id), page.image.local_name])
+            gif_image = Image.open(image_dir)
+            gif_image.seek(0)
+            image = gif_image.convert('RGB')
         else:
             image_dir = os.sep.join(['cache', str(radio.radio_id), page.image.local_name])
+            image = Image.open(image_dir)
         qr_dir = os.sep.join(['cache', str(radio.radio_id), 'qr_quotes', page.image.local_name.split('.')[0] + ".png"])
 
-        image = cv2.imread(image_dir)
-        image_suffix = page.image.suffix
+
+        #image_suffix = page.image.suffix
         background_image = Frame.expand_frame(image, Frame.width, Frame.height)
-        background_image = cv2.GaussianBlur(background_image, (255, 255), 255)
-        content_image = Frame.shrink_frame(image, 550, 550)
+        background_image = background_image.filter(ImageFilter.GaussianBlur(radius=255))
+        content_image = Frame.shrink_frame(image, int(round(550/1920 * Frame.width)), int(round(550/1080 * Frame.height)))
 
         # Convert to PIL accepted RGB channel order
-        background_rgb = cv2.cvtColor(background_image, cv2.COLOR_BGR2RGB)
-        content_rgb = cv2.cvtColor(content_image, cv2.COLOR_BGR2RGB)
+        background_rgb = background_image.convert('RGBA')
+        content_rgb = content_image.convert('RGB')
 
         # Convert to RGBA for transparency rendering
-        frame = Image.fromarray(background_rgb).convert('RGBA')
+        frame = Image.alpha_composite(background_rgb, Image.new('RGBA', background_rgb.size, (0, 0, 0, 128)))
 
-        mask = Image.new('RGBA', (Frame.width, Frame.height), color=(0, 0, 0, 128))
-        frame.paste(mask, (0, 0), mask=mask)
+        left_offset = int(round(245/1920 * Frame.width)) + int(round((550 - content_image.size[0])/2))
+        top_offset = int(round(210/1080 * Frame.height)) + int(round((550 - content_image.size[1])/2))
 
-        left_offset = int(round(245/1920 * Frame.width)) + int(round((550 - content_image.shape[1])/2))
-        top_offset = int(round(210/1080 * Frame.height)) + int(round((550 - content_image.shape[0])/2))
+        content_frame = content_rgb
+        content_image_mask = Image.new('RGBA', content_image.size, color=(0, 0, 0, 26))
 
-        content_frame = Image.fromarray(content_rgb)
-        content_image_mask = Image.new('RGBA', (content_image.shape[1], content_image.shape[0]), color=(0, 0, 0, 26))
-        if (image_suffix == "" or image_suffix.lower() == '.gif'):
-            # if image is not properly downloaded or is gif, no content image should be added.
-            print("GIF will not be rendered in this page...")
-        else:
-            frame.paste(content_frame, (left_offset, top_offset))
-            frame.paste(content_image_mask, (left_offset, top_offset), mask = content_image_mask)
+        frame.paste(content_frame, (left_offset, top_offset))
+        frame.paste(content_image_mask, (left_offset, top_offset), mask = content_image_mask)
 
         try:
             logo_image = Image.open(config['gcores_logo_name']).convert('RGBA')
@@ -149,62 +140,56 @@ def create_page(page: Page, radio: Radio):
         Frame.content_font = ImageFont.truetype(config['content_font'], config['content_font_size'], encoding="utf-8")
         Frame.content_wrapper = Wrapper(Frame.content_font)
 
-        cv2charimg = np.array(frame)
-        result = cv2.cvtColor(cv2charimg, cv2.COLOR_RGB2BGR)
+        result = np.array(frame)
         # cv2.imwrite('test.jpg',result)
         # cv2.waitKey()
         return result
 
     @staticmethod
     def expand_frame(image, target_width, target_height):
         """Expand a frame so it is larger than the rectangle
-
         Arguments:
             image {Image} -- cv2 image
             target_width {int} -- target width of rectangle
             target_height {int} -- target width of rectangle
-
         Raises:
             NotImplementedError: [description]
-
         Returns:
             Image -- resized image
         """
         # shape[0]: height, shape[1]: width
-        width_ratio = image.shape[1] / target_width
-        height_ratio = image.shape[0] / target_height
+        width_ratio = image.size[0] / target_width
+        height_ratio = image.size[1] / target_height
         ratio = min(width_ratio, height_ratio)
         # in case width or height smaller than target after rounding.
-        actual_width = max(int(image.shape[1] / ratio), target_width)
-        actuai_height = max(int(image.shape[0] / ratio), target_height)
-        result = cv2.resize(image, (actual_width, actuai_height),
-                            interpolation=cv2.INTER_CUBIC)
-        left = int((result.shape[1] - target_width) / 2)
+        actual_width = max(int(image.size[0] / ratio), target_width)
+        actuai_height = max(int(image.size[1] / ratio), target_height)
+        result = image.resize((actual_width, actuai_height), Image.ANTIALIAS)
+        left = int((result.size[0] - target_width) / 2)
         right = left + target_width
-        top = int((result.shape[0] - target_height) / 2)
+        top = int((result.size[1] - target_height) / 2)
         bottom = top + target_height
-        return result[top:bottom, left:right]
+        return result.crop((left, top, right, bottom))
+
 
     @staticmethod
     def shrink_frame(image, target_width, target_height):
         """Shrink a frame so it is smaller than the rectangle
-
         Arguments:
             image {Image} -- np array
             target_width {int} -- target width of rectangle
             target_height {int} -- target height of rectangle
-
         Returns:
             np.array -- resized image
         """
-
+    
         # shape[0] : height, shape[1]: width
-        width_ratio = image.shape[1] / target_width
-        height_ratio = image.shape[0] / target_height
+        width_ratio = image.size[0] / target_width
+        height_ratio = image.size[1] / target_height
         ratio = max(width_ratio, height_ratio)
-        actual_width = min(int(image.shape[1] / ratio), target_width)
-        actual_height = min(int(image.shape[0] / ratio), target_height)
-        result = cv2.resize(image, (actual_width, actual_height), interpolation=cv2.INTER_CUBIC)
+        actual_width = min(int(image.size[0] / ratio), target_width)
+        actual_height = min(int(image.size[1] / ratio), target_height)
+        result = image.resize((actual_width, actual_height), Image.ANTIALIAS)
         return result
 
     @staticmethod

gadio/media/readme.md

@@ -0,0 +1,13 @@
+总体还是按[兔子主义大哥姐(?)的项目](https://github.com/rabbitism/GadioVideo)做的是PPT式视频，这次更新做了以下改变
+
+1. 移除了cv2及moviepy相关代码
+2. frame.py 增加了对 .gif 图片的支持（把第一帧截下来用）
+3. 因为改用了ffmpeg，对各类图片格式的支持应该也是没有问题的（没仔细测过）
+4. frame.py 尝试改进了分辨率自适应，简单尝试了一下，起码图片没乱飞了
+5. video.py 的生成也改成了全部使用ffmpeg, 步骤是
+   - 将生成帧（PPT）转化成对应长度的视频段
+   - 将所有视频段合并为视频
+   - 将视频与音频合并
+   - **如果有更简便的方法请帮帮忙**
+6. 生成帧与视频段姑且没有被设置为会被删除，他们被保存在了`./cache/电台ID/images/` 与 `./cache/电台ID/videos/` 下，如需程序自动删除请去掉 video.py 中仅有的3行的注释
+

gadio/media/video.py

@@ -1,5 +1,8 @@
-from cv2 import VideoWriter, VideoWriter_fourcc
-from moviepy.editor import *
+from PIL import Image
+import subprocess as sp
+import os
+# from shutil import rmtree
+import ffmpeg_downloader as ffdl
 
 from gadio.configs.config import config
 from gadio.media.frame import Frame
@@ -8,39 +11,62 @@
 
 class Video():
 
-    fourcc = VideoWriter_fourcc(*'mp4v')
     fps = config['fps']
     width = config['width']
     height = config['height']
-    output_dir = os.sep.join(['.', 'output'])
+    output_dir = os.path.join('.', 'output')
 
-    def __init__(self, *args, **kwargs):
-        return super().__init__(*args, **kwargs)
+    def __init__(self, config, *args, **kwargs):
+        return super().__init__(config, *args, **kwargs)
 
     @staticmethod
     def create_video(radio: Radio):
-        if not os.path.exists(Video.output_dir):
-            print("Folder", Video.output_dir, 'does not exist. Creating...')
-            os.makedirs(Video.output_dir)
-        video = VideoWriter(Video.output_dir + os.sep + str(radio.radio_id) + '_temp.mp4', Video.fourcc, Video.fps, (Video.width, Video.height))
+        images_loc = os.path.join(os.curdir, 'cache', str(radio.radio_id), 'images')
+        os.makedirs(images_loc, exist_ok=True)
+        vclips_loc = os.path.join(os.curdir, 'cache', str(radio.radio_id), 'videos')
+        os.makedirs(vclips_loc, exist_ok=True)
+        textlist = os.path.join(vclips_loc, 'list.txt')
+        if os.path.exists(textlist):
+            os.remove(textlist)
+
+        os.makedirs(Video.output_dir, exist_ok=True)
         clip_count = len(radio.timestamps) - 1
+
         for i in range(clip_count):
             if (radio.timestamps[i] not in radio.timeline.keys()):
                 print(radio.timestamps[i], "has no corresponding image, load cover as backup")
                 frame = Frame.create_cover(radio)
             else:
                 frame = Frame.create_page(radio.timeline[radio.timestamps[i]], radio)
-            frame_count = (radio.timestamps[i + 1] - radio.timestamps[i]) * Video.fps
-            for j in range(frame_count):
-                video.write(frame)
-        video.release()
+
+            sequence = '%05d' % i
+            frame_time = str(radio.timestamps[i + 1] - radio.timestamps[i])
+
+            Image.fromarray(frame).save(os.path.join(images_loc, sequence + '.png'))
+            sp.run([ffdl.ffmpeg_path, 
+                    '-r', str(Video.fps), 
+                    '-loop', '1', 
+                    '-i', os.path.join(images_loc, sequence + '.png'), 
+                    '-c:v', 'libx264', 
+                    '-pix_fmt', 'yuv420p', 
+                    '-crf', '24', 
+                    '-t', frame_time, 
+                    os.path.join(vclips_loc, sequence + '.mp4')])
+
+            with open(textlist, 'a+') as f:
+                f.write("file '{}'\n".format(sequence + '.mp4'))
+        f.close()
 
-        video_clip = VideoFileClip(Video.output_dir + os.sep + str(radio.radio_id) + '_temp.mp4')
-        print(video_clip.duration)
-        audio_clip = AudioFileClip(os.sep.join(['.', 'cache', str(radio.radio_id), 'audio', radio.audio.local_name]))
-        video_clip.audio = audio_clip
-        if config['test']:
-            video_clip = video_clip.subclip(0, min(200, video_clip.duration))
-        video_clip.write_videofile(Video.output_dir +os.sep+ str(radio.radio_id)+" "+radio.title +".mp4", fps=Video.fps)
+        audio_clip = os.path.join('.', 'cache', str(radio.radio_id), 'audio', radio.audio.local_name)
+        sp.run([ffdl.ffmpeg_path, 
+                '-f', 'concat', 
+                '-safe', '0', 
+                '-i', textlist, 
+                '-i', audio_clip, 
+                '-c:v', 'copy', 
+                '-c:a', 'aac', 
+                os.path.join(Video.output_dir, radio.title + '.mp4')])
+
         print("{} finished!".format(radio.title))
-        # os.remove(Video.output_dir+os.sep+str(radio.radio_id)+'_temp.mp4')
+        # rmtree(os.path.join(os.curdir, 'cache', str(radio.radio_id), 'images'))
+        # rmtree(os.path.join(os.curdir, 'cache', str(radio.radio_id), 'videos'))

readme.md

@@ -11,15 +11,17 @@
 
 ## 运行环境
 
-python 3.5, 3.6, 3.7
+python 3.5, 3.6, 3.7, 3.8, 3.9
 
 ## 使用说明
 
 ### 安装依赖
 
 ```bash
 pip3 install -r requirements.txt
+ffdl install --add-path
 ```
+ffdl(ffmpeg-downloader) 会自动安装最新的ffmpeg, Linux仅需输入 `ffdl install` , 它会自动添加链接, Windows和macOS则需额外输入 `--add-path` 来添加环境, 或请自行下载ffmpeg和添加环境
 
 ### 运行示例
 

requirements.txt

@@ -1,6 +1,7 @@
-opencv-python==4.5.5.62
-Pillow==9.0.0
-urllib3==1.26.5
-moviepy==1.0.0
-requests==2.27.1
-myqr==2.3.1
+opencv-python==4.5.5.62
+Pillow==9.0.0
+urllib3==1.26.5
+moviepy==1.0.0
+requests==2.27.1
+myqr==2.3.1
+ffmpeg-downloader==0.2.0