From 9ee23d5f8b820e1821619cc678b3fccc89d37e76 Mon Sep 17 00:00:00 2001 From: zhuang2002 <108931120+zhuang2002@users.noreply.github.com> Date: Sat, 9 Dec 2023 13:31:37 +0800 Subject: [PATCH] [Enhance] add outpainting to gradio_PowerPaint.py (#2084) * Update gradio_PowerPaint.py * Update gradio_PowerPaint.py * Update gradio_PowerPaint.py * Update gradio_PowerPaint.py * Update README.md * Update README.md * Update gradio_PowerPaint.py --- projects/powerpaint/README.md | 4 +- projects/powerpaint/gradio_PowerPaint.py | 141 ++++++++++++++++++++--- 2 files changed, 124 insertions(+), 21 deletions(-) diff --git a/projects/powerpaint/README.md b/projects/powerpaint/README.md index 7fd6d2cd0..26cca91a0 100644 --- a/projects/powerpaint/README.md +++ b/projects/powerpaint/README.md @@ -1,10 +1,10 @@ # A Task is Worth One Word: Learning with Task Prompts for High-Quality Versatile Image Inpainting -### [Project Page](https://powerpaint.github.io/) | [Paper](https://arxiv.org/abs/2312.03594) | [OpenXLab](https://openxlab.org.cn/apps/detail/rangoliu/PowerPaint#basic-information) +### [Project Page](https://powerpaint.github.io/) | [Paper](https://arxiv.org/abs/2312.03594) | [Online Demo(OpenXlab)](https://openxlab.org.cn/apps/detail/rangoliu/PowerPaint#basic-information) This README provides a step-by-step guide to download the repository, set up the required virtual environment named "PowerPaint" using conda, and run PowerPaint with or without ControlNet. -**Stronger Model Weights and Online Demo Coming Soon！** +**Stronger Model Weights Coming Soon！** diff --git a/projects/powerpaint/gradio_PowerPaint.py b/projects/powerpaint/gradio_PowerPaint.py index 373245687..ef2cd677a 100644 --- a/projects/powerpaint/gradio_PowerPaint.py +++ b/projects/powerpaint/gradio_PowerPaint.py @@ -82,6 +82,7 @@ def get_depth_map(image): def add_task(prompt, negative_prompt, control_type): + # print(control_type) if control_type == 'object-removal': promptA = prompt + ' P_ctxt' promptB = prompt + ' P_ctxt' @@ -92,6 +93,11 @@ def add_task(prompt, negative_prompt, control_type): promptB = prompt + ' P_ctxt' negative_promptA = negative_prompt + ' P_shape' negative_promptB = negative_prompt + ' P_ctxt' + elif control_type == 'image-outpainting': + promptA = prompt + ' P_ctxt' + promptB = prompt + ' P_ctxt' + negative_promptA = negative_prompt + ' P_obj' + negative_promptB = negative_prompt + ' P_obj' else: promptA = prompt + ' P_obj' promptB = prompt + ' P_obj' @@ -102,16 +108,63 @@ def add_task(prompt, negative_prompt, control_type): def predict(input_image, prompt, fitting_degree, ddim_steps, scale, seed, - negative_prompt, task): - promptA, promptB, negative_promptA, negative_promptB = add_task( - prompt, negative_prompt, task) + negative_prompt, task, vertical_expansion_ratio, + horizontal_expansion_ratio): size1, size2 = input_image['image'].convert('RGB').size - if size1 < size2: - input_image['image'] = input_image['image'].convert('RGB').resize( - (640, int(size2 / size1 * 640))) + + if task != 'image-outpainting': + if size1 < size2: + input_image['image'] = input_image['image'].convert('RGB').resize( + (640, int(size2 / size1 * 640))) + else: + input_image['image'] = input_image['image'].convert('RGB').resize( + (int(size1 / size2 * 640), 640)) else: - input_image['image'] = input_image['image'].convert('RGB').resize( - (int(size1 / size2 * 640), 640)) + if size1 < size2: + input_image['image'] = input_image['image'].convert('RGB').resize( + (512, int(size2 / size1 * 512))) + else: + input_image['image'] = input_image['image'].convert('RGB').resize( + (int(size1 / size2 * 512), 512)) + + if (vertical_expansion_ratio is not None) and (horizontal_expansion_ratio + is not None): # noqa + o_W, o_H = input_image['image'].convert('RGB').size + c_W = int(horizontal_expansion_ratio * o_W) + c_H = int(vertical_expansion_ratio * o_H) + + expand_img = np.ones((c_H, c_W, 3), dtype=np.uint8) * 127 + original_img = np.array(input_image['image']) + expand_img[int((c_H - o_H) / 2.0):int((c_H - o_H) / 2.0) + o_H, + int((c_W - o_W) / 2.0):int((c_W - o_W) / 2.0) + + o_W, :] = original_img + + blurry_gap = 10 + + expand_mask = np.ones((c_H, c_W, 3), dtype=np.uint8) * 255 + if vertical_expansion_ratio == 1 and horizontal_expansion_ratio != 1: + expand_mask[int((c_H - o_H) / 2.0):int((c_H - o_H) / 2.0) + o_H, + int((c_W - o_W) / 2.0) + + blurry_gap:int((c_W - o_W) / 2.0) + o_W - + blurry_gap, :] = 0 + elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio != 1: + expand_mask[int((c_H - o_H) / 2.0) + + blurry_gap:int((c_H - o_H) / 2.0) + o_H - blurry_gap, + int((c_W - o_W) / 2.0) + + blurry_gap:int((c_W - o_W) / 2.0) + o_W - + blurry_gap, :] = 0 + elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio == 1: + expand_mask[int((c_H - o_H) / 2.0) + + blurry_gap:int((c_H - o_H) / 2.0) + o_H - blurry_gap, + int((c_W - o_W) / + 2.0):int((c_W - o_W) / 2.0) + o_W, :] = 0 + + input_image['image'] = Image.fromarray(expand_img) + input_image['mask'] = Image.fromarray(expand_mask) + + promptA, promptB, negative_promptA, negative_promptB = add_task( + prompt, negative_prompt, task) + # print(promptA, promptB, negative_promptA, negative_promptB) img = np.array(input_image['image'].convert('RGB')) W = int(np.shape(img)[0] - np.shape(img)[0] % 8) @@ -143,7 +196,7 @@ def predict(input_image, prompt, fitting_degree, ddim_steps, scale, seed, (result_m.astype('float') * (1 - mask_np.astype('float') / 512.0) + mask_np.astype('float') / 512.0 * red).astype('uint8')) m_img = input_image['mask'].convert('RGB').filter( - ImageFilter.GaussianBlur(radius=4)) + ImageFilter.GaussianBlur(radius=3)) m_img = np.asarray(m_img) / 255.0 img_np = np.asarray(input_image['image'].convert('RGB')) / 255.0 ours_np = np.asarray(result) / 255.0 @@ -161,13 +214,19 @@ def predict_controlnet(input_image, input_control_image, control_type, prompt, promptB = prompt + ' P_obj' negative_promptA = negative_prompt + ' P_obj' negative_promptB = negative_prompt + ' P_obj' + size1, size2 = input_image['image'].convert('RGB').size + + if size1 < size2: + input_image['image'] = input_image['image'].convert('RGB').resize( + (640, int(size2 / size1 * 640))) + else: + input_image['image'] = input_image['image'].convert('RGB').resize( + (int(size1 / size2 * 640), 640)) img = np.array(input_image['image'].convert('RGB')) W = int(np.shape(img)[0] - np.shape(img)[0] % 8) H = int(np.shape(img)[1] - np.shape(img)[1] % 8) input_image['image'] = input_image['image'].resize((H, W)) input_image['mask'] = input_image['mask'].resize((H, W)) - print(np.shape(np.array(input_image['mask'].convert('RGB')))) - print(np.shape(np.array(input_image['image'].convert('RGB')))) global current_control global pipe @@ -251,7 +310,8 @@ def predict_controlnet(input_image, input_control_image, control_type, prompt, def infer(input_image, text_guided_prompt, text_guided_negative_prompt, shape_guided_prompt, shape_guided_negative_prompt, fitting_degree, ddim_steps, scale, seed, task, enable_control, input_control_image, - control_type): + control_type, vertical_expansion_ratio, horizontal_expansion_ratio, + outpaint_prompt, outpaint_negative_prompt): if task == 'text-guided': prompt = text_guided_prompt negative_prompt = text_guided_negative_prompt @@ -261,19 +321,24 @@ def infer(input_image, text_guided_prompt, text_guided_negative_prompt, elif task == 'object-removal': prompt = '' negative_prompt = '' - scale = 10.0 + elif task == 'image-outpainting': + prompt = outpaint_prompt + negative_prompt = outpaint_negative_prompt + return predict(input_image, prompt, fitting_degree, ddim_steps, scale, + seed, negative_prompt, task, vertical_expansion_ratio, + horizontal_expansion_ratio) else: task = 'text-guided' prompt = text_guided_prompt negative_prompt = text_guided_negative_prompt - if enable_control: + if enable_control and task == 'text-guided': return predict_controlnet(input_image, input_control_image, control_type, prompt, ddim_steps, scale, seed, negative_prompt) else: return predict(input_image, prompt, fitting_degree, ddim_steps, scale, - seed, negative_prompt, task) + seed, negative_prompt, task, None, None) def select_tab_text_guided(): @@ -284,6 +349,10 @@ def select_tab_object_removal(): return 'object-removal' +def select_tab_image_outpainting(): + return 'image-outpainting' + + def select_tab_shape_guided(): return 'shape-guided' @@ -305,7 +374,10 @@ def select_tab_shape_guided(): gr.Markdown('### Input image and draw mask') input_image = gr.Image(source='upload', tool='sketch', type='pil') - task = gr.Radio(['text-guided', 'object-removal', 'shape-guided'], + task = gr.Radio([ + 'text-guided', 'object-removal', 'shape-guided', + 'image-outpainting' + ], show_label=False, visible=False) @@ -337,6 +409,32 @@ def select_tab_shape_guided(): tab_object_removal.select( fn=select_tab_object_removal, inputs=None, outputs=task) + # Object image outpainting + with gr.Tab('Image outpainting') as tab_image_outpainting: + enable_object_removal = gr.Checkbox( + label='Enable image outpainting', + value=True, + interactive=False) + outpaint_prompt = gr.Textbox(label='Outpainting_prompt') + outpaint_negative_prompt = gr.Textbox( + label='Outpainting_negative_prompt') + horizontal_expansion_ratio = gr.Slider( + label='horizontal expansion ratio', + minimum=1, + maximum=4, + step=0.05, + value=1, + ) + vertical_expansion_ratio = gr.Slider( + label='vertical expansion ratio', + minimum=1, + maximum=4, + step=0.05, + value=1, + ) + tab_image_outpainting.select( + fn=select_tab_image_outpainting, inputs=None, outputs=task) + # Shape-guided object inpainting with gr.Tab('Shape-guided object inpainting') as tab_shape_guided: enable_shape_guided = gr.Checkbox( @@ -351,7 +449,7 @@ def select_tab_shape_guided(): minimum=0, maximum=1, step=0.05, - randomize=True, + value=1, ) tab_shape_guided.select( fn=select_tab_shape_guided, inputs=None, outputs=task) @@ -362,6 +460,10 @@ def select_tab_shape_guided(): label='Steps', minimum=1, maximum=50, value=45, step=1) scale = gr.Slider( label='Guidance Scale', + info='For object removal, \ + it is recommended to set the value at 10 or above, \ + while for image outpainting, \ + it is advisable to set it at 18 or above.', minimum=0.1, maximum=30.0, value=7.5, @@ -386,9 +488,10 @@ def select_tab_shape_guided(): input_image, text_guided_prompt, text_guided_negative_prompt, shape_guided_prompt, shape_guided_negative_prompt, fitting_degree, ddim_steps, scale, seed, task, enable_control, input_control_image, - control_type + control_type, vertical_expansion_ratio, horizontal_expansion_ratio, + outpaint_prompt, outpaint_negative_prompt ], outputs=[inpaint_result, gallery]) demo.queue() -demo.launch(share=False, server_name='0.0.0.0', server_port=7890) +demo.launch(share=False, server_name='0.0.0.0', server_port=7860)