From c2b1ede25bc801f622f442b9f64882e57d876aeb Mon Sep 17 00:00:00 2001 From: zhuang2002 <108931120+zhuang2002@users.noreply.github.com> Date: Sat, 9 Dec 2023 12:30:35 +0800 Subject: [PATCH 1/7] Update gradio_PowerPaint.py --- projects/powerpaint/gradio_PowerPaint.py | 120 +++++++++++++++++++---- 1 file changed, 101 insertions(+), 19 deletions(-) diff --git a/projects/powerpaint/gradio_PowerPaint.py b/projects/powerpaint/gradio_PowerPaint.py index 373245687..ebdbfeb7c 100644 --- a/projects/powerpaint/gradio_PowerPaint.py +++ b/projects/powerpaint/gradio_PowerPaint.py @@ -82,6 +82,7 @@ def get_depth_map(image): def add_task(prompt, negative_prompt, control_type): + # print(control_type) if control_type == 'object-removal': promptA = prompt + ' P_ctxt' promptB = prompt + ' P_ctxt' @@ -92,6 +93,11 @@ def add_task(prompt, negative_prompt, control_type): promptB = prompt + ' P_ctxt' negative_promptA = negative_prompt + ' P_shape' negative_promptB = negative_prompt + ' P_ctxt' + elif control_type == 'image-outpainting': + promptA = prompt + ' P_ctxt' + promptB = prompt + ' P_ctxt' + negative_promptA = negative_prompt + ' P_obj' + negative_promptB = negative_prompt + ' P_obj' else: promptA = prompt + ' P_obj' promptB = prompt + ' P_obj' @@ -102,16 +108,52 @@ def add_task(prompt, negative_prompt, control_type): def predict(input_image, prompt, fitting_degree, ddim_steps, scale, seed, - negative_prompt, task): - promptA, promptB, negative_promptA, negative_promptB = add_task( - prompt, negative_prompt, task) + negative_prompt, task,vertical_expansion_ratio,horizontal_expansion_ratio): size1, size2 = input_image['image'].convert('RGB').size - if size1 < size2: - input_image['image'] = input_image['image'].convert('RGB').resize( - (640, int(size2 / size1 * 640))) + + if task!='image-outpainting': + if size1 < size2: + input_image['image'] = input_image['image'].convert('RGB').resize( + (640, int(size2 / size1 * 640))) + else: + input_image['image'] = input_image['image'].convert('RGB').resize( + (int(size1 / size2 * 640), 640)) else: - input_image['image'] = input_image['image'].convert('RGB').resize( - (int(size1 / size2 * 640), 640)) + if size1 < size2: + input_image['image'] = input_image['image'].convert('RGB').resize( + (512, int(size2 / size1 * 512))) + else: + input_image['image'] = input_image['image'].convert('RGB').resize( + (int(size1 / size2 * 512), 512)) + + if vertical_expansion_ratio!=None and horizontal_expansion_ratio!=None: + o_W,o_H = input_image['image'].convert('RGB').size + c_W = int(horizontal_expansion_ratio*o_W) + c_H = int(vertical_expansion_ratio*o_H) + + expand_img = np.ones((c_H, c_W,3), dtype=np.uint8)*127 + original_img = np.array(input_image['image']) + expand_img[int((c_H-o_H)/2.0):int((c_H-o_H)/2.0)+o_H,int((c_W-o_W)/2.0):int((c_W-o_W)/2.0)+o_W,:] = original_img + + blurry_gap = 10 + + expand_mask = np.ones((c_H, c_W,3), dtype=np.uint8)*255 + if vertical_expansion_ratio == 1 and horizontal_expansion_ratio!=1: + expand_mask[int((c_H-o_H)/2.0):int((c_H-o_H)/2.0)+o_H,int((c_W-o_W)/2.0)+blurry_gap:int((c_W-o_W)/2.0)+o_W-blurry_gap,:] = 0 + elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio!=1: + expand_mask[int((c_H-o_H)/2.0)+blurry_gap:int((c_H-o_H)/2.0)+o_H-blurry_gap,int((c_W-o_W)/2.0)+blurry_gap:int((c_W-o_W)/2.0)+o_W-blurry_gap,:] = 0 + elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio==1: + expand_mask[int((c_H-o_H)/2.0)+blurry_gap:int((c_H-o_H)/2.0)+o_H-blurry_gap,int((c_W-o_W)/2.0):int((c_W-o_W)/2.0)+o_W,:] = 0 + + input_image['image'] = Image.fromarray(expand_img) + input_image['mask'] = Image.fromarray(expand_mask) + + + + + promptA, promptB, negative_promptA, negative_promptB = add_task( + prompt, negative_prompt, task) + # print(promptA, promptB, negative_promptA, negative_promptB) img = np.array(input_image['image'].convert('RGB')) W = int(np.shape(img)[0] - np.shape(img)[0] % 8) @@ -143,7 +185,7 @@ def predict(input_image, prompt, fitting_degree, ddim_steps, scale, seed, (result_m.astype('float') * (1 - mask_np.astype('float') / 512.0) + mask_np.astype('float') / 512.0 * red).astype('uint8')) m_img = input_image['mask'].convert('RGB').filter( - ImageFilter.GaussianBlur(radius=4)) + ImageFilter.GaussianBlur(radius=3)) m_img = np.asarray(m_img) / 255.0 img_np = np.asarray(input_image['image'].convert('RGB')) / 255.0 ours_np = np.asarray(result) / 255.0 @@ -161,13 +203,19 @@ def predict_controlnet(input_image, input_control_image, control_type, prompt, promptB = prompt + ' P_obj' negative_promptA = negative_prompt + ' P_obj' negative_promptB = negative_prompt + ' P_obj' + size1, size2 = input_image['image'].convert('RGB').size + + if size1 < size2: + input_image['image'] = input_image['image'].convert('RGB').resize( + (640, int(size2 / size1 * 640))) + else: + input_image['image'] = input_image['image'].convert('RGB').resize( + (int(size1 / size2 * 640), 640)) img = np.array(input_image['image'].convert('RGB')) W = int(np.shape(img)[0] - np.shape(img)[0] % 8) H = int(np.shape(img)[1] - np.shape(img)[1] % 8) input_image['image'] = input_image['image'].resize((H, W)) input_image['mask'] = input_image['mask'].resize((H, W)) - print(np.shape(np.array(input_image['mask'].convert('RGB')))) - print(np.shape(np.array(input_image['image'].convert('RGB')))) global current_control global pipe @@ -251,7 +299,7 @@ def predict_controlnet(input_image, input_control_image, control_type, prompt, def infer(input_image, text_guided_prompt, text_guided_negative_prompt, shape_guided_prompt, shape_guided_negative_prompt, fitting_degree, ddim_steps, scale, seed, task, enable_control, input_control_image, - control_type): + control_type,vertical_expansion_ratio,horizontal_expansion_ratio,outpaint_prompt,outpaint_negative_prompt): if task == 'text-guided': prompt = text_guided_prompt negative_prompt = text_guided_negative_prompt @@ -261,19 +309,23 @@ def infer(input_image, text_guided_prompt, text_guided_negative_prompt, elif task == 'object-removal': prompt = '' negative_prompt = '' - scale = 10.0 + elif task == 'image-outpainting': + prompt = outpaint_prompt + negative_prompt = outpaint_negative_prompt + return predict(input_image, prompt, fitting_degree, ddim_steps, scale, + seed, negative_prompt, task,vertical_expansion_ratio,horizontal_expansion_ratio) else: task = 'text-guided' prompt = text_guided_prompt negative_prompt = text_guided_negative_prompt - if enable_control: + if enable_control and task == 'text-guided': return predict_controlnet(input_image, input_control_image, control_type, prompt, ddim_steps, scale, seed, negative_prompt) else: return predict(input_image, prompt, fitting_degree, ddim_steps, scale, - seed, negative_prompt, task) + seed, negative_prompt, task,None,None) def select_tab_text_guided(): @@ -283,6 +335,9 @@ def select_tab_text_guided(): def select_tab_object_removal(): return 'object-removal' +def select_tab_image_outpainting(): + return 'image-outpainting' + def select_tab_shape_guided(): return 'shape-guided' @@ -305,7 +360,7 @@ def select_tab_shape_guided(): gr.Markdown('### Input image and draw mask') input_image = gr.Image(source='upload', tool='sketch', type='pil') - task = gr.Radio(['text-guided', 'object-removal', 'shape-guided'], + task = gr.Radio(['text-guided', 'object-removal', 'shape-guided', 'image-outpainting'], show_label=False, visible=False) @@ -336,6 +391,32 @@ def select_tab_shape_guided(): interactive=False) tab_object_removal.select( fn=select_tab_object_removal, inputs=None, outputs=task) + + # Object image outpainting + with gr.Tab('Image outpainting') as tab_image_outpainting: + enable_object_removal = gr.Checkbox( + label='Enable image outpainting', + value=True, + interactive=False) + outpaint_prompt = gr.Textbox(label='Outpainting_prompt') + outpaint_negative_prompt = gr.Textbox( + label='Outpainting_negative_prompt') + horizontal_expansion_ratio = gr.Slider( + label='horizontal expansion ratio', + minimum=1, + maximum=4, + step=0.05, + value=1, + ) + vertical_expansion_ratio = gr.Slider( + label='vertical expansion ratio', + minimum=1, + maximum=4, + step=0.05, + value=1, + ) + tab_image_outpainting.select( + fn=select_tab_image_outpainting, inputs=None, outputs=task) # Shape-guided object inpainting with gr.Tab('Shape-guided object inpainting') as tab_shape_guided: @@ -351,7 +432,7 @@ def select_tab_shape_guided(): minimum=0, maximum=1, step=0.05, - randomize=True, + value=1, ) tab_shape_guided.select( fn=select_tab_shape_guided, inputs=None, outputs=task) @@ -362,6 +443,7 @@ def select_tab_shape_guided(): label='Steps', minimum=1, maximum=50, value=45, step=1) scale = gr.Slider( label='Guidance Scale', + info='For object removal, it is recommended to set the value at 10 or above, while for image outpainting, it is advisable to set it at 18 or above.', minimum=0.1, maximum=30.0, value=7.5, @@ -386,9 +468,9 @@ def select_tab_shape_guided(): input_image, text_guided_prompt, text_guided_negative_prompt, shape_guided_prompt, shape_guided_negative_prompt, fitting_degree, ddim_steps, scale, seed, task, enable_control, input_control_image, - control_type + control_type,vertical_expansion_ratio,horizontal_expansion_ratio,outpaint_prompt,outpaint_negative_prompt ], outputs=[inpaint_result, gallery]) demo.queue() -demo.launch(share=False, server_name='0.0.0.0', server_port=7890) +demo.launch(share=False, server_name='0.0.0.0', server_port=7860) From 3cbf6eed016959ffd67212875899e9787da7dd9f Mon Sep 17 00:00:00 2001 From: zhuang2002 <108931120+zhuang2002@users.noreply.github.com> Date: Sat, 9 Dec 2023 12:52:35 +0800 Subject: [PATCH 2/7] Update gradio_PowerPaint.py --- projects/powerpaint/gradio_PowerPaint.py | 72 +++++++++++++++--------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/projects/powerpaint/gradio_PowerPaint.py b/projects/powerpaint/gradio_PowerPaint.py index ebdbfeb7c..ff6ec9bb4 100644 --- a/projects/powerpaint/gradio_PowerPaint.py +++ b/projects/powerpaint/gradio_PowerPaint.py @@ -108,10 +108,11 @@ def add_task(prompt, negative_prompt, control_type): def predict(input_image, prompt, fitting_degree, ddim_steps, scale, seed, - negative_prompt, task,vertical_expansion_ratio,horizontal_expansion_ratio): + negative_prompt, task, vertical_expansion_ratio, + horizontal_expansion_ratio): size1, size2 = input_image['image'].convert('RGB').size - if task!='image-outpainting': + if task != 'image-outpainting': if size1 < size2: input_image['image'] = input_image['image'].convert('RGB').resize( (640, int(size2 / size1 * 640))) @@ -125,32 +126,41 @@ def predict(input_image, prompt, fitting_degree, ddim_steps, scale, seed, else: input_image['image'] = input_image['image'].convert('RGB').resize( (int(size1 / size2 * 512), 512)) - - if vertical_expansion_ratio!=None and horizontal_expansion_ratio!=None: - o_W,o_H = input_image['image'].convert('RGB').size - c_W = int(horizontal_expansion_ratio*o_W) - c_H = int(vertical_expansion_ratio*o_H) - expand_img = np.ones((c_H, c_W,3), dtype=np.uint8)*127 + if vertical_expansion_ratio != None and horizontal_expansion_ratio != None: + o_W, o_H = input_image['image'].convert('RGB').size + c_W = int(horizontal_expansion_ratio * o_W) + c_H = int(vertical_expansion_ratio * o_H) + + expand_img = np.ones((c_H, c_W, 3), dtype=np.uint8) * 127 original_img = np.array(input_image['image']) - expand_img[int((c_H-o_H)/2.0):int((c_H-o_H)/2.0)+o_H,int((c_W-o_W)/2.0):int((c_W-o_W)/2.0)+o_W,:] = original_img + expand_img[int((c_H - o_H) / 2.0):int((c_H - o_H) / 2.0) + o_H, + int((c_W - o_W) / 2.0):int((c_W - o_W) / 2.0) + + o_W, :] = original_img blurry_gap = 10 - expand_mask = np.ones((c_H, c_W,3), dtype=np.uint8)*255 - if vertical_expansion_ratio == 1 and horizontal_expansion_ratio!=1: - expand_mask[int((c_H-o_H)/2.0):int((c_H-o_H)/2.0)+o_H,int((c_W-o_W)/2.0)+blurry_gap:int((c_W-o_W)/2.0)+o_W-blurry_gap,:] = 0 - elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio!=1: - expand_mask[int((c_H-o_H)/2.0)+blurry_gap:int((c_H-o_H)/2.0)+o_H-blurry_gap,int((c_W-o_W)/2.0)+blurry_gap:int((c_W-o_W)/2.0)+o_W-blurry_gap,:] = 0 - elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio==1: - expand_mask[int((c_H-o_H)/2.0)+blurry_gap:int((c_H-o_H)/2.0)+o_H-blurry_gap,int((c_W-o_W)/2.0):int((c_W-o_W)/2.0)+o_W,:] = 0 - + expand_mask = np.ones((c_H, c_W, 3), dtype=np.uint8) * 255 + if vertical_expansion_ratio == 1 and horizontal_expansion_ratio != 1: + expand_mask[int((c_H - o_H) / 2.0):int((c_H - o_H) / 2.0) + o_H, + int((c_W - o_W) / 2.0) + + blurry_gap:int((c_W - o_W) / 2.0) + o_W - + blurry_gap, :] = 0 + elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio != 1: + expand_mask[int((c_H - o_H) / 2.0) + + blurry_gap:int((c_H - o_H) / 2.0) + o_H - blurry_gap, + int((c_W - o_W) / 2.0) + + blurry_gap:int((c_W - o_W) / 2.0) + o_W - + blurry_gap, :] = 0 + elif vertical_expansion_ratio != 1 and horizontal_expansion_ratio == 1: + expand_mask[int((c_H - o_H) / 2.0) + + blurry_gap:int((c_H - o_H) / 2.0) + o_H - blurry_gap, + int((c_W - o_W) / + 2.0):int((c_W - o_W) / 2.0) + o_W, :] = 0 + input_image['image'] = Image.fromarray(expand_img) input_image['mask'] = Image.fromarray(expand_mask) - - - promptA, promptB, negative_promptA, negative_promptB = add_task( prompt, negative_prompt, task) # print(promptA, promptB, negative_promptA, negative_promptB) @@ -299,7 +309,8 @@ def predict_controlnet(input_image, input_control_image, control_type, prompt, def infer(input_image, text_guided_prompt, text_guided_negative_prompt, shape_guided_prompt, shape_guided_negative_prompt, fitting_degree, ddim_steps, scale, seed, task, enable_control, input_control_image, - control_type,vertical_expansion_ratio,horizontal_expansion_ratio,outpaint_prompt,outpaint_negative_prompt): + control_type, vertical_expansion_ratio, horizontal_expansion_ratio, + outpaint_prompt, outpaint_negative_prompt): if task == 'text-guided': prompt = text_guided_prompt negative_prompt = text_guided_negative_prompt @@ -313,7 +324,8 @@ def infer(input_image, text_guided_prompt, text_guided_negative_prompt, prompt = outpaint_prompt negative_prompt = outpaint_negative_prompt return predict(input_image, prompt, fitting_degree, ddim_steps, scale, - seed, negative_prompt, task,vertical_expansion_ratio,horizontal_expansion_ratio) + seed, negative_prompt, task, vertical_expansion_ratio, + horizontal_expansion_ratio) else: task = 'text-guided' prompt = text_guided_prompt @@ -325,7 +337,7 @@ def infer(input_image, text_guided_prompt, text_guided_negative_prompt, seed, negative_prompt) else: return predict(input_image, prompt, fitting_degree, ddim_steps, scale, - seed, negative_prompt, task,None,None) + seed, negative_prompt, task, None, None) def select_tab_text_guided(): @@ -335,6 +347,7 @@ def select_tab_text_guided(): def select_tab_object_removal(): return 'object-removal' + def select_tab_image_outpainting(): return 'image-outpainting' @@ -360,7 +373,10 @@ def select_tab_shape_guided(): gr.Markdown('### Input image and draw mask') input_image = gr.Image(source='upload', tool='sketch', type='pil') - task = gr.Radio(['text-guided', 'object-removal', 'shape-guided', 'image-outpainting'], + task = gr.Radio([ + 'text-guided', 'object-removal', 'shape-guided', + 'image-outpainting' + ], show_label=False, visible=False) @@ -391,7 +407,7 @@ def select_tab_shape_guided(): interactive=False) tab_object_removal.select( fn=select_tab_object_removal, inputs=None, outputs=task) - + # Object image outpainting with gr.Tab('Image outpainting') as tab_image_outpainting: enable_object_removal = gr.Checkbox( @@ -443,7 +459,8 @@ def select_tab_shape_guided(): label='Steps', minimum=1, maximum=50, value=45, step=1) scale = gr.Slider( label='Guidance Scale', - info='For object removal, it is recommended to set the value at 10 or above, while for image outpainting, it is advisable to set it at 18 or above.', + info= + 'For object removal, it is recommended to set the value at 10 or above, while for image outpainting, it is advisable to set it at 18 or above.', minimum=0.1, maximum=30.0, value=7.5, @@ -468,7 +485,8 @@ def select_tab_shape_guided(): input_image, text_guided_prompt, text_guided_negative_prompt, shape_guided_prompt, shape_guided_negative_prompt, fitting_degree, ddim_steps, scale, seed, task, enable_control, input_control_image, - control_type,vertical_expansion_ratio,horizontal_expansion_ratio,outpaint_prompt,outpaint_negative_prompt + control_type, vertical_expansion_ratio, horizontal_expansion_ratio, + outpaint_prompt, outpaint_negative_prompt ], outputs=[inpaint_result, gallery]) From 60585ee929ee9711ce4b24c8112d2b7b1bb69eb4 Mon Sep 17 00:00:00 2001 From: zhuang2002 <108931120+zhuang2002@users.noreply.github.com> Date: Sat, 9 Dec 2023 12:55:22 +0800 Subject: [PATCH 3/7] Update gradio_PowerPaint.py --- projects/powerpaint/gradio_PowerPaint.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/powerpaint/gradio_PowerPaint.py b/projects/powerpaint/gradio_PowerPaint.py index ff6ec9bb4..b8b04d31b 100644 --- a/projects/powerpaint/gradio_PowerPaint.py +++ b/projects/powerpaint/gradio_PowerPaint.py @@ -460,7 +460,10 @@ def select_tab_shape_guided(): scale = gr.Slider( label='Guidance Scale', info= - 'For object removal, it is recommended to set the value at 10 or above, while for image outpainting, it is advisable to set it at 18 or above.', + 'For object removal, \ + it is recommended to set the value at 10 or above, \ + while for image outpainting, \ + it is advisable to set it at 18 or above.', minimum=0.1, maximum=30.0, value=7.5, From 168b4149a4ec5111e7fdbf02dbec15df5f993178 Mon Sep 17 00:00:00 2001 From: zhuang2002 <108931120+zhuang2002@users.noreply.github.com> Date: Sat, 9 Dec 2023 13:07:12 +0800 Subject: [PATCH 4/7] Update gradio_PowerPaint.py --- projects/powerpaint/gradio_PowerPaint.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/projects/powerpaint/gradio_PowerPaint.py b/projects/powerpaint/gradio_PowerPaint.py index b8b04d31b..6fc512501 100644 --- a/projects/powerpaint/gradio_PowerPaint.py +++ b/projects/powerpaint/gradio_PowerPaint.py @@ -127,7 +127,8 @@ def predict(input_image, prompt, fitting_degree, ddim_steps, scale, seed, input_image['image'] = input_image['image'].convert('RGB').resize( (int(size1 / size2 * 512), 512)) - if vertical_expansion_ratio != None and horizontal_expansion_ratio != None: + if (vertical_expansion_ratio is not None) and (horizontal_expansion_ratio + is not None): # noqa o_W, o_H = input_image['image'].convert('RGB').size c_W = int(horizontal_expansion_ratio * o_W) c_H = int(vertical_expansion_ratio * o_H) @@ -358,14 +359,22 @@ def select_tab_shape_guided(): with gr.Blocks(css='style.css') as demo: with gr.Row(): - gr.Markdown( - "