From 459b8ca81a32799aebca681c0ab97e29ddca318f Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 5 Dec 2022 17:58:35 +0100 Subject: [PATCH] Research folder (#1553) * Research folder * Update examples/research_projects/README.md * up --- examples/dreambooth/README.md | 26 +------------------ examples/research_projects/README.md | 14 ++++++++++ .../dreambooth_inpaint/README.md | 26 +++++++++++++++++++ .../dreambooth_inpaint/requirements.txt | 7 +++++ .../train_dreambooth_inpaint.py | 0 .../versatile_diffusion/modeling_text_unet.py | 2 +- 6 files changed, 49 insertions(+), 26 deletions(-) create mode 100644 examples/research_projects/README.md create mode 100644 examples/research_projects/dreambooth_inpaint/README.md create mode 100644 examples/research_projects/dreambooth_inpaint/requirements.txt rename examples/{dreambooth => research_projects/dreambooth_inpaint}/train_dreambooth_inpaint.py (100%) diff --git a/examples/dreambooth/README.md b/examples/dreambooth/README.md index e6438f1a053b..b4f0bf7b1330 100644 --- a/examples/dreambooth/README.md +++ b/examples/dreambooth/README.md @@ -312,30 +312,6 @@ python train_dreambooth_flax.py \ --max_train_steps=800 ``` -## Dreambooth for the inpainting model - - -```bash -export MODEL_NAME="runwayml/stable-diffusion-inpainting" -export INSTANCE_DIR="path-to-instance-images" -export OUTPUT_DIR="path-to-save-model" - -accelerate launch train_dreambooth_inpaint.py \ - --pretrained_model_name_or_path=$MODEL_NAME \ - --instance_data_dir=$INSTANCE_DIR \ - --output_dir=$OUTPUT_DIR \ - --instance_prompt="a photo of sks dog" \ - --resolution=512 \ - --train_batch_size=1 \ - --gradient_accumulation_steps=1 \ - --learning_rate=5e-6 \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --max_train_steps=400 -``` - -The script is also compatible with prior preservation loss and gradient checkpointing - ### Training with prior-preservation loss Prior-preservation is used to avoid overfitting and language-drift. Refer to the paper to learn more about it. For prior-preservation we first generate images using the model with a class prompt and then use those during training along with our data. @@ -428,4 +404,4 @@ accelerate launch train_dreambooth_inpaint.py \ --lr_warmup_steps=0 \ --num_class_images=200 \ --max_train_steps=800 -``` \ No newline at end of file +``` diff --git a/examples/research_projects/README.md b/examples/research_projects/README.md new file mode 100644 index 000000000000..ef50d423e68f --- /dev/null +++ b/examples/research_projects/README.md @@ -0,0 +1,14 @@ +# Research projects + +This folder contains various research projects using 🧨 Diffusers. +They are not really maintained by the core maintainers of this library and often require a specific version of Diffusers that is indicated in the requirements file of each folder. +Updating them to the most recent version of the library will require some work. + +To use any of them, just run the command + +``` +pip install -r requirements.txt +``` +inside the folder of your choice. + +If you need help with any of those, please open an issue where you directly ping the author(s), as indicated at the top of the README of each folder. diff --git a/examples/research_projects/dreambooth_inpaint/README.md b/examples/research_projects/dreambooth_inpaint/README.md new file mode 100644 index 000000000000..6830d1e2cb44 --- /dev/null +++ b/examples/research_projects/dreambooth_inpaint/README.md @@ -0,0 +1,26 @@ +# Dreambooth for the inpainting model + +This script was added by @thedarkzeno . + +Please note that this script is not actively maintained, you can open an issue and tag @thedarkzeno or @patil-suraj though. + +```bash +export MODEL_NAME="runwayml/stable-diffusion-inpainting" +export INSTANCE_DIR="path-to-instance-images" +export OUTPUT_DIR="path-to-save-model" + +accelerate launch train_dreambooth_inpaint.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --instance_data_dir=$INSTANCE_DIR \ + --output_dir=$OUTPUT_DIR \ + --instance_prompt="a photo of sks dog" \ + --resolution=512 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --learning_rate=5e-6 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --max_train_steps=400 +``` + +The script is also compatible with prior preservation loss and gradient checkpointing diff --git a/examples/research_projects/dreambooth_inpaint/requirements.txt b/examples/research_projects/dreambooth_inpaint/requirements.txt new file mode 100644 index 000000000000..6063931146f9 --- /dev/null +++ b/examples/research_projects/dreambooth_inpaint/requirements.txt @@ -0,0 +1,7 @@ +diffusers==0.9.0 +accelerate +torchvision +transformers>=4.21.0 +ftfy +tensorboard +modelcards diff --git a/examples/dreambooth/train_dreambooth_inpaint.py b/examples/research_projects/dreambooth_inpaint/train_dreambooth_inpaint.py similarity index 100% rename from examples/dreambooth/train_dreambooth_inpaint.py rename to examples/research_projects/dreambooth_inpaint/train_dreambooth_inpaint.py diff --git a/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py b/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py index fdf97481feb9..6429693469dd 100644 --- a/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py +++ b/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py @@ -314,7 +314,7 @@ def set_attention_slice(self, slice_size): in several steps. This is useful to save some memory in exchange for a small speed decrease. Args: - slice_size (`str` or `int`, *optional*, defaults to `"auto"`): + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If `"max"`, maxium amount of memory will be saved by running only one slice at a time. If a number is provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`