-
Notifications
You must be signed in to change notification settings - Fork 1
/
sparkleschat.yaml
115 lines (103 loc) · 3.88 KB
/
sparkleschat.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
model:
arch: sparkleschat
model_type: pretrain_vicuna
freeze_vit: True
freeze_qformer: True
# ckpt: '/mnt/localdata/Users/yupanhuang/data/Sparkles/models/pretrained/minigpt4_7b_stage1.pth'
ckpt: '/mnt/localdata/Users/yupanhuang/models/cheetah/cheetah_vicuna_7b.pth'
end_sym: "###"
prompt_template: '###Human: {} ###Assistant: '
max_txt_len: 160 # for MiniGPT-4
prompt_path: "prompts/alignment.txt" # for MiniGPT-4
datasets:
SparklesDialogueCC_turn1_1img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueCC/'
anno_path: 'annotations/SparklesDialogueCC_turn1_1img.json'
img_root: 'images'
sample_ratio: 2
SparklesDialogueCC_turn1_2img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueCC/'
anno_path: 'annotations/SparklesDialogueCC_turn1_2img.json'
img_root: 'images'
sample_ratio: 2
SparklesDialogueCC_turn1_3img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueCC/'
anno_path: 'annotations/SparklesDialogueCC_turn1_3img.json'
img_root: 'images'
sample_ratio: 2
SparklesDialogueCC_turn2_2img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueCC/'
anno_path: 'annotations/SparklesDialogueCC_turn2_2img.json'
img_root: 'images'
sample_ratio: 1
SparklesDialogueCC_turn2_3img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueCC/'
anno_path: 'annotations/SparklesDialogueCC_turn2_3img.json'
img_root: 'images'
sample_ratio: 1
SparklesDialogueCC_turn2_4img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueCC/'
anno_path: 'annotations/SparklesDialogueCC_turn2_4img.json'
img_root: 'images'
sample_ratio: 1
SparklesDialogueVG_turn1_2img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueVG/'
anno_path: 'annotations/SparklesDialogueVG_turn1_2img.json'
img_root: 'images'
sample_ratio: 2
SparklesDialogueVG_turn1_3img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueVG/'
anno_path: 'annotations/SparklesDialogueVG_turn1_3img.json'
img_root: 'images'
sample_ratio: 2
SparklesDialogueVG_turn2_3img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueVG/'
anno_path: 'annotations/SparklesDialogueVG_turn2_3img.json'
img_root: 'images'
sample_ratio: 1
SparklesDialogueVG_turn2_4img:
root_dir: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/SparklesDialogueVG/'
anno_path: 'annotations/SparklesDialogueVG_turn2_4img.json'
img_root: 'images'
sample_ratio: 1
# cc_sbu_align:
# sample_ratio: 0
# build_info:
# storage: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/cc_sbu_align/'
# ann_paths: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/cc_sbu_align/filter_cap.json'
# vis_root: '/mnt/localdata/Users/yupanhuang/data/Sparkles/data/cc_sbu_align/image'
#
# LLaVA_description:
# root_dir: '/path/to/'
# anno_path: 'Sparkles/data/LLaVa/LLaVA_detail_23k_filtered_for_Sparkles.json'
# img_root: 'coco/train2017'
# sample_ratio: 0
# LLaVA_reasoning:
# root_dir: '/path/to/'
# anno_path: 'Sparkles/data/LLaVa/LLaVA_complex_reasoning_77k_filtered_for_Sparkles.json'
# img_root: 'coco/train2017'
# sample_ratio: 0
run:
task: image_text_pretrain
lr_sched: "linear_warmup_cosine_lr"
init_lr: 3e-5
min_lr: 1e-5
warmup_lr: 1e-6
weight_decay: 0.05
max_epoch: 5
iters_per_epoch: 400
batch_size_train: 1
accum_grad_iters: 1
batch_size_eval: 1
num_workers: 8
warmup_steps: 400
seed: 42
output_dir: "/mnt/localdata/Users/yupanhuang/models/Sparkles/init_from_cheetah_vicuna_7b/"
amp: True
resume_ckpt_path: null
evaluate: False
train_splits: ["train"]
device: "cuda"
world_size: 1
dist_url: "env://"
distributed: True