minigpt_vlm_attack.py

# 1. Download Vicuna's weights to ./models   (it's a delta version)
# 2. Download LLaMA's weight via: https://huggingface.co/huggyllama/llama-13b/tree/main
# 3. merge them and setup config
# 4. Download the mini-gpt4 compoents' pretrained ckpts
# 5. vision part will be automatically download when launching the model


import argparse
import os
import random

import numpy as np
import torch
import torch.backends.cudnn as cudnn
from PIL import Image
from torchvision.utils import save_image
from minigpt_utils import vlm_attacker, prompt_wrapper

from minigpt4.common.config import Config
from minigpt4.common.dist_utils import get_rank
from minigpt4.common.registry import registry



def parse_args():

    parser = argparse.ArgumentParser(description="Demo")
    parser.add_argument("--cfg_path", default="eval_configs/minigpt4_eval.yaml", help="path to configuration file.")
    parser.add_argument("--gpu_id", type=int, default=0, help="specify the gpu to load the model.")
    parser.add_argument("--n_iters", type=int, default=500, help="specify the number of iterations for attack.")
    parser.add_argument('--eps', type=int, default=32, help="epsilon of the attack budget")
    parser.add_argument('--alpha', type=int, default=1, help="step_size of the attack")


    parser.add_argument("--save_dir", type=str, default='output',
                        help="save directory")

    parser.add_argument(
        "--options",
        nargs="+",
        help="override some settings in the used config, the key-value pair "
             "in xxx=yyy format will be merged into config file (deprecate), "
             "change to --cfg-options instead.",
    )
    args = parser.parse_args()
    return args


def setup_seeds(config):
    seed = config.run_cfg.seed + get_rank()

    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)

    cudnn.benchmark = False
    cudnn.deterministic = True


# ========================================
#             Model Initialization
# ========================================


print('>>> Initializing Models')

args = parse_args()
cfg = Config(args)

model_config = cfg.model_cfg
model_config.device_8bit = args.gpu_id
model_cls = registry.get_model_class(model_config.arch)
model = model_cls.from_config(model_config).to('cuda:{}'.format(args.gpu_id))
model.eval()

vis_processor_cfg = cfg.datasets_cfg.cc_sbu_align.vis_processor.train
vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)

print('[Initialization Finished]\n')



if not os.path.exists(args.save_dir):
    os.mkdir(args.save_dir)



import csv


file = open("harmful_corpus/harmful_behaviors.csv", "r")

data = list(csv.reader(file))
file.close()
queries=[];targets = []
num = len(data)



for i in range(1,num):
    queries.append(data[i][0])
    targets.append(data[i][1])
    

all_numbers = list(range(len(queries)))


selected_numbers = random.sample(all_numbers, 66)


queries = [queries[index] for index in selected_numbers]
targets = [targets[index]+':' for index in selected_numbers]



my_attacker = vlm_attacker.Attacker(args, model, queries, targets, device=model.device, is_rtp=False)


template_img = 'adversarial_images/prompt_unconstrained_init.bmp'
img = Image.open(template_img).convert('RGB')
img = vis_processor(img).unsqueeze(0).to(model.device)



text_prompt_template = prompt_wrapper.minigpt4_chatbot_prompt






adv_img_prompt = my_attacker.attack_unconstrained(text_prompt_template,
                                                            img=img, batch_size = 8,
                                                            num_iter=250, alpha=args.alpha/255)



save_image(adv_img_prompt, '%s/bad_vlm_prompt.bmp' % args.save_dir)
print('[Done]')