From 90c4a6126be0c8576422805b62603162cb3c5172 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 28 Oct 2024 13:43:09 -0400 Subject: [PATCH] ip-instruct: experimental Signed-off-by: Vladimir Mandic --- .gitignore | 4 +- repositories/ip_instruct | 1 + scripts/ipinstruct.py | 111 +++++++++++++++++++++++++++++++++++++++ wiki | 2 +- 4 files changed, 116 insertions(+), 2 deletions(-) create mode 160000 repositories/ip_instruct create mode 100644 scripts/ipinstruct.py diff --git a/.gitignore b/.gitignore index 2bc819860..4d0ab36c6 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,9 @@ tunableop_results*.csv !webui.sh !package.json +# dynamically generated +/repositories/ip-instruct/ + # all dynamic stuff /extensions/**/* /outputs/**/* @@ -59,7 +62,6 @@ tunableop_results*.csv .vscode/ .idea/ /localizations - .*/ # force included diff --git a/repositories/ip_instruct b/repositories/ip_instruct new file mode 160000 index 000000000..57efbe40a --- /dev/null +++ b/repositories/ip_instruct @@ -0,0 +1 @@ +Subproject commit 57efbe40a4a05d5688a869804244f9312a660efb diff --git a/scripts/ipinstruct.py b/scripts/ipinstruct.py new file mode 100644 index 000000000..2ba4c4ff5 --- /dev/null +++ b/scripts/ipinstruct.py @@ -0,0 +1,111 @@ +""" +Repo: +Models: +adapter: `sd15`=0.35GB `sdxl`=2.12GB `sd3`=1.56GB +encoder: `laion/CLIP-ViT-H-14-laion2B-s32B-b79K`=3.94GB +""" +import os +import importlib +import gradio as gr +from modules import scripts, processing, shared, sd_models, devices + + +repo = 'https://github.com/vladmandic/IP-Instruct' +repo_id = 'CiaraRowles/IP-Adapter-Instruct' +encoder = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K" +folder = os.path.join('repositories', 'ip_instruct') + + +class Script(scripts.Script): + def __init__(self): + super().__init__() + self.orig_pipe = None + self.lib = None + + def title(self): + return 'IP Instruct' + + def show(self, is_img2img): + if shared.cmd_opts.experimental: + return not is_img2img if shared.native else False + else: + return False + + def install(self): + if not os.path.exists(folder): + from installer import clone + clone(repo, folder) + if self.lib is None: + self.lib = importlib.import_module('ip_instruct.ip_adapter') + + + def ui(self, _is_img2img): # ui elements + with gr.Row(): + gr.HTML('  IP Adapter Instruct
') + with gr.Row(): + query = gr.Textbox(lines=1, label='Query', placeholder='use the composition from the image') + with gr.Row(): + image = gr.Image(value=None, label='Image', type='pil', source='upload', width=256, height=256) + with gr.Row(): + strength = gr.Slider(label="Strength", value=1.0, minimum=0, maximum=2.0, step=0.05) + tokens = gr.Slider(label="Tokens", value=4, minimum=1, maximum=32, step=1) + with gr.Row(): + instruct_guidance = gr.Slider(label="Guidance", value=6.0, minimum=1.0, maximum=15.0, step=0.05) + image_guidance = gr.Slider(label="Guidance", value=0.5, minimum=0, maximum=1.0, step=0.05) + return [query, image, strength, tokens, instruct_guidance, image_guidance] + + def run(self, p: processing.StableDiffusionProcessing, query, image, strength, tokens, instruct_guidance, image_guidance): # pylint: disable=arguments-differ + supported_model_list = ['sd', 'sdxl', 'sd3'] + if shared.sd_model_type not in supported_model_list: + shared.log.warning(f'IP-Instruct: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') + return None + self.install() + if self.lib is None: + shared.log.error('IP-Instruct: failed to import library') + return None + self.orig_pipe = shared.sd_model + if shared.sd_model_type == 'sdxl': + pipe = self.lib.StableDiffusionXLPipelineExtraCFG + cls = self.lib.IPAdapterInstructSDXL + ckpt = "ip-adapter-instruct-sdxl.bin" + elif shared.sd_model_type == 'sd3': + pipe = self.lib.StableDiffusion3PipelineExtraCFG + cls = self.lib.IPAdapter_sd3_Instruct + ckpt = "ip-adapter-instruct-sd3.bin" + else: + pipe = self.lib.StableDiffusionPipelineCFG + cls = self.lib.IPAdapterInstruct + ckpt = "ip-adapter-instruct-sd15.bin" + + shared.sd_model = sd_models.switch_pipe(pipe, shared.sd_model) + + import huggingface_hub as hf + ip_ckpt = hf.hf_hub_download(repo_id=repo_id, filename=ckpt, cache_dir=shared.opts.hfcache_dir) + ip_model = cls(shared.sd_model, encoder, ip_ckpt, device=devices.device, dtypein=devices.dtype, num_tokens=tokens) + processing.fix_seed(p) + shared.log.debug(f'IP-Instruct: class={shared.sd_model.__class__.__name__} wrapper={ip_model.__class__.__name__} encoder={encoder} adapter={ckpt}') + shared.log.info(f'IP-Instruct: image={image} query="{query}" strength={strength} tokens={tokens} instruct_guidance={instruct_guidance} image_guidance={image_guidance}') + + image_list = ip_model.generate( + query = query, + scale = strength, + instruct_guidance_scale = instruct_guidance, + image_guidance_scale = image_guidance, + + prompt = p.prompt, + pil_image = image, + num_samples = 1, + num_inference_steps = p.steps, + seed = p.seed, + guidance_scale = p.cfg_scale, + auto_scale = False, + simple_cfg_mode = False, + ) + processed = processing.Processed(p, images_list=image_list, seed=p.seed, subseed=p.subseed, index_of_first_image=0) # manually created processed object + p.extra_generation_params["IPInstruct"] = f'' + return processed + + def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, **kwargs): # pylint: disable=unused-argument + if self.orig_pipe is not None: + shared.sd_model = self.orig_pipe + return processed diff --git a/wiki b/wiki index 23cc41bc4..b2d3110d4 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 23cc41bc44ce06a9f6620452817f3cbfbed86f84 +Subproject commit b2d3110d42ef1417008295425437721b25369cb1