diff --git a/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py b/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py index 09befc12..179fedec 100644 --- a/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py +++ b/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py @@ -1,52 +1,33 @@ -from modules_forge.supported_preprocessor import Preprocessor, PreprocessorParameter -from modules_forge.shared import preprocessor_dir, add_supported_preprocessor -from modules.modelloader import load_file_from_url +from modules_forge.supported_preprocessor import PreprocessorClipVision +from modules_forge.shared import add_supported_preprocessor from modules_forge.forge_util import numpy_to_pytorch -import ldm_patched.modules.clip_vision - -class PreprocessorClipVision(Preprocessor): +class PreprocessorClipVisionForIPAdapter(PreprocessorClipVision): def __init__(self, name, url, filename): - super().__init__() - self.name = name - self.url = url - self.filename = filename + super().__init__(name, url, filename) self.tags = ['IP-Adapter'] self.model_filename_filters = ['IP-Adapter', 'IP_Adapter'] - self.slider_resolution = PreprocessorParameter(visible=False) - self.corp_image_with_a1111_mask_when_in_img2img_inpaint_tab = False - self.show_control_mode = False - self.sorting_priority = 1 - self.clipvision = None def __call__(self, input_image, resolution, slider_1=None, slider_2=None, slider_3=None, **kwargs): - if self.clipvision is None: - ckpt_path = load_file_from_url( - url=self.url, - model_dir=preprocessor_dir, - file_name=self.filename - ) - self.clipvision = ldm_patched.modules.clip_vision.load(ckpt_path) - - # For IPAdapter Format - return self.clipvision, numpy_to_pytorch(input_image) + clipvision = self.load_clipvision() + return clipvision, numpy_to_pytorch(input_image) -add_supported_preprocessor(PreprocessorClipVision( - name='CLIP-ViT-H', +add_supported_preprocessor(PreprocessorClipVisionForIPAdapter( + name='CLIP-ViT-H (IPAdapter)', url='https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors', filename='CLIP-ViT-H-14.safetensors' )) -add_supported_preprocessor(PreprocessorClipVision( - name='CLIP-ViT-bigG', +add_supported_preprocessor(PreprocessorClipVisionForIPAdapter( + name='CLIP-ViT-bigG (IPAdapter)', url='https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors', filename='CLIP-ViT-bigG.safetensors' )) -add_supported_preprocessor(PreprocessorClipVision( - name='CLIP-ViT-L', +add_supported_preprocessor(PreprocessorClipVisionForIPAdapter( + name='CLIP-ViT-L (IPAdapter)', url='https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin', filename='CLIP-ViT-bigG.safetensors' )) diff --git a/modules_forge/supported_preprocessor.py b/modules_forge/supported_preprocessor.py index 2a0e2fab..ae08847b 100644 --- a/modules_forge/supported_preprocessor.py +++ b/modules_forge/supported_preprocessor.py @@ -1,10 +1,13 @@ import cv2 import torch -from modules_forge.shared import add_supported_preprocessor +from modules_forge.shared import add_supported_preprocessor, preprocessor_dir from ldm_patched.modules import model_management from ldm_patched.modules.model_patcher import ModelPatcher from modules_forge.forge_util import resize_image_with_pad +import ldm_patched.modules.clip_vision +from modules.modelloader import load_file_from_url +from modules_forge.forge_util import numpy_to_pytorch class PreprocessorParameter: @@ -87,4 +90,41 @@ class PreprocessorCanny(Preprocessor): add_supported_preprocessor(PreprocessorNone()) -add_supported_preprocessor(PreprocessorCanny()) \ No newline at end of file +add_supported_preprocessor(PreprocessorCanny()) + + +class PreprocessorClipVision(Preprocessor): + global_cache = {} + + def __init__(self, name, url, filename): + super().__init__() + self.name = name + self.url = url + self.filename = filename + self.slider_resolution = PreprocessorParameter(visible=False) + self.corp_image_with_a1111_mask_when_in_img2img_inpaint_tab = False + self.show_control_mode = False + self.sorting_priority = 1 + self.clipvision = None + + def load_clipvision(self): + if self.clipvision is not None: + return self.clipvision + + ckpt_path = load_file_from_url( + url=self.url, + model_dir=preprocessor_dir, + file_name=self.filename + ) + + if ckpt_path in PreprocessorClipVision.global_cache: + self.clipvision = PreprocessorClipVision.global_cache[ckpt_path] + else: + self.clipvision = ldm_patched.modules.clip_vision.load(ckpt_path) + PreprocessorClipVision.global_cache[ckpt_path] = self.clipvision + + return self.clipvision + + def __call__(self, input_image, resolution, slider_1=None, slider_2=None, slider_3=None, **kwargs): + clipvision = self.load_clipvision() + return clipvision.encode_image(numpy_to_pytorch(input_image))