soaska/my-sd
soaska/my-sd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Activity

better clipvision

Browse Source
This commit is contained in:
lllyasviel 2024-01-30 08:58:41 -08:00
parent 3aafda41bb
commit e3dcb2c5a3
2 changed files with 54 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py
View File

@ -1,52 +1,33 @@
from modules_forge.supported_preprocessor import Preprocessor, PreprocessorParameter
from modules_forge.shared import preprocessor_dir, add_supported_preprocessor
from modules.modelloader import load_file_from_url
from modules_forge.supported_preprocessor import PreprocessorClipVision
from modules_forge.shared import add_supported_preprocessor
from modules_forge.forge_util import numpy_to_pytorch
import ldm_patched.modules.clip_vision
class PreprocessorClipVision(Preprocessor):
class PreprocessorClipVisionForIPAdapter(PreprocessorClipVision):
def __init__(self, name, url, filename):
super().__init__()
self.name = name
self.url = url
self.filename = filename
super().__init__(name, url, filename)
self.tags = ['IP-Adapter']
self.model_filename_filters = ['IP-Adapter', 'IP_Adapter']
self.slider_resolution = PreprocessorParameter(visible=False)
self.corp_image_with_a1111_mask_when_in_img2img_inpaint_tab = False
self.show_control_mode = False
self.sorting_priority = 1
self.clipvision = None
def __call__(self, input_image, resolution, slider_1=None, slider_2=None, slider_3=None, **kwargs):
if self.clipvision is None:
ckpt_path = load_file_from_url(
url=self.url,
model_dir=preprocessor_dir,
file_name=self.filename
)
self.clipvision = ldm_patched.modules.clip_vision.load(ckpt_path)
# For IPAdapter Format
return self.clipvision, numpy_to_pytorch(input_image)
clipvision = self.load_clipvision()
return clipvision, numpy_to_pytorch(input_image)
add_supported_preprocessor(PreprocessorClipVision(
name='CLIP-ViT-H',
add_supported_preprocessor(PreprocessorClipVisionForIPAdapter(
name='CLIP-ViT-H (IPAdapter)',
url='https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors',
filename='CLIP-ViT-H-14.safetensors'
))
add_supported_preprocessor(PreprocessorClipVision(
name='CLIP-ViT-bigG',
add_supported_preprocessor(PreprocessorClipVisionForIPAdapter(
name='CLIP-ViT-bigG (IPAdapter)',
url='https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors',
filename='CLIP-ViT-bigG.safetensors'
))
add_supported_preprocessor(PreprocessorClipVision(
name='CLIP-ViT-L',
add_supported_preprocessor(PreprocessorClipVisionForIPAdapter(
name='CLIP-ViT-L (IPAdapter)',
url='https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin',
filename='CLIP-ViT-bigG.safetensors'
))

44
modules_forge/supported_preprocessor.py
View File

@ -1,10 +1,13 @@
import cv2
import torch
from modules_forge.shared import add_supported_preprocessor
from modules_forge.shared import add_supported_preprocessor, preprocessor_dir
from ldm_patched.modules import model_management
from ldm_patched.modules.model_patcher import ModelPatcher
from modules_forge.forge_util import resize_image_with_pad
import ldm_patched.modules.clip_vision
from modules.modelloader import load_file_from_url
from modules_forge.forge_util import numpy_to_pytorch
class PreprocessorParameter:
@ -87,4 +90,41 @@ class PreprocessorCanny(Preprocessor):
add_supported_preprocessor(PreprocessorNone())
add_supported_preprocessor(PreprocessorCanny())
add_supported_preprocessor(PreprocessorCanny())
class PreprocessorClipVision(Preprocessor):
global_cache = {}
def __init__(self, name, url, filename):
super().__init__()
self.name = name
self.url = url
self.filename = filename
self.slider_resolution = PreprocessorParameter(visible=False)
self.corp_image_with_a1111_mask_when_in_img2img_inpaint_tab = False
self.show_control_mode = False
self.sorting_priority = 1
self.clipvision = None
def load_clipvision(self):
if self.clipvision is not None:
return self.clipvision
ckpt_path = load_file_from_url(
url=self.url,
model_dir=preprocessor_dir,
file_name=self.filename
)
if ckpt_path in PreprocessorClipVision.global_cache:
self.clipvision = PreprocessorClipVision.global_cache[ckpt_path]
else:
self.clipvision = ldm_patched.modules.clip_vision.load(ckpt_path)
PreprocessorClipVision.global_cache[ckpt_path] = self.clipvision
return self.clipvision
def __call__(self, input_image, resolution, slider_1=None, slider_2=None, slider_3=None, **kwargs):
clipvision = self.load_clipvision()
return clipvision.encode_image(numpy_to_pytorch(input_image))