diff --git a/README.md b/README.md index 61b916c0..b21b6cf6 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,7 @@ The memory optimization in this example is fully automatic. You do not need to c import cv2 import gradio as gr +import torch from modules import scripts from modules.shared_cmd_options import cmd_opts @@ -472,18 +473,26 @@ class ControlNetExampleForge(scripts.Script): sigma_min = unet.model.model_sampling.sigma_min advanced_sigma_weighting = lambda s: (s - sigma_min) / (sigma_max - sigma_min) + # You can even input a tensor to mask all control injections + # The mask will be automatically resized during inference in UNet. + # The size should be B 1 H W and the H and W are not important + # because they will be resized automatically + advanced_mask_weighting = torch.ones(size=(1, 1, 512, 512)) + # But in this simple example we do not use them positive_advanced_weighting = None negative_advanced_weighting = None advanced_frame_weighting = None advanced_sigma_weighting = None + advanced_mask_weighting = None unet = apply_controlnet_advanced(unet=unet, controlnet=self.model, image_bchw=control_image_bchw, strength=0.6, start_percent=0.0, end_percent=0.8, positive_advanced_weighting=positive_advanced_weighting, negative_advanced_weighting=negative_advanced_weighting, advanced_frame_weighting=advanced_frame_weighting, - advanced_sigma_weighting=advanced_sigma_weighting) + advanced_sigma_weighting=advanced_sigma_weighting, + advanced_mask_weighting=advanced_mask_weighting) p.sd_model.forge_objects.unet = unet diff --git a/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py b/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py index 2f9a168c..9c10cb23 100644 --- a/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py +++ b/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py @@ -2,6 +2,7 @@ import cv2 import gradio as gr +import torch from modules import scripts from modules.shared_cmd_options import cmd_opts @@ -122,18 +123,26 @@ class ControlNetExampleForge(scripts.Script): sigma_min = unet.model.model_sampling.sigma_min advanced_sigma_weighting = lambda s: (s - sigma_min) / (sigma_max - sigma_min) + # You can even input a tensor to mask all control injections + # The mask will be automatically resized during inference in UNet. + # The size should be B 1 H W and the H and W are not important + # because they will be resized automatically + advanced_mask_weighting = torch.ones(size=(1, 1, 512, 512)) + # But in this simple example we do not use them positive_advanced_weighting = None negative_advanced_weighting = None advanced_frame_weighting = None advanced_sigma_weighting = None + advanced_mask_weighting = None unet = apply_controlnet_advanced(unet=unet, controlnet=self.model, image_bchw=control_image_bchw, strength=0.6, start_percent=0.0, end_percent=0.8, positive_advanced_weighting=positive_advanced_weighting, negative_advanced_weighting=negative_advanced_weighting, advanced_frame_weighting=advanced_frame_weighting, - advanced_sigma_weighting=advanced_sigma_weighting) + advanced_sigma_weighting=advanced_sigma_weighting, + advanced_mask_weighting=advanced_mask_weighting) p.sd_model.forge_objects.unet = unet diff --git a/ldm_patched/modules/controlnet.py b/ldm_patched/modules/controlnet.py index b9879235..9ddd3c6b 100644 --- a/ldm_patched/modules/controlnet.py +++ b/ldm_patched/modules/controlnet.py @@ -87,7 +87,7 @@ def compute_controlnet_weighting(control, cnet): final_weight = final_weight * sigma_weight * frame_weight if isinstance(advanced_mask_weighting, torch.Tensor): - control_signal = control_signal * torch.nn.functional.interpolate(advanced_mask_weighting, size=(H, W), mode='bilinear') + control_signal = control_signal * torch.nn.functional.interpolate(advanced_mask_weighting.to(control_signal), size=(H, W), mode='bilinear') control[k][i] = control_signal * final_weight[:, None, None, None]