From 736a5aa3ac4be5aeaa35697d6f1d160710c718e9 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 21 Oct 2023 13:36:02 -0700 Subject: [PATCH 01/78] improve anime Improve Fooocus Anime a bit by using better SD1.5 refining formulation. --- fooocus_extras/vae_interpose.py | 7 ++-- fooocus_version.py | 2 +- modules/core.py | 16 +++++---- modules/default_pipeline.py | 62 ++++++++++----------------------- modules/sample_hijack.py | 4 --- update_log.md | 4 +++ 6 files changed, 36 insertions(+), 59 deletions(-) diff --git a/fooocus_extras/vae_interpose.py b/fooocus_extras/vae_interpose.py index 41f8192..b069b2f 100644 --- a/fooocus_extras/vae_interpose.py +++ b/fooocus_extras/vae_interpose.py @@ -69,7 +69,7 @@ vae_approx_filename = os.path.join(vae_approx_path, 'xl-to-v1_interposer-v3.1.sa def parse(x): global vae_approx_model - x_origin = x['samples'].clone() + x_origin = x.clone() if vae_approx_model is None: model = Interposer() @@ -89,6 +89,5 @@ def parse(x): fcbh.model_management.load_model_gpu(vae_approx_model) x = x_origin.to(device=vae_approx_model.load_device, dtype=vae_approx_model.dtype) - x = vae_approx_model.model(x) - - return {'samples': x.to(x_origin)} + x = vae_approx_model.model(x).to(x_origin) + return x diff --git a/fooocus_version.py b/fooocus_version.py index 8d20af6..c264cd8 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.722' +version = '2.1.723' diff --git a/modules/core.py b/modules/core.py index c58b0fa..8cad32a 100644 --- a/modules/core.py +++ b/modules/core.py @@ -218,19 +218,21 @@ def get_previewer(model): def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sampler_name='dpmpp_2m_sde_gpu', scheduler='karras', denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, callback_function=None, refiner=None, refiner_switch=-1, - previewer_start=None, previewer_end=None, sigmas=None, noise=None): + previewer_start=None, previewer_end=None, sigmas=None, extra_noise=0.0): if sigmas is not None: sigmas = sigmas.clone().to(fcbh.model_management.get_torch_device()) latent_image = latent["samples"] - if noise is None: - if disable_noise: - noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") - else: - batch_inds = latent["batch_index"] if "batch_index" in latent else None - noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds) + if disable_noise: + noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") + else: + batch_inds = latent["batch_index"] if "batch_index" in latent else None + noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds) + + if extra_noise > 0.0: + noise = noise * (1.0 + extra_noise) noise_mask = None if "noise_mask" in latent: diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 5322e84..d20a1ec 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -6,7 +6,7 @@ import modules.path import fcbh.model_management import fcbh.latent_formats import modules.inpaint_worker -import modules.sample_hijack as sample_hijack +import fooocus_extras.vae_interpose as vae_interpose from fcbh.model_base import SDXL, SDXLRefiner from modules.expansion import FooocusExpansion @@ -270,22 +270,14 @@ refresh_everything( @torch.no_grad() @torch.inference_mode() -def vae_parse(x, tiled=False, use_interpose=True): - if final_vae is None or final_refiner_vae is None: - return x - - if use_interpose: - print('VAE interposing ...') - import fooocus_extras.vae_interpose - x = fooocus_extras.vae_interpose.parse(x) - print('VAE interposed ...') +def vae_parse(latent, k=1.0): + if final_refiner_vae is None: + result = latent["samples"] else: - print('VAE parsing ...') - x = core.decode_vae(vae=final_vae, latent_image=x, tiled=tiled) - x = core.encode_vae(vae=final_refiner_vae, pixels=x, tiled=tiled) - print('VAE parsed ...') - - return x + result = vae_interpose.parse(latent["samples"]) + if k != 1.0: + result = result * k + return {'samples': result} @torch.no_grad() @@ -444,8 +436,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.unswap() - sample_hijack.history_record = [] - core.ksampler( + sampled_latent = core.ksampler( model=final_unet, positive=positive_cond, negative=negative_cond, @@ -467,34 +458,20 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height target_model = final_unet print('Use base model to refine itself - this may because of developer mode.') + # Fooocus' vae parameters + k_data = 1.05 + k_noise = 0.15 + k_sigmas = 1.4 + + sampled_latent = vae_parse(sampled_latent, k=k_data) + sigmas = calculate_sigmas(sampler=sampler_name, scheduler=scheduler_name, model=target_model.model, steps=steps, - denoise=denoise)[switch:] - k1 = target_model.model.latent_format.scale_factor - k2 = final_unet.model.latent_format.scale_factor - k_sigmas = float(k1) / float(k2) - sigmas = sigmas * k_sigmas + denoise=denoise)[switch:] * k_sigmas len_sigmas = len(sigmas) - 1 - last_step, last_clean_latent, last_noisy_latent = sample_hijack.history_record[-1] - last_clean_latent = final_unet.model.process_latent_out(last_clean_latent.cpu().to(torch.float32)) - last_noisy_latent = final_unet.model.process_latent_out(last_noisy_latent.cpu().to(torch.float32)) - last_noise = last_noisy_latent - last_clean_latent - last_noise = last_noise / last_noise.std() - - noise_mean = torch.mean(last_noise, dim=1, keepdim=True).repeat(1, 4, 1, 1) / k_sigmas - - refiner_noise = torch.normal( - mean=noise_mean, - std=torch.ones_like(noise_mean), - generator=torch.manual_seed(image_seed+1) # Avoid artifacts - ).to(last_noise) - - sampled_latent = {'samples': last_clean_latent} - sampled_latent = vae_parse(sampled_latent) - if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.swap() @@ -504,7 +481,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height negative=clip_separate(negative_cond, target_model=target_model.model, target_clip=final_clip), latent=sampled_latent, steps=len_sigmas, start_step=0, last_step=len_sigmas, disable_noise=False, force_full_denoise=True, - seed=image_seed+2, # Avoid artifacts + seed=image_seed, denoise=denoise, callback_function=callback, cfg=cfg_scale, @@ -513,7 +490,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height previewer_start=switch, previewer_end=steps, sigmas=sigmas, - noise=refiner_noise + extra_noise=k_noise ) target_model = final_refiner_vae @@ -522,5 +499,4 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height decoded_latent = core.decode_vae(vae=target_model, latent_image=sampled_latent, tiled=tiled) images = core.pytorch_to_numpy(decoded_latent) - sample_hijack.history_record = None return images diff --git a/modules/sample_hijack.py b/modules/sample_hijack.py index ed184dd..bf7ea09 100644 --- a/modules/sample_hijack.py +++ b/modules/sample_hijack.py @@ -11,7 +11,6 @@ from fcbh.samplers import resolve_areas_and_cond_masks, wrap_model, calculate_st current_refiner = None refiner_switch_step = -1 -history_record = None @torch.no_grad() @@ -118,9 +117,6 @@ def sample_hacked(model, noise, positive, negative, cfg, device, sampler, sigmas return def callback_wrap(step, x0, x, total_steps): - global history_record - if isinstance(history_record, list): - history_record.append((step, x0, x)) if step == refiner_switch_step and current_refiner is not None: refiner_switch() if callback is not None: diff --git a/update_log.md b/update_log.md index 1a6bfa3..ff88b3f 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,7 @@ +# 2.1.723 + +* Improve Fooocus Anime a bit by using better SD1.5 refining formulation. + # 2.1.722 * Now it is possible to translate 100% all texts in the UI. From 3cc95314d6773756af3ac7f3822a646f5368e165 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 21 Oct 2023 17:12:42 -0700 Subject: [PATCH 02/78] Fooocus GitHub Bot Commit This commit is generated by a GitHub bot of Fooocus --- .../headless/fcbh/ldm/modules/attention.py | 12 +-- .../headless/fcbh_extras/nodes_hypertile.py | 83 +++++++++++++++++++ backend/headless/nodes.py | 3 +- fooocus_version.py | 2 +- 4 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 backend/headless/fcbh_extras/nodes_hypertile.py diff --git a/backend/headless/fcbh/ldm/modules/attention.py b/backend/headless/fcbh/ldm/modules/attention.py index d00a9af..1a3b9f0 100644 --- a/backend/headless/fcbh/ldm/modules/attention.py +++ b/backend/headless/fcbh/ldm/modules/attention.py @@ -253,12 +253,14 @@ def attention_split(q, k, v, heads, mask=None): return r2 def attention_xformers(q, k, v, heads, mask=None): - b, _, _ = q.shape + b, _, dim_head = q.shape + dim_head //= heads + q, k, v = map( lambda t: t.unsqueeze(3) - .reshape(b, t.shape[1], heads, -1) + .reshape(b, -1, heads, dim_head) .permute(0, 2, 1, 3) - .reshape(b * heads, t.shape[1], -1) + .reshape(b * heads, -1, dim_head) .contiguous(), (q, k, v), ) @@ -270,9 +272,9 @@ def attention_xformers(q, k, v, heads, mask=None): raise NotImplementedError out = ( out.unsqueeze(0) - .reshape(b, heads, out.shape[1], -1) + .reshape(b, heads, -1, dim_head) .permute(0, 2, 1, 3) - .reshape(b, out.shape[1], -1) + .reshape(b, -1, heads * dim_head) ) return out diff --git a/backend/headless/fcbh_extras/nodes_hypertile.py b/backend/headless/fcbh_extras/nodes_hypertile.py new file mode 100644 index 0000000..0d7d4c9 --- /dev/null +++ b/backend/headless/fcbh_extras/nodes_hypertile.py @@ -0,0 +1,83 @@ +#Taken from: https://github.com/tfernd/HyperTile/ + +import math +from einops import rearrange +import random + +def random_divisor(value: int, min_value: int, /, max_options: int = 1, counter = 0) -> int: + min_value = min(min_value, value) + + # All big divisors of value (inclusive) + divisors = [i for i in range(min_value, value + 1) if value % i == 0] + + ns = [value // i for i in divisors[:max_options]] # has at least 1 element + + random.seed(counter) + idx = random.randint(0, len(ns) - 1) + + return ns[idx] + +class HyperTile: + @classmethod + def INPUT_TYPES(s): + return {"required": { "model": ("MODEL",), + "tile_size": ("INT", {"default": 256, "min": 1, "max": 2048}), + "swap_size": ("INT", {"default": 2, "min": 1, "max": 128}), + "max_depth": ("INT", {"default": 0, "min": 0, "max": 10}), + "scale_depth": ("BOOLEAN", {"default": False}), + }} + RETURN_TYPES = ("MODEL",) + FUNCTION = "patch" + + CATEGORY = "_for_testing" + + def patch(self, model, tile_size, swap_size, max_depth, scale_depth): + model_channels = model.model.model_config.unet_config["model_channels"] + + apply_to = set() + temp = model_channels + for x in range(max_depth + 1): + apply_to.add(temp) + temp *= 2 + + latent_tile_size = max(32, tile_size) // 8 + self.temp = None + self.counter = 1 + + def hypertile_in(q, k, v, extra_options): + if q.shape[-1] in apply_to: + shape = extra_options["original_shape"] + aspect_ratio = shape[-1] / shape[-2] + + hw = q.size(1) + h, w = round(math.sqrt(hw * aspect_ratio)), round(math.sqrt(hw / aspect_ratio)) + + factor = 2**((q.shape[-1] // model_channels) - 1) if scale_depth else 1 + nh = random_divisor(h, latent_tile_size * factor, swap_size, self.counter) + self.counter += 1 + nw = random_divisor(w, latent_tile_size * factor, swap_size, self.counter) + self.counter += 1 + + if nh * nw > 1: + q = rearrange(q, "b (nh h nw w) c -> (b nh nw) (h w) c", h=h // nh, w=w // nw, nh=nh, nw=nw) + self.temp = (nh, nw, h, w) + return q, k, v + + return q, k, v + def hypertile_out(out, extra_options): + if self.temp is not None: + nh, nw, h, w = self.temp + self.temp = None + out = rearrange(out, "(b nh nw) hw c -> b nh nw hw c", nh=nh, nw=nw) + out = rearrange(out, "b nh nw (h w) c -> b (nh h nw w) c", h=h // nh, w=w // nw) + return out + + + m = model.clone() + m.set_model_attn1_patch(hypertile_in) + m.set_model_attn1_output_patch(hypertile_out) + return (m, ) + +NODE_CLASS_MAPPINGS = { + "HyperTile": HyperTile, +} diff --git a/backend/headless/nodes.py b/backend/headless/nodes.py index 2c1a25f..b57cd82 100644 --- a/backend/headless/nodes.py +++ b/backend/headless/nodes.py @@ -1796,7 +1796,8 @@ def init_custom_nodes(): "nodes_clip_sdxl.py", "nodes_canny.py", "nodes_freelunch.py", - "nodes_custom_sampler.py" + "nodes_custom_sampler.py", + "nodes_hypertile.py", ] for node_file in extras_files: diff --git a/fooocus_version.py b/fooocus_version.py index c264cd8..e8adf2e 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.723' +version = '2.1.724' From dde311639e21e0cbe84f0c636a6b5108edf3922a Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 21 Oct 2023 17:39:53 -0700 Subject: [PATCH 03/78] Avoid artifacts --- modules/default_pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index d20a1ec..05fa31f 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -459,8 +459,8 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height print('Use base model to refine itself - this may because of developer mode.') # Fooocus' vae parameters - k_data = 1.05 - k_noise = 0.15 + k_data = 1.015 + k_noise = 0.115 k_sigmas = 1.4 sampled_latent = vae_parse(sampled_latent, k=k_data) @@ -481,7 +481,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height negative=clip_separate(negative_cond, target_model=target_model.model, target_clip=final_clip), latent=sampled_latent, steps=len_sigmas, start_step=0, last_step=len_sigmas, disable_noise=False, force_full_denoise=True, - seed=image_seed, + seed=image_seed + 1, # Avoid artifacts denoise=denoise, callback_function=callback, cfg=cfg_scale, From cb950bd2475b60da46b41d1a6deb695b937b3f7c Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 03:37:25 -0700 Subject: [PATCH 04/78] additive noise additive noise --- modules/core.py | 51 +++++++++++++++++++++++++++++++++---- modules/default_pipeline.py | 4 +-- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/modules/core.py b/modules/core.py index 8cad32a..a78aa47 100644 --- a/modules/core.py +++ b/modules/core.py @@ -213,26 +213,67 @@ def get_previewer(model): return preview_function +@torch.no_grad() +@torch.inference_mode() +def prepare_noise(latent_image, generator, noise_inds=None): + if noise_inds is None: + return torch.randn(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, + generator=generator, device="cpu") + + unique_inds, inverse = np.unique(noise_inds, return_inverse=True) + noises = [] + for i in range(unique_inds[-1] + 1): + noise = torch.randn([1] + list(latent_image.size())[1:], dtype=latent_image.dtype, layout=latent_image.layout, + generator=generator, device="cpu") + if i in unique_inds: + noises.append(noise) + noises = [noises[i] for i in inverse] + noises = torch.cat(noises, dim=0) + return noises + + +@torch.no_grad() +@torch.inference_mode() +def prepare_additive_noise(latent_image, generator, noise_inds=None): + B, C, H, W = latent_image.shape + if noise_inds is None: + return torch.rand([B, 1, H, W], dtype=latent_image.dtype, layout=latent_image.layout, + generator=generator, device="cpu") * 2.0 - 1.0 + + unique_inds, inverse = np.unique(noise_inds, return_inverse=True) + noises = [] + for i in range(unique_inds[-1] + 1): + noise = torch.rand([1, 1, H, W], dtype=latent_image.dtype, layout=latent_image.layout, + generator=generator, device="cpu") * 2.0 - 1.0 + if i in unique_inds: + noises.append(noise) + noises = [noises[i] for i in inverse] + noises = torch.cat(noises, dim=0) + return noises + + @torch.no_grad() @torch.inference_mode() def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sampler_name='dpmpp_2m_sde_gpu', scheduler='karras', denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, callback_function=None, refiner=None, refiner_switch=-1, - previewer_start=None, previewer_end=None, sigmas=None, extra_noise=0.0): + previewer_start=None, previewer_end=None, sigmas=None, extra_noise=None): if sigmas is not None: sigmas = sigmas.clone().to(fcbh.model_management.get_torch_device()) latent_image = latent["samples"] + batch_inds = latent["batch_index"] if "batch_index" in latent else None + rng = torch.manual_seed(seed) if disable_noise: noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") else: - batch_inds = latent["batch_index"] if "batch_index" in latent else None - noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds) + noise = prepare_noise(latent_image, rng, batch_inds) - if extra_noise > 0.0: - noise = noise * (1.0 + extra_noise) + if isinstance(extra_noise, float): + additive_noise = prepare_additive_noise(latent_image, rng, batch_inds) + noise = noise + additive_noise * extra_noise noise_mask = None if "noise_mask" in latent: diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 05fa31f..813edf1 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -459,8 +459,8 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height print('Use base model to refine itself - this may because of developer mode.') # Fooocus' vae parameters - k_data = 1.015 - k_noise = 0.115 + k_data = 1.025 + k_noise = 0.25 k_sigmas = 1.4 sampled_latent = vae_parse(sampled_latent, k=k_data) From 3acf1d64947766953ea45074c8aba5f3ea005b49 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 04:04:57 -0700 Subject: [PATCH 05/78] Fooocus GitHub Bot Commit This commit is generated by a GitHub bot of Fooocus --- backend/headless/fcbh/controlnet.py | 2 +- .../headless/fcbh/ldm/modules/attention.py | 62 ++++++++++++++----- fooocus_version.py | 2 +- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/backend/headless/fcbh/controlnet.py b/backend/headless/fcbh/controlnet.py index a085839..dcdd0c1 100644 --- a/backend/headless/fcbh/controlnet.py +++ b/backend/headless/fcbh/controlnet.py @@ -416,7 +416,7 @@ class T2IAdapter(ControlBase): if control_prev is not None: return control_prev else: - return {} + return None if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]: if self.cond_hint is not None: diff --git a/backend/headless/fcbh/ldm/modules/attention.py b/backend/headless/fcbh/ldm/modules/attention.py index 1a3b9f0..a0af385 100644 --- a/backend/headless/fcbh/ldm/modules/attention.py +++ b/backend/headless/fcbh/ldm/modules/attention.py @@ -95,9 +95,19 @@ def Normalize(in_channels, dtype=None, device=None): return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device) def attention_basic(q, k, v, heads, mask=None): + b, _, dim_head = q.shape + dim_head //= heads + scale = dim_head ** -0.5 + h = heads - scale = (q.shape[-1] // heads) ** -0.5 - q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v)) + q, k, v = map( + lambda t: t.unsqueeze(3) + .reshape(b, -1, heads, dim_head) + .permute(0, 2, 1, 3) + .reshape(b * heads, -1, dim_head) + .contiguous(), + (q, k, v), + ) # force cast to fp32 to avoid overflowing if _ATTN_PRECISION =="fp32": @@ -119,16 +129,24 @@ def attention_basic(q, k, v, heads, mask=None): sim = sim.softmax(dim=-1) out = einsum('b i j, b j d -> b i d', sim.to(v.dtype), v) - out = rearrange(out, '(b h) n d -> b n (h d)', h=h) + out = ( + out.unsqueeze(0) + .reshape(b, heads, -1, dim_head) + .permute(0, 2, 1, 3) + .reshape(b, -1, heads * dim_head) + ) return out def attention_sub_quad(query, key, value, heads, mask=None): - scale = (query.shape[-1] // heads) ** -0.5 - query = query.unflatten(-1, (heads, -1)).transpose(1,2).flatten(end_dim=1) - key_t = key.transpose(1,2).unflatten(1, (heads, -1)).flatten(end_dim=1) - del key - value = value.unflatten(-1, (heads, -1)).transpose(1,2).flatten(end_dim=1) + b, _, dim_head = query.shape + dim_head //= heads + + scale = dim_head ** -0.5 + query = query.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head) + value = value.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 1, 3).reshape(b * heads, -1, dim_head) + + key = key.unsqueeze(3).reshape(b, -1, heads, dim_head).permute(0, 2, 3, 1).reshape(b * heads, dim_head, -1) dtype = query.dtype upcast_attention = _ATTN_PRECISION =="fp32" and query.dtype != torch.float32 @@ -137,7 +155,7 @@ def attention_sub_quad(query, key, value, heads, mask=None): else: bytes_per_token = torch.finfo(query.dtype).bits//8 batch_x_heads, q_tokens, _ = query.shape - _, _, k_tokens = key_t.shape + _, _, k_tokens = key.shape qk_matmul_size_bytes = batch_x_heads * bytes_per_token * q_tokens * k_tokens mem_free_total, mem_free_torch = model_management.get_free_memory(query.device, True) @@ -171,7 +189,7 @@ def attention_sub_quad(query, key, value, heads, mask=None): hidden_states = efficient_dot_product_attention( query, - key_t, + key, value, query_chunk_size=query_chunk_size, kv_chunk_size=kv_chunk_size, @@ -186,9 +204,19 @@ def attention_sub_quad(query, key, value, heads, mask=None): return hidden_states def attention_split(q, k, v, heads, mask=None): - scale = (q.shape[-1] // heads) ** -0.5 + b, _, dim_head = q.shape + dim_head //= heads + scale = dim_head ** -0.5 + h = heads - q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v)) + q, k, v = map( + lambda t: t.unsqueeze(3) + .reshape(b, -1, heads, dim_head) + .permute(0, 2, 1, 3) + .reshape(b * heads, -1, dim_head) + .contiguous(), + (q, k, v), + ) r1 = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype) @@ -248,9 +276,13 @@ def attention_split(q, k, v, heads, mask=None): del q, k, v - r2 = rearrange(r1, '(b h) n d -> b n (h d)', h=h) - del r1 - return r2 + r1 = ( + r1.unsqueeze(0) + .reshape(b, heads, -1, dim_head) + .permute(0, 2, 1, 3) + .reshape(b, -1, heads * dim_head) + ) + return r1 def attention_xformers(q, k, v, heads, mask=None): b, _, dim_head = q.shape diff --git a/fooocus_version.py b/fooocus_version.py index e8adf2e..47dcbe7 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.724' +version = '2.1.725' From 7d81eeed7e1584699d94be64b87b53b8906d6a0c Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 06:02:35 -0700 Subject: [PATCH 06/78] revise noise formulation revise noise formulation --- fooocus_version.py | 2 +- modules/core.py | 57 +++++-------------------------------- modules/default_pipeline.py | 28 +++++++++--------- modules/patch.py | 5 +++- 4 files changed, 26 insertions(+), 66 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 47dcbe7..1b54464 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.725' +version = '2.1.726' diff --git a/modules/core.py b/modules/core.py index a78aa47..c58b0fa 100644 --- a/modules/core.py +++ b/modules/core.py @@ -213,67 +213,24 @@ def get_previewer(model): return preview_function -@torch.no_grad() -@torch.inference_mode() -def prepare_noise(latent_image, generator, noise_inds=None): - if noise_inds is None: - return torch.randn(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, - generator=generator, device="cpu") - - unique_inds, inverse = np.unique(noise_inds, return_inverse=True) - noises = [] - for i in range(unique_inds[-1] + 1): - noise = torch.randn([1] + list(latent_image.size())[1:], dtype=latent_image.dtype, layout=latent_image.layout, - generator=generator, device="cpu") - if i in unique_inds: - noises.append(noise) - noises = [noises[i] for i in inverse] - noises = torch.cat(noises, dim=0) - return noises - - -@torch.no_grad() -@torch.inference_mode() -def prepare_additive_noise(latent_image, generator, noise_inds=None): - B, C, H, W = latent_image.shape - if noise_inds is None: - return torch.rand([B, 1, H, W], dtype=latent_image.dtype, layout=latent_image.layout, - generator=generator, device="cpu") * 2.0 - 1.0 - - unique_inds, inverse = np.unique(noise_inds, return_inverse=True) - noises = [] - for i in range(unique_inds[-1] + 1): - noise = torch.rand([1, 1, H, W], dtype=latent_image.dtype, layout=latent_image.layout, - generator=generator, device="cpu") * 2.0 - 1.0 - if i in unique_inds: - noises.append(noise) - noises = [noises[i] for i in inverse] - noises = torch.cat(noises, dim=0) - return noises - - @torch.no_grad() @torch.inference_mode() def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sampler_name='dpmpp_2m_sde_gpu', scheduler='karras', denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, callback_function=None, refiner=None, refiner_switch=-1, - previewer_start=None, previewer_end=None, sigmas=None, extra_noise=None): + previewer_start=None, previewer_end=None, sigmas=None, noise=None): if sigmas is not None: sigmas = sigmas.clone().to(fcbh.model_management.get_torch_device()) latent_image = latent["samples"] - batch_inds = latent["batch_index"] if "batch_index" in latent else None - rng = torch.manual_seed(seed) - if disable_noise: - noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") - else: - noise = prepare_noise(latent_image, rng, batch_inds) - - if isinstance(extra_noise, float): - additive_noise = prepare_additive_noise(latent_image, rng, batch_inds) - noise = noise + additive_noise * extra_noise + if noise is None: + if disable_noise: + noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") + else: + batch_inds = latent["batch_index"] if "batch_index" in latent else None + noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds) noise_mask = None if "noise_mask" in latent: diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 813edf1..d233bdb 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -270,13 +270,11 @@ refresh_everything( @torch.no_grad() @torch.inference_mode() -def vae_parse(latent, k=1.0): +def vae_parse(latent): if final_refiner_vae is None: - result = latent["samples"] - else: - result = vae_interpose.parse(latent["samples"]) - if k != 1.0: - result = result * k + return latent + + result = vae_interpose.parse(latent["samples"]) return {'samples': result} @@ -433,6 +431,8 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height decoded_latent = core.decode_vae(vae=target_model, latent_image=sampled_latent, tiled=tiled) if refiner_swap_method == 'vae': + modules.patch.eps_record = 'vae' + if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.unswap() @@ -458,13 +458,9 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height target_model = final_unet print('Use base model to refine itself - this may because of developer mode.') - # Fooocus' vae parameters - k_data = 1.025 - k_noise = 0.25 + sampled_latent = vae_parse(sampled_latent) + k_sigmas = 1.4 - - sampled_latent = vae_parse(sampled_latent, k=k_data) - sigmas = calculate_sigmas(sampler=sampler_name, scheduler=scheduler_name, model=target_model.model, @@ -472,6 +468,9 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height denoise=denoise)[switch:] * k_sigmas len_sigmas = len(sigmas) - 1 + assert isinstance(modules.patch.eps_record, torch.Tensor) + residual_noise = modules.patch.eps_record / modules.patch.eps_record.std() + if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.swap() @@ -481,7 +480,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height negative=clip_separate(negative_cond, target_model=target_model.model, target_clip=final_clip), latent=sampled_latent, steps=len_sigmas, start_step=0, last_step=len_sigmas, disable_noise=False, force_full_denoise=True, - seed=image_seed + 1, # Avoid artifacts + seed=image_seed, denoise=denoise, callback_function=callback, cfg=cfg_scale, @@ -490,7 +489,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height previewer_start=switch, previewer_end=steps, sigmas=sigmas, - extra_noise=k_noise + noise=residual_noise ) target_model = final_refiner_vae @@ -499,4 +498,5 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height decoded_latent = core.decode_vae(vae=target_model, latent_image=sampled_latent, tiled=tiled) images = core.pytorch_to_numpy(decoded_latent) + modules.patch.eps_record = None return images diff --git a/modules/patch.py b/modules/patch.py index 7b6a38c..4a590cd 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -38,6 +38,7 @@ cfg_x0 = 0.0 cfg_s = 1.0 cfg_cin = 1.0 adaptive_cfg = 0.7 +eps_record = None def calculate_weight_patched(self, patches, weight, key): @@ -192,10 +193,12 @@ def patched_sampler_cfg_function(args): def patched_discrete_eps_ddpm_denoiser_forward(self, input, sigma, **kwargs): - global cfg_x0, cfg_s, cfg_cin + global cfg_x0, cfg_s, cfg_cin, eps_record c_out, c_in = [utils.append_dims(x, input.ndim) for x in self.get_scalings(sigma)] cfg_x0, cfg_s, cfg_cin = input, c_out, c_in eps = self.get_eps(input * c_in, self.sigma_to_t(sigma), **kwargs) + if eps_record is not None: + eps_record = eps.clone().cpu() return input + eps * c_out From 566cf087508a4eb888086bbe551482723ac05e40 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 06:32:02 -0700 Subject: [PATCH 07/78] fix --- fooocus_version.py | 2 +- modules/default_pipeline.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 1b54464..0ed06a4 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.726' +version = '2.1.727' diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index d233bdb..95865ad 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -468,8 +468,8 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height denoise=denoise)[switch:] * k_sigmas len_sigmas = len(sigmas) - 1 - assert isinstance(modules.patch.eps_record, torch.Tensor) - residual_noise = modules.patch.eps_record / modules.patch.eps_record.std() + residual_noise = modules.patch.eps_record + assert isinstance(residual_noise, torch.Tensor) if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.swap() From 22fc28958ffdc0988f031afbc28b6286281ffefc Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 06:53:34 -0700 Subject: [PATCH 08/78] colab --- fooocus_colab.ipynb | 4 ++-- readme.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fooocus_colab.ipynb b/fooocus_colab.ipynb index c3497c6..469573c 100644 --- a/fooocus_colab.ipynb +++ b/fooocus_colab.ipynb @@ -10,10 +10,10 @@ "source": [ "!pip install pygit2==1.12.2\n", "%cd /content\n", - "!git clone https://github.com/lllyasviel/Fooocus\n", + "!git clone https://github.com/lllyasviel/Fooocus.git\n", "%cd /content/Fooocus\n", "!cp colab_fix.txt user_path_config.txt\n", - "!python entry_with_update.py --share\n" + "!python entry_with_update.py --preset realistic --share\n" ] } ], diff --git a/readme.md b/readme.md index 06465ed..a0b27de 100644 --- a/readme.md +++ b/readme.md @@ -106,7 +106,7 @@ Please open an issue if you use similar devices but still cannot achieve accepta | --- | --- | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/lllyasviel/Fooocus/blob/main/fooocus_colab.ipynb) | Fooocus Official -In Colab, you can modify the last line to `!python entry_with_update.py --preset anime --share` or `!python entry_with_update.py --preset realistic --share` for Fooocus Anime/Realistic Edition. +In Colab, you can modify the last line to `!python entry_with_update.py --share` or `!python entry_with_update.py --preset anime --share` or `!python entry_with_update.py --preset realistic --share` for Fooocus Default/Anime/Realistic Edition. Note that this Colab will disable refiner by default because Colab free's resource is relatively limited. From 4cf0c778da33fb91d591771bbe90123a45c54d3c Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 07:30:30 -0700 Subject: [PATCH 09/78] fix potential numerical problems --- fooocus_version.py | 2 +- modules/default_pipeline.py | 17 ++++++++++++++--- update_log.md | 4 ++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 0ed06a4..adf222a 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.727' +version = '2.1.728' diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 95865ad..b30f32a 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -3,6 +3,7 @@ import os import torch import modules.patch import modules.path +import fcbh.sample import fcbh.model_management import fcbh.latent_formats import modules.inpaint_worker @@ -278,6 +279,14 @@ def vae_parse(latent): return {'samples': result} +@torch.no_grad() +@torch.inference_mode() +def noise_parse(latent: torch.Tensor, seed: int, noise_inds=None): + noise = fcbh.sample.prepare_noise(latent, seed=seed, noise_inds=noise_inds) + s, m = torch.std_mean(latent, dim=1, keepdim=True) + return m + s * noise + + @torch.no_grad() @torch.inference_mode() def calculate_sigmas_all(sampler, model, scheduler, steps): @@ -468,8 +477,10 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height denoise=denoise)[switch:] * k_sigmas len_sigmas = len(sigmas) - 1 - residual_noise = modules.patch.eps_record - assert isinstance(residual_noise, torch.Tensor) + residual_noise = noise_parse( + modules.patch.eps_record, + seed=image_seed+1, + noise_inds=sampled_latent["batch_index"] if "batch_index" in sampled_latent else None) if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.swap() @@ -480,7 +491,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height negative=clip_separate(negative_cond, target_model=target_model.model, target_clip=final_clip), latent=sampled_latent, steps=len_sigmas, start_step=0, last_step=len_sigmas, disable_noise=False, force_full_denoise=True, - seed=image_seed, + seed=image_seed+1, denoise=denoise, callback_function=callback, cfg=cfg_scale, diff --git a/update_log.md b/update_log.md index ff88b3f..c122d66 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,7 @@ +# 2.1.728 + +* Fixed some potential numerical problems since 2.1.723 + # 2.1.723 * Improve Fooocus Anime a bit by using better SD1.5 refining formulation. From 4c94f0f590b17830102d7eda6116a62f39497ac8 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 08:08:36 -0700 Subject: [PATCH 10/78] reproduce previous results --- fooocus_version.py | 2 +- modules/default_pipeline.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index adf222a..175d3fe 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.728' +version = '2.1.729' diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index b30f32a..fe2754b 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -281,10 +281,10 @@ def vae_parse(latent): @torch.no_grad() @torch.inference_mode() -def noise_parse(latent: torch.Tensor, seed: int, noise_inds=None): +def noise_parse(latent: torch.Tensor, seed: int, noise_inds=None, k=0.9): noise = fcbh.sample.prepare_noise(latent, seed=seed, noise_inds=noise_inds) - s, m = torch.std_mean(latent, dim=1, keepdim=True) - return m + s * noise + offset = torch.mean(latent, dim=1, keepdim=True) + return offset * k + noise @torch.no_grad() From 576aa9283f492255ce86f48e056d2c9f8c17e0be Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 08:28:25 -0700 Subject: [PATCH 11/78] ling --- fooocus_version.py | 2 +- modules/core.py | 16 +++++++++------- modules/default_pipeline.py | 16 ++-------------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 175d3fe..5637cda 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.729' +version = '2.1.730' diff --git a/modules/core.py b/modules/core.py index c58b0fa..3000f0a 100644 --- a/modules/core.py +++ b/modules/core.py @@ -218,19 +218,21 @@ def get_previewer(model): def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sampler_name='dpmpp_2m_sde_gpu', scheduler='karras', denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, callback_function=None, refiner=None, refiner_switch=-1, - previewer_start=None, previewer_end=None, sigmas=None, noise=None): + previewer_start=None, previewer_end=None, sigmas=None, noise_offset=None): if sigmas is not None: sigmas = sigmas.clone().to(fcbh.model_management.get_torch_device()) latent_image = latent["samples"] - if noise is None: - if disable_noise: - noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") - else: - batch_inds = latent["batch_index"] if "batch_index" in latent else None - noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds) + if disable_noise: + noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu") + else: + batch_inds = latent["batch_index"] if "batch_index" in latent else None + noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds) + + if isinstance(noise_offset, torch.Tensor): + noise = noise + noise_offset noise_mask = None if "noise_mask" in latent: diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index fe2754b..a1e19e3 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -3,7 +3,6 @@ import os import torch import modules.patch import modules.path -import fcbh.sample import fcbh.model_management import fcbh.latent_formats import modules.inpaint_worker @@ -279,14 +278,6 @@ def vae_parse(latent): return {'samples': result} -@torch.no_grad() -@torch.inference_mode() -def noise_parse(latent: torch.Tensor, seed: int, noise_inds=None, k=0.9): - noise = fcbh.sample.prepare_noise(latent, seed=seed, noise_inds=noise_inds) - offset = torch.mean(latent, dim=1, keepdim=True) - return offset * k + noise - - @torch.no_grad() @torch.inference_mode() def calculate_sigmas_all(sampler, model, scheduler, steps): @@ -477,10 +468,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height denoise=denoise)[switch:] * k_sigmas len_sigmas = len(sigmas) - 1 - residual_noise = noise_parse( - modules.patch.eps_record, - seed=image_seed+1, - noise_inds=sampled_latent["batch_index"] if "batch_index" in sampled_latent else None) + noise_offset = torch.mean(modules.patch.eps_record, dim=1, keepdim=True) * 0.9 if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.swap() @@ -500,7 +488,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height previewer_start=switch, previewer_end=steps, sigmas=sigmas, - noise=residual_noise + noise_offset=noise_offset ) target_model = final_refiner_vae From 6f3fa4b670f8cbffe1c574ea183d97f5f0603942 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 09:00:16 -0700 Subject: [PATCH 12/78] remove unused codes --- modules/patch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/patch.py b/modules/patch.py index 4a590cd..77958ae 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -285,8 +285,6 @@ globalBrownianTreeNoiseSampler = None @torch.no_grad() def sample_dpmpp_fooocus_2m_sde_inpaint_seamless(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., **kwargs): - global sigma_min, sigma_max - print('[Sampler] Fooocus sampler is activated.') seed = extra_args.get("seed", None) From 43bc2df32884b21ef9e34f7a148a45cbbe2fe152 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 12:09:24 -0700 Subject: [PATCH 13/78] revise math --- fooocus_version.py | 2 +- modules/core.py | 6 +++--- modules/default_pipeline.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 5637cda..08913c7 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.730' +version = '2.1.731' diff --git a/modules/core.py b/modules/core.py index 3000f0a..60ebdf6 100644 --- a/modules/core.py +++ b/modules/core.py @@ -218,7 +218,7 @@ def get_previewer(model): def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sampler_name='dpmpp_2m_sde_gpu', scheduler='karras', denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, callback_function=None, refiner=None, refiner_switch=-1, - previewer_start=None, previewer_end=None, sigmas=None, noise_offset=None): + previewer_start=None, previewer_end=None, sigmas=None, noise_mean=None): if sigmas is not None: sigmas = sigmas.clone().to(fcbh.model_management.get_torch_device()) @@ -231,8 +231,8 @@ def ksampler(model, positive, negative, latent, seed=None, steps=30, cfg=7.0, sa batch_inds = latent["batch_index"] if "batch_index" in latent else None noise = fcbh.sample.prepare_noise(latent_image, seed, batch_inds) - if isinstance(noise_offset, torch.Tensor): - noise = noise + noise_offset + if isinstance(noise_mean, torch.Tensor): + noise = noise + noise_mean - torch.mean(noise, dim=1, keepdim=True) noise_mask = None if "noise_mask" in latent: diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index a1e19e3..8557ac2 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -468,7 +468,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height denoise=denoise)[switch:] * k_sigmas len_sigmas = len(sigmas) - 1 - noise_offset = torch.mean(modules.patch.eps_record, dim=1, keepdim=True) * 0.9 + noise_mean = torch.mean(modules.patch.eps_record, dim=1, keepdim=True) if modules.inpaint_worker.current_task is not None: modules.inpaint_worker.current_task.swap() @@ -488,7 +488,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height previewer_start=switch, previewer_end=steps, sigmas=sigmas, - noise_offset=noise_offset + noise_mean=noise_mean ) target_model = final_refiner_vae From 23a7559db84865a9d448b60fdfb16dc8858d9395 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 22 Oct 2023 12:16:36 -0700 Subject: [PATCH 14/78] Fooocus GitHub Bot Commit This commit is generated by a GitHub bot of Fooocus --- backend/headless/fcbh/model_management.py | 6 +++++- fooocus_version.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/headless/fcbh/model_management.py b/backend/headless/fcbh/model_management.py index d6cee6c..75108ee 100644 --- a/backend/headless/fcbh/model_management.py +++ b/backend/headless/fcbh/model_management.py @@ -339,7 +339,11 @@ def free_memory(memory_required, device, keep_loaded=[]): if unloaded_model: soft_empty_cache() - + else: + if vram_state != VRAMState.HIGH_VRAM: + mem_free_total, mem_free_torch = get_free_memory(device, torch_free_too=True) + if mem_free_torch > mem_free_total * 0.25: + soft_empty_cache() def load_models_gpu(models, memory_required=0): global vram_state diff --git a/fooocus_version.py b/fooocus_version.py index 08913c7..024a3d1 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.731' +version = '2.1.732' From 81650a4305048b1c01d87bac2ef63e07ca647843 Mon Sep 17 00:00:00 2001 From: MoonRide303 Date: Mon, 23 Oct 2023 12:20:14 +0200 Subject: [PATCH 15/78] Increased allowed random seed range --- fooocus_version.py | 2 +- modules/async_worker.py | 9 ++------- modules/expansion.py | 5 ++++- update_log.md | 4 ++++ webui.py | 18 ++++++++++++++---- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 024a3d1..383afae 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.732' +version = '2.1.733' diff --git a/modules/async_worker.py b/modules/async_worker.py index b7c3c18..2509d76 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -123,13 +123,8 @@ def worker(): controlnet_cpds_path = None clip_vision_path, ip_negative_path, ip_adapter_path = None, None, None - seed = image_seed - max_seed = int(1024 * 1024 * 1024) - if not isinstance(seed, int): - seed = random.randint(1, max_seed) - if seed < 0: - seed = - seed - seed = seed % max_seed + seed = int(image_seed) + print(f'[Parameters] Seed = {seed}') if performance_selection == 'Speed': steps = 30 diff --git a/modules/expansion.py b/modules/expansion.py index 2145a70..f65089f 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -6,6 +6,9 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed from modules.path import fooocus_expansion_path from fcbh.model_patcher import ModelPatcher +# limitation of np.random.seed(), called from transformers.set_seed() +SEED_LIMIT_NUMPY = 2**32 + fooocus_magic_split = [ ', extremely', @@ -54,7 +57,7 @@ class FooocusExpansion: print('Fooocus Expansion loaded by itself.') model_management.load_model_gpu(self.patcher) - seed = int(seed) + seed = int(seed) % SEED_LIMIT_NUMPY set_seed(seed) origin = safe_str(prompt) prompt = origin + fooocus_magic_split[seed % len(fooocus_magic_split)] diff --git a/update_log.md b/update_log.md index c122d66..9d33ba2 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,7 @@ +# 2.1.733 + +* Increased allowed random seed range. + # 2.1.728 * Fixed some potential numerical problems since 2.1.723 diff --git a/webui.py b/webui.py index c8c6bc4..a6829ae 100644 --- a/webui.py +++ b/webui.py @@ -18,6 +18,10 @@ from modules.sdxl_styles import legal_style_names, aspect_ratios from modules.private_logger import get_current_html_path from modules.ui_gradio_extensions import reload_javascript +# as in k-diffusion (sampling.py) +MIN_SEED = 0 +MAX_SEED = 2**63 - 1 + def generate_clicked(*args): execution_start_time = time.perf_counter() @@ -193,16 +197,22 @@ with shared.gradio_root: info='Describing what you do not want to see.', lines=2, value=modules.path.default_negative_prompt) seed_random = gr.Checkbox(label='Random', value=True) - image_seed = gr.Number(label='Seed', value=0, precision=0, visible=False) + image_seed = gr.Textbox(label='Seed', value=0, max_lines=1, visible=False) # workaround for https://github.com/gradio-app/gradio/issues/5354 def random_checked(r): return gr.update(visible=not r) - def refresh_seed(r, s): + def refresh_seed(r, seed_string): if r: - return random.randint(1, 1024*1024*1024) + return random.randint(MIN_SEED, MAX_SEED) else: - return s + try: + seed_value = int(seed_string) + if MIN_SEED <= seed_value <= MAX_SEED: + return seed_value + except ValueError: + pass + return random.randint(MIN_SEED, MAX_SEED) seed_random.change(random_checked, inputs=[seed_random], outputs=[image_seed], queue=False) From 1272d50d355fa380522274685fff8f2b74be7755 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 06:40:18 -0700 Subject: [PATCH 16/78] fix math --- fooocus_version.py | 2 +- modules/util.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 383afae..e634d37 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.733' +version = '2.1.734' diff --git a/modules/util.py b/modules/util.py index 2bd46fd..c5e8d1b 100644 --- a/modules/util.py +++ b/modules/util.py @@ -84,11 +84,22 @@ def get_image_shape_ceil(im): def set_image_shape_ceil(im, shape_ceil): - H, W, _ = im.shape - shape_ceil_before = get_shape_ceil(H, W) - k = float(shape_ceil) / shape_ceil_before - H = int(round(float(H) * k / 64.0) * 64) - W = int(round(float(W) * k / 64.0) * 64) + shape_ceil = float(shape_ceil) + + H_origin, W_origin, _ = im.shape + H, W = H_origin, W_origin + + for _ in range(256): + current_shape_ceil = get_shape_ceil(H, W) + if abs(current_shape_ceil - shape_ceil) < 0.1: + break + k = shape_ceil / current_shape_ceil + H = int(round(float(H) * k / 64.0) * 64) + W = int(round(float(W) * k / 64.0) * 64) + + if H == H_origin and W == W_origin: + return im + return resample_image(im, width=W, height=H) From 47281e52c42dcf1e3b30c58201ba199e7f523bde Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 13:07:55 -0700 Subject: [PATCH 17/78] Fixed many autocast problems. --- fooocus_version.py | 2 +- modules/patch.py | 49 +++++++++++++++++++++++++++++++++++++--------- update_log.md | 4 ++++ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index e634d37..571dc63 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.734' +version = '2.1.735' diff --git a/modules/patch.py b/modules/patch.py index 77958ae..f550ed8 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -1,3 +1,4 @@ +import contextlib import os import torch import time @@ -463,16 +464,36 @@ def text_encoder_device_patched(): return fcbh.model_management.get_torch_device() -def patched_get_autocast_device(dev): +def patched_autocast(device_type, dtype=None, enabled=True, cache_enabled=None): # https://github.com/lllyasviel/Fooocus/discussions/571 # https://github.com/lllyasviel/Fooocus/issues/620 - result = '' - if hasattr(dev, 'type'): - result = str(dev.type) - if 'cuda' in result: - return 'cuda' - else: - return 'cpu' + # https://github.com/lllyasviel/Fooocus/issues/759 + + supported = False + + if device_type == 'cuda' and dtype == torch.float32 and enabled: + supported = True + + if device_type == 'cuda' and dtype == torch.float16 and enabled: + supported = True + + if device_type == 'cuda' and dtype == torch.bfloat16 and enabled: + supported = True + + if not supported: + print(f'[Fooocus Autocast Warning] Requested unsupported torch autocast [' + f'device_type={str(device_type)}, ' + f'dtype={str(dtype)}, ' + f'enabled={str(enabled)}, ' + f'cache_enabled={str(cache_enabled)}]. ' + f'Fooocus fixed it automatically, feel free to report to Fooocus on GitHub if this may cause potential problems.') + return contextlib.nullcontext() + + return torch.amp.autocast_mode.autocast_origin( + device_type=device_type, + dtype=dtype, + enabled=enabled, + cache_enabled=cache_enabled) def patched_load_models_gpu(*args, **kwargs): @@ -535,8 +556,18 @@ def patch_all(): if not hasattr(fcbh.model_management, 'load_models_gpu_origin'): fcbh.model_management.load_models_gpu_origin = fcbh.model_management.load_models_gpu + if not hasattr(torch.amp.autocast_mode, 'autocast_origin'): + torch.amp.autocast_mode.autocast_origin = torch.amp.autocast_mode.autocast + + torch.amp.autocast_mode.autocast = patched_autocast + torch.amp.autocast = patched_autocast + torch.autocast = patched_autocast + + # # Test if this will fail + # with torch.autocast(device_type='cpu', dtype=torch.float32): + # print(torch.ones(10)) + fcbh.model_management.load_models_gpu = patched_load_models_gpu - fcbh.model_management.get_autocast_device = patched_get_autocast_device fcbh.model_management.text_encoder_device = text_encoder_device_patched fcbh.model_patcher.ModelPatcher.calculate_weight = calculate_weight_patched fcbh.cldm.cldm.ControlNet.forward = patched_cldm_forward diff --git a/update_log.md b/update_log.md index 9d33ba2..e913c12 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,7 @@ +# 2.1.735 + +* Fixed many problems related to torch autocast. + # 2.1.733 * Increased allowed random seed range. From 7a6775acdcee7deb6059e46a348be297c9c8c1de Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 14:15:20 -0700 Subject: [PATCH 18/78] fix autocast in less aggressive way --- fooocus_version.py | 2 +- modules/patch.py | 41 +++++++++++------------------------------ 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 571dc63..a70e273 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.735' +version = '2.1.736' diff --git a/modules/patch.py b/modules/patch.py index f550ed8..f4783bb 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -464,36 +464,19 @@ def text_encoder_device_patched(): return fcbh.model_management.get_torch_device() -def patched_autocast(device_type, dtype=None, enabled=True, cache_enabled=None): +def patched_autocast_enter(self): # https://github.com/lllyasviel/Fooocus/discussions/571 # https://github.com/lllyasviel/Fooocus/issues/620 # https://github.com/lllyasviel/Fooocus/issues/759 - supported = False - - if device_type == 'cuda' and dtype == torch.float32 and enabled: - supported = True - - if device_type == 'cuda' and dtype == torch.float16 and enabled: - supported = True - - if device_type == 'cuda' and dtype == torch.bfloat16 and enabled: - supported = True - - if not supported: - print(f'[Fooocus Autocast Warning] Requested unsupported torch autocast [' - f'device_type={str(device_type)}, ' - f'dtype={str(dtype)}, ' - f'enabled={str(enabled)}, ' - f'cache_enabled={str(cache_enabled)}]. ' + try: + result = self.enter_origin() + except Exception as e: + result = self + print(f'[Fooocus Autocast Warning] {str(e)}. \n' f'Fooocus fixed it automatically, feel free to report to Fooocus on GitHub if this may cause potential problems.') - return contextlib.nullcontext() - return torch.amp.autocast_mode.autocast_origin( - device_type=device_type, - dtype=dtype, - enabled=enabled, - cache_enabled=cache_enabled) + return result def patched_load_models_gpu(*args, **kwargs): @@ -556,14 +539,12 @@ def patch_all(): if not hasattr(fcbh.model_management, 'load_models_gpu_origin'): fcbh.model_management.load_models_gpu_origin = fcbh.model_management.load_models_gpu - if not hasattr(torch.amp.autocast_mode, 'autocast_origin'): - torch.amp.autocast_mode.autocast_origin = torch.amp.autocast_mode.autocast + if not hasattr(torch.amp.autocast_mode.autocast, 'enter_origin'): + torch.amp.autocast_mode.autocast.enter_origin = torch.amp.autocast_mode.autocast.__enter__ - torch.amp.autocast_mode.autocast = patched_autocast - torch.amp.autocast = patched_autocast - torch.autocast = patched_autocast + torch.amp.autocast_mode.autocast.__enter__ = patched_autocast_enter - # # Test if this will fail + # # Test if this would fail # with torch.autocast(device_type='cpu', dtype=torch.float32): # print(torch.ones(10)) From 0b90fd9e8e9be13afd723c0f06dbefd4ed79a77a Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 14:22:35 -0700 Subject: [PATCH 19/78] Revert "fix autocast in less aggressive way" This reverts commit 7a6775acdcee7deb6059e46a348be297c9c8c1de. --- fooocus_version.py | 2 +- modules/patch.py | 43 +++++++++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index a70e273..571dc63 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.736' +version = '2.1.735' diff --git a/modules/patch.py b/modules/patch.py index f4783bb..f550ed8 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -464,19 +464,36 @@ def text_encoder_device_patched(): return fcbh.model_management.get_torch_device() -def patched_autocast_enter(self): +def patched_autocast(device_type, dtype=None, enabled=True, cache_enabled=None): # https://github.com/lllyasviel/Fooocus/discussions/571 # https://github.com/lllyasviel/Fooocus/issues/620 # https://github.com/lllyasviel/Fooocus/issues/759 - try: - result = self.enter_origin() - except Exception as e: - result = self - print(f'[Fooocus Autocast Warning] {str(e)}. \n' - f'Fooocus fixed it automatically, feel free to report to Fooocus on GitHub if this may cause potential problems.') + supported = False - return result + if device_type == 'cuda' and dtype == torch.float32 and enabled: + supported = True + + if device_type == 'cuda' and dtype == torch.float16 and enabled: + supported = True + + if device_type == 'cuda' and dtype == torch.bfloat16 and enabled: + supported = True + + if not supported: + print(f'[Fooocus Autocast Warning] Requested unsupported torch autocast [' + f'device_type={str(device_type)}, ' + f'dtype={str(dtype)}, ' + f'enabled={str(enabled)}, ' + f'cache_enabled={str(cache_enabled)}]. ' + f'Fooocus fixed it automatically, feel free to report to Fooocus on GitHub if this may cause potential problems.') + return contextlib.nullcontext() + + return torch.amp.autocast_mode.autocast_origin( + device_type=device_type, + dtype=dtype, + enabled=enabled, + cache_enabled=cache_enabled) def patched_load_models_gpu(*args, **kwargs): @@ -539,12 +556,14 @@ def patch_all(): if not hasattr(fcbh.model_management, 'load_models_gpu_origin'): fcbh.model_management.load_models_gpu_origin = fcbh.model_management.load_models_gpu - if not hasattr(torch.amp.autocast_mode.autocast, 'enter_origin'): - torch.amp.autocast_mode.autocast.enter_origin = torch.amp.autocast_mode.autocast.__enter__ + if not hasattr(torch.amp.autocast_mode, 'autocast_origin'): + torch.amp.autocast_mode.autocast_origin = torch.amp.autocast_mode.autocast - torch.amp.autocast_mode.autocast.__enter__ = patched_autocast_enter + torch.amp.autocast_mode.autocast = patched_autocast + torch.amp.autocast = patched_autocast + torch.autocast = patched_autocast - # # Test if this would fail + # # Test if this will fail # with torch.autocast(device_type='cpu', dtype=torch.float32): # print(torch.ones(10)) From ee2fe66811575776d259527161352d968a067c2f Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 14:26:14 -0700 Subject: [PATCH 20/78] fix --- webui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui.py b/webui.py index a6829ae..666fe31 100644 --- a/webui.py +++ b/webui.py @@ -20,7 +20,7 @@ from modules.ui_gradio_extensions import reload_javascript # as in k-diffusion (sampling.py) MIN_SEED = 0 -MAX_SEED = 2**63 - 1 +MAX_SEED = 2**63 - 1024 # for image number increasing safety def generate_clicked(*args): From 504e5478b0b38e247d656016dcc7f81021faf747 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 15:09:24 -0700 Subject: [PATCH 21/78] remove unused/unstable codes --- fooocus_version.py | 2 +- modules/patch.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 571dc63..a70e273 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.735' +version = '2.1.736' diff --git a/modules/patch.py b/modules/patch.py index f550ed8..9fa01f8 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -459,11 +459,6 @@ def patched_unet_forward(self, x, timesteps=None, context=None, y=None, control= return self.out(h) -def text_encoder_device_patched(): - # Fooocus's style system uses text encoder much more times than fcbh so this makes things much faster. - return fcbh.model_management.get_torch_device() - - def patched_autocast(device_type, dtype=None, enabled=True, cache_enabled=None): # https://github.com/lllyasviel/Fooocus/discussions/571 # https://github.com/lllyasviel/Fooocus/issues/620 @@ -568,7 +563,6 @@ def patch_all(): # print(torch.ones(10)) fcbh.model_management.load_models_gpu = patched_load_models_gpu - fcbh.model_management.text_encoder_device = text_encoder_device_patched fcbh.model_patcher.ModelPatcher.calculate_weight = calculate_weight_patched fcbh.cldm.cldm.ControlNet.forward = patched_cldm_forward fcbh.ldm.modules.diffusionmodules.openaimodel.UNetModel.forward = patched_unet_forward From 60cb91c4063b072c762d5450a4088b101efa0ad9 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 19:42:23 -0700 Subject: [PATCH 22/78] allow ar config --- fooocus_version.py | 2 +- modules/async_worker.py | 7 +++++-- modules/path.py | 14 +++++++++++--- modules/sdxl_styles.py | 42 ----------------------------------------- update_log.md | 18 ++++++++++++++++++ webui.py | 4 ++-- 6 files changed, 37 insertions(+), 50 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index a70e273..bb26d01 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.736' +version = '2.1.737' diff --git a/modules/async_worker.py b/modules/async_worker.py index 2509d76..48ec32c 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -26,7 +26,7 @@ def worker(): import modules.advanced_parameters as advanced_parameters import fooocus_extras.ip_adapter as ip_adapter - from modules.sdxl_styles import apply_style, apply_wildcards, aspect_ratios, fooocus_expansion + from modules.sdxl_styles import apply_style, apply_wildcards, fooocus_expansion from modules.private_logger import log from modules.expansion import safe_str from modules.util import join_prompts, remove_empty_str, HWC3, resize_image, \ @@ -112,7 +112,10 @@ def worker(): denoising_strength = 1.0 tiled = False inpaint_worker.current_task = None - width, height = aspect_ratios[aspect_ratios_selection] + + width, height = aspect_ratios_selection.split('×') + width, height = int(width), int(height) + skip_prompt_processing = False refiner_swap_method = advanced_parameters.refiner_swap_method diff --git a/modules/path.py b/modules/path.py index e23b3dc..90860ff 100644 --- a/modules/path.py +++ b/modules/path.py @@ -163,11 +163,16 @@ embeddings_downloads = get_config_item_or_set_default( default_value={}, validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()) ) +available_aspect_ratios = get_config_item_or_set_default( + key='available_aspect_ratios', + default_value=['704*1408', '704*1344', '768*1344', '768*1280', '832*1216', '832*1152', '896*1152', '896*1088', '960*1088', '960*1024', '1024*1024', '1024*960', '1088*960', '1088*896', '1152*896', '1152*832', '1216*832', '1280*768', '1344*768', '1344*704', '1408*704', '1472*704', '1536*640', '1600*640', '1664*576', '1728*576'], + validator=lambda x: isinstance(x, list) and all('*' in v for v in x) and len(x) > 1 +) default_aspect_ratio = get_config_item_or_set_default( key='default_aspect_ratio', - default_value='1152*896', - validator=lambda x: x.replace('*', '×') in modules.sdxl_styles.aspect_ratios -).replace('*', '×') + default_value='1152*896' if '1152*896' in available_aspect_ratios else available_aspect_ratios[0], + validator=lambda x: x in available_aspect_ratios +) if preset is None: # Do not overwrite user config if preset is applied. @@ -179,6 +184,9 @@ os.makedirs(temp_outputs_path, exist_ok=True) model_filenames = [] lora_filenames = [] +available_aspect_ratios = [x.replace('*', '×') for x in available_aspect_ratios] +default_aspect_ratio = default_aspect_ratio.replace('*', '×') + def get_model_filenames(folder_path, name_filter=None): return get_files_from_folder(folder_path, ['.pth', '.ckpt', '.bin', '.safetensors', '.fooocus.patch'], name_filter) diff --git a/modules/sdxl_styles.py b/modules/sdxl_styles.py index 87af512..6979aa9 100644 --- a/modules/sdxl_styles.py +++ b/modules/sdxl_styles.py @@ -51,48 +51,6 @@ fooocus_expansion = "Fooocus V2" legal_style_names = [fooocus_expansion] + style_keys -SD_XL_BASE_RATIOS = { - "0.5": (704, 1408), - "0.52": (704, 1344), - "0.57": (768, 1344), - "0.6": (768, 1280), - "0.68": (832, 1216), - "0.72": (832, 1152), - "0.78": (896, 1152), - "0.82": (896, 1088), - "0.88": (960, 1088), - "0.94": (960, 1024), - "1.0": (1024, 1024), - "1.07": (1024, 960), - "1.13": (1088, 960), - "1.21": (1088, 896), - "1.29": (1152, 896), - "1.38": (1152, 832), - "1.46": (1216, 832), - "1.67": (1280, 768), - "1.75": (1344, 768), - "1.91": (1344, 704), - "2.0": (1408, 704), - "2.09": (1472, 704), - "2.4": (1536, 640), - "2.5": (1600, 640), - "2.89": (1664, 576), - "3.0": (1728, 576), -} - -aspect_ratios = {} - -# import math - -for k, (w, h) in SD_XL_BASE_RATIOS.items(): - txt = f'{w}×{h}' - - # gcd = math.gcd(w, h) - # txt += f' {w//gcd}:{h//gcd}' - - aspect_ratios[txt] = (w, h) - - def apply_style(style, positive): p, n = styles[style] return p.replace('{prompt}', positive), n diff --git a/update_log.md b/update_log.md index e913c12..0f89296 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,21 @@ +# 2.1.737 + +* Allowed customizing resolutions in config. + +Modifying this will make results worse if you do not understand how Positional Encoding works. + +You have been warned. + +If you do not know why numbers must be transformed with many Sin and Cos functions (yes, those Trigonometric functions that you learn in junior high school) before they are fed to SDXL, we do not encourage you to change this - you will become a victim of Positional Encoding. You are likely to suffer from an easy-to-fail tool, rather than getting more control. + +Your knowledge gained from SD1.5 (for example, resolution numbers divided by 8 or 64 are good enough for UNet) does not work in SDXL. The SDXL uses Positional Encoding. The SD1.5 does not use Positional Encoding. They are completely different. + +Your knowledge gained from other resources (for example, resolutions around 1024 are good enough for SDXL) is wrong. The SDXL uses Positional Encoding. People who say "all resolutions around 1024 are good" do not understand what is Positional Encoding. They are not intentionally misleading. They are just not aware of the fact that SDXL is using Positional Encoding. + +The number 1152 must be exactly 1152, not 1152-1, not 1152+1, not 1152-8, not 1152+8. The number 1152 must be exactly 1152. Just Google what is a Positional Encoding. + +Again, if you do not understand how Positional Encoding works, just do not change the resolution numbers. + # 2.1.735 * Fixed many problems related to torch autocast. diff --git a/webui.py b/webui.py index 666fe31..e0702d8 100644 --- a/webui.py +++ b/webui.py @@ -14,7 +14,7 @@ import modules.gradio_hijack as grh import modules.advanced_parameters as advanced_parameters import args_manager -from modules.sdxl_styles import legal_style_names, aspect_ratios +from modules.sdxl_styles import legal_style_names from modules.private_logger import get_current_html_path from modules.ui_gradio_extensions import reload_javascript @@ -190,7 +190,7 @@ with shared.gradio_root: with gr.Column(scale=1, visible=modules.path.default_advanced_checkbox) as advanced_column: with gr.Tab(label='Setting'): performance_selection = gr.Radio(label='Performance', choices=['Speed', 'Quality'], value='Speed') - aspect_ratios_selection = gr.Radio(label='Aspect Ratios', choices=list(aspect_ratios.keys()), + aspect_ratios_selection = gr.Radio(label='Aspect Ratios', choices=modules.path.available_aspect_ratios, value=modules.path.default_aspect_ratio, info='width × height') image_number = gr.Slider(label='Image Number', minimum=1, maximum=32, step=1, value=modules.path.default_image_number) negative_prompt = gr.Textbox(label='Negative Prompt', show_label=True, placeholder="Type prompt here.", From 10a9f0fc9dc4db0301afc47973d1a9665f17714b Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 23 Oct 2023 20:19:24 -0700 Subject: [PATCH 23/78] try fix some mps problems --- fooocus_extras/ip_adapter.py | 6 +++-- fooocus_version.py | 2 +- modules/patch.py | 43 ------------------------------------ 3 files changed, 5 insertions(+), 46 deletions(-) diff --git a/fooocus_extras/ip_adapter.py b/fooocus_extras/ip_adapter.py index ac2bed2..0d2ca01 100644 --- a/fooocus_extras/ip_adapter.py +++ b/fooocus_extras/ip_adapter.py @@ -162,9 +162,11 @@ def preprocess(img): outputs = clip_vision.model(pixel_values=pixel_values, output_hidden_states=True) if ip_adapter.plus: - cond = outputs.hidden_states[-2].to(ip_adapter.dtype) + cond = outputs.hidden_states[-2] else: - cond = outputs.image_embeds.to(ip_adapter.dtype) + cond = outputs.image_embeds + + cond = cond.to(device=ip_adapter.load_device, dtype=ip_adapter.dtype) fcbh.model_management.load_model_gpu(image_proj_model) cond = image_proj_model.model(cond).to(device=ip_adapter.load_device, dtype=ip_adapter.dtype) diff --git a/fooocus_version.py b/fooocus_version.py index bb26d01..ac9cb17 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.737' +version = '2.1.738' diff --git a/modules/patch.py b/modules/patch.py index 9fa01f8..8303c22 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -459,38 +459,6 @@ def patched_unet_forward(self, x, timesteps=None, context=None, y=None, control= return self.out(h) -def patched_autocast(device_type, dtype=None, enabled=True, cache_enabled=None): - # https://github.com/lllyasviel/Fooocus/discussions/571 - # https://github.com/lllyasviel/Fooocus/issues/620 - # https://github.com/lllyasviel/Fooocus/issues/759 - - supported = False - - if device_type == 'cuda' and dtype == torch.float32 and enabled: - supported = True - - if device_type == 'cuda' and dtype == torch.float16 and enabled: - supported = True - - if device_type == 'cuda' and dtype == torch.bfloat16 and enabled: - supported = True - - if not supported: - print(f'[Fooocus Autocast Warning] Requested unsupported torch autocast [' - f'device_type={str(device_type)}, ' - f'dtype={str(dtype)}, ' - f'enabled={str(enabled)}, ' - f'cache_enabled={str(cache_enabled)}]. ' - f'Fooocus fixed it automatically, feel free to report to Fooocus on GitHub if this may cause potential problems.') - return contextlib.nullcontext() - - return torch.amp.autocast_mode.autocast_origin( - device_type=device_type, - dtype=dtype, - enabled=enabled, - cache_enabled=cache_enabled) - - def patched_load_models_gpu(*args, **kwargs): execution_start_time = time.perf_counter() y = fcbh.model_management.load_models_gpu_origin(*args, **kwargs) @@ -551,17 +519,6 @@ def patch_all(): if not hasattr(fcbh.model_management, 'load_models_gpu_origin'): fcbh.model_management.load_models_gpu_origin = fcbh.model_management.load_models_gpu - if not hasattr(torch.amp.autocast_mode, 'autocast_origin'): - torch.amp.autocast_mode.autocast_origin = torch.amp.autocast_mode.autocast - - torch.amp.autocast_mode.autocast = patched_autocast - torch.amp.autocast = patched_autocast - torch.autocast = patched_autocast - - # # Test if this will fail - # with torch.autocast(device_type='cpu', dtype=torch.float32): - # print(torch.ones(10)) - fcbh.model_management.load_models_gpu = patched_load_models_gpu fcbh.model_patcher.ModelPatcher.calculate_weight = calculate_weight_patched fcbh.cldm.cldm.ControlNet.forward = patched_cldm_forward From 523aa190db1cd23aa7d9c52b0d5404d46bfac898 Mon Sep 17 00:00:00 2001 From: MoonRide303 Date: Tue, 24 Oct 2023 12:15:43 +0200 Subject: [PATCH 24/78] Refactored handling max random seed --- modules/async_worker.py | 3 ++- modules/constants.py | 3 +++ webui.py | 11 ++++------- 3 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 modules/constants.py diff --git a/modules/async_worker.py b/modules/async_worker.py index 48ec32c..820bdc3 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -23,6 +23,7 @@ def worker(): import fcbh.model_management import fooocus_extras.preprocessors as preprocessors import modules.inpaint_worker as inpaint_worker + import modules.constants as constants import modules.advanced_parameters as advanced_parameters import fooocus_extras.ip_adapter as ip_adapter @@ -225,7 +226,7 @@ def worker(): progressbar(3, 'Processing prompts ...') tasks = [] for i in range(image_number): - task_seed = seed + i + task_seed = (seed + i) % (constants.MAX_SEED + 1) # randint is inclusive, % is not task_rng = random.Random(task_seed) # may bind to inpaint noise in the future task_prompt = apply_wildcards(prompt, task_rng) diff --git a/modules/constants.py b/modules/constants.py new file mode 100644 index 0000000..47c268f --- /dev/null +++ b/modules/constants.py @@ -0,0 +1,3 @@ +# as in k-diffusion (sampling.py) +MIN_SEED = 0 +MAX_SEED = 2**63 - 1 diff --git a/webui.py b/webui.py index e0702d8..7bd082f 100644 --- a/webui.py +++ b/webui.py @@ -9,6 +9,7 @@ import modules.path import fooocus_version import modules.html import modules.async_worker as worker +import modules.constants as constants import modules.flags as flags import modules.gradio_hijack as grh import modules.advanced_parameters as advanced_parameters @@ -18,10 +19,6 @@ from modules.sdxl_styles import legal_style_names from modules.private_logger import get_current_html_path from modules.ui_gradio_extensions import reload_javascript -# as in k-diffusion (sampling.py) -MIN_SEED = 0 -MAX_SEED = 2**63 - 1024 # for image number increasing safety - def generate_clicked(*args): execution_start_time = time.perf_counter() @@ -204,15 +201,15 @@ with shared.gradio_root: def refresh_seed(r, seed_string): if r: - return random.randint(MIN_SEED, MAX_SEED) + return random.randint(constants.MIN_SEED, constants.MAX_SEED) else: try: seed_value = int(seed_string) - if MIN_SEED <= seed_value <= MAX_SEED: + if constants.MIN_SEED <= seed_value <= constants.MAX_SEED: return seed_value except ValueError: pass - return random.randint(MIN_SEED, MAX_SEED) + return random.randint(constants.MIN_SEED, constants.MAX_SEED) seed_random.change(random_checked, inputs=[seed_random], outputs=[image_seed], queue=False) From fbbaf86232557301283b36dbc59f50caa4c912b7 Mon Sep 17 00:00:00 2001 From: MoonRide303 Date: Tue, 24 Oct 2023 18:29:37 +0200 Subject: [PATCH 25/78] Added support for authentication in --share mode (via auth.json) --- .gitignore | 1 + auth-example.json | 6 ++++++ fooocus_version.py | 2 +- modules/auth.py | 40 ++++++++++++++++++++++++++++++++++++++++ update_log.md | 4 ++++ webui.py | 4 +++- 6 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 auth-example.json create mode 100644 modules/auth.py diff --git a/.gitignore b/.gitignore index ce656b0..30b6ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ experiment.py /node_modules /package-lock.json /.coverage* +/auth.json diff --git a/auth-example.json b/auth-example.json new file mode 100644 index 0000000..59e321d --- /dev/null +++ b/auth-example.json @@ -0,0 +1,6 @@ +[ + { + "user": "sitting-duck-1", + "pass": "very-bad-publicly-known-password-change-it" + } +] diff --git a/fooocus_version.py b/fooocus_version.py index ac9cb17..0eab2d2 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.738' +version = '2.1.739' diff --git a/modules/auth.py b/modules/auth.py new file mode 100644 index 0000000..e609806 --- /dev/null +++ b/modules/auth.py @@ -0,0 +1,40 @@ +import json +import hashlib + +from os.path import exists + + +def auth_list_to_dict(auth_list): + auth_dict = {} + for auth_data in auth_list: + if 'user' in auth_data: + if 'hash' in auth_data: + auth_dict |= {auth_data['user']: auth_data['hash']} + elif 'pass' in auth_data: + auth_dict |= {auth_data['user']: hashlib.sha256(bytes(auth_data['pass'], encoding='utf-8')).hexdigest()} + return auth_dict + + +def load_auth_data(filename=None): + auth_dict = None + if filename != None and exists(filename): + with open(filename, encoding='utf-8') as auth_file: + try: + auth_obj = json.load(auth_file) + if isinstance(auth_obj, list) and len(auth_obj) > 0: + auth_dict = auth_list_to_dict(auth_obj) + except Exception as e: + print('load_auth_data, e: ' + str(e)) + return auth_dict + + +auth_dict = load_auth_data('auth.json') + +auth_enabled = auth_dict != None + + +def check_auth(user, password): + if user not in auth_dict: + return False + else: + return hashlib.sha256(bytes(password, encoding='utf-8')).hexdigest() == auth_dict[user] diff --git a/update_log.md b/update_log.md index 0f89296..a90e927 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,7 @@ +# 2.1.739 + +* Added support for authentication in --share mode (via auth.json). + # 2.1.737 * Allowed customizing resolutions in config. diff --git a/webui.py b/webui.py index 7bd082f..01bfa04 100644 --- a/webui.py +++ b/webui.py @@ -18,6 +18,7 @@ import args_manager from modules.sdxl_styles import legal_style_names from modules.private_logger import get_current_html_path from modules.ui_gradio_extensions import reload_javascript +from modules.auth import auth_enabled, check_auth def generate_clicked(*args): @@ -376,5 +377,6 @@ shared.gradio_root.launch( inbrowser=args_manager.args.auto_launch, server_name=args_manager.args.listen, server_port=args_manager.args.port, - share=args_manager.args.share + share=args_manager.args.share, + auth=check_auth if args_manager.args.share and auth_enabled else None ) From 5d32c208b25d801d7a60f0129b9f6a1074db015a Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Tue, 24 Oct 2023 11:38:03 -0700 Subject: [PATCH 26/78] remove unused codes --- modules/launch_util.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/launch_util.py b/modules/launch_util.py index d4641aa..00fff8a 100644 --- a/modules/launch_util.py +++ b/modules/launch_util.py @@ -1,16 +1,12 @@ import os import importlib import importlib.util -import shutil import subprocess import sys import re import logging -import pygit2 -pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0) - logging.getLogger("torch.distributed.nn").setLevel(logging.ERROR) # sshh... logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage()) From 19a03f1d0adfc115ba7d6666b9f9207528225318 Mon Sep 17 00:00:00 2001 From: MoonRide303 Date: Tue, 24 Oct 2023 21:55:29 +0200 Subject: [PATCH 27/78] Blocked remote access to auth.json --- modules/auth.py | 3 ++- modules/constants.py | 2 ++ webui.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/auth.py b/modules/auth.py index e609806..3ba1114 100644 --- a/modules/auth.py +++ b/modules/auth.py @@ -1,5 +1,6 @@ import json import hashlib +import modules.constants as constants from os.path import exists @@ -28,7 +29,7 @@ def load_auth_data(filename=None): return auth_dict -auth_dict = load_auth_data('auth.json') +auth_dict = load_auth_data(constants.AUTH_FILENAME) auth_enabled = auth_dict != None diff --git a/modules/constants.py b/modules/constants.py index 47c268f..667fa86 100644 --- a/modules/constants.py +++ b/modules/constants.py @@ -1,3 +1,5 @@ # as in k-diffusion (sampling.py) MIN_SEED = 0 MAX_SEED = 2**63 - 1 + +AUTH_FILENAME = 'auth.json' diff --git a/webui.py b/webui.py index 01bfa04..180825d 100644 --- a/webui.py +++ b/webui.py @@ -378,5 +378,6 @@ shared.gradio_root.launch( server_name=args_manager.args.listen, server_port=args_manager.args.port, share=args_manager.args.share, - auth=check_auth if args_manager.args.share and auth_enabled else None + auth=check_auth if args_manager.args.share and auth_enabled else None, + blocked_paths=[constants.AUTH_FILENAME] ) From 9bb16d0d767a99c8708d45270ff96a6e019ddc55 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Tue, 24 Oct 2023 14:09:56 -0700 Subject: [PATCH 28/78] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 14 ++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..3ac19a0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,14 @@ +--- +name: Bug report +about: Describe a problem +title: '' +labels: '' +assignees: '' + +--- + +**Describe the problem** +A clear and concise description of what the bug is. + +**Full Console Log** +Paste **full** console log here. You will make our job easier if you give a **full** log. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..8101bc3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,14 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the idea you'd like** +A clear and concise description of what you want to happen. From bb965067e0a62881e1812cdaf6e04257fa06f26a Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 25 Oct 2023 07:16:53 -0700 Subject: [PATCH 29/78] Update readme.md --- readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index a0b27de..296e29d 100644 --- a/readme.md +++ b/readme.md @@ -281,11 +281,11 @@ A safter way is just to try "run_anime.bat" or "run_realistic.bat" - they should [Click here to browse the advanced features.](https://github.com/lllyasviel/Fooocus/discussions/117) -Fooocus also has many community forks, just like SD-WebUI, for enthusiastic users who want to try! +Fooocus also has many community forks, just like SD-WebUI's [vladmandic/automatic](https://github.com/vladmandic/automatic) and [anapnoe/stable-diffusion-webui-ux](https://github.com/anapnoe/stable-diffusion-webui-ux), for enthusiastic users who want to try! -| SD-WebUI's forks | Fooocus' forks | -| - | - | -| [vladmandic/automatic](https://github.com/vladmandic/automatic)
[anapnoe/stable-diffusion-webui-ux](https://github.com/anapnoe/stable-diffusion-webui-ux)
and so on ... | [runew0lf/RuinedFooocus](https://github.com/runew0lf/RuinedFooocus)
[MoonRide303/Fooocus-MRE](https://github.com/MoonRide303/Fooocus-MRE)
and so on ... | +| Fooocus' forks | +| - | +| [fenneishi/Fooocus-Control](https://github.com/fenneishi/Fooocus-Control)
[runew0lf/RuinedFooocus](https://github.com/runew0lf/RuinedFooocus)
[MoonRide303/Fooocus-MRE](https://github.com/MoonRide303/Fooocus-MRE)
and so on ... | See also [About Forking and Promotion of Forks](https://github.com/lllyasviel/Fooocus/discussions/699). From 38e70cebccd6c1f05856d4333718d1eb432588af Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 25 Oct 2023 08:07:41 -0700 Subject: [PATCH 30/78] Update Backend Update Backend --- backend/headless/fcbh/conds.py | 64 ++++++++ backend/headless/fcbh/controlnet.py | 2 +- backend/headless/fcbh/model_base.py | 21 ++- backend/headless/fcbh/sample.py | 31 ++-- backend/headless/fcbh/samplers.py | 239 +++++++++++----------------- fooocus_version.py | 2 +- modules/async_worker.py | 1 - modules/default_pipeline.py | 2 +- modules/patch.py | 102 ++++++------ modules/sample_hijack.py | 72 ++++++--- webui.py | 4 +- 11 files changed, 288 insertions(+), 252 deletions(-) create mode 100644 backend/headless/fcbh/conds.py diff --git a/backend/headless/fcbh/conds.py b/backend/headless/fcbh/conds.py new file mode 100644 index 0000000..252bb86 --- /dev/null +++ b/backend/headless/fcbh/conds.py @@ -0,0 +1,64 @@ +import enum +import torch +import math +import fcbh.utils + + +def lcm(a, b): #TODO: eventually replace by math.lcm (added in python3.9) + return abs(a*b) // math.gcd(a, b) + +class CONDRegular: + def __init__(self, cond): + self.cond = cond + + def _copy_with(self, cond): + return self.__class__(cond) + + def process_cond(self, batch_size, device, **kwargs): + return self._copy_with(fcbh.utils.repeat_to_batch_size(self.cond, batch_size).to(device)) + + def can_concat(self, other): + if self.cond.shape != other.cond.shape: + return False + return True + + def concat(self, others): + conds = [self.cond] + for x in others: + conds.append(x.cond) + return torch.cat(conds) + +class CONDNoiseShape(CONDRegular): + def process_cond(self, batch_size, device, area, **kwargs): + data = self.cond[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] + return self._copy_with(fcbh.utils.repeat_to_batch_size(data, batch_size).to(device)) + + +class CONDCrossAttn(CONDRegular): + def can_concat(self, other): + s1 = self.cond.shape + s2 = other.cond.shape + if s1 != s2: + if s1[0] != s2[0] or s1[2] != s2[2]: #these 2 cases should not happen + return False + + mult_min = lcm(s1[1], s2[1]) + diff = mult_min // min(s1[1], s2[1]) + if diff > 4: #arbitrary limit on the padding because it's probably going to impact performance negatively if it's too much + return False + return True + + def concat(self, others): + conds = [self.cond] + crossattn_max_len = self.cond.shape[1] + for x in others: + c = x.cond + crossattn_max_len = lcm(crossattn_max_len, c.shape[1]) + conds.append(c) + + out = [] + for c in conds: + if c.shape[1] < crossattn_max_len: + c = c.repeat(1, crossattn_max_len // c.shape[1], 1) #padding with repeat doesn't change result + out.append(c) + return torch.cat(out) diff --git a/backend/headless/fcbh/controlnet.py b/backend/headless/fcbh/controlnet.py index dcdd0c1..ab6c38f 100644 --- a/backend/headless/fcbh/controlnet.py +++ b/backend/headless/fcbh/controlnet.py @@ -156,7 +156,7 @@ class ControlNet(ControlBase): context = cond['c_crossattn'] - y = cond.get('c_adm', None) + y = cond.get('y', None) if y is not None: y = y.to(self.control_model.dtype) control = self.control_model(x=x_noisy.to(self.control_model.dtype), hint=self.cond_hint, timesteps=t, context=context.to(self.control_model.dtype), y=y) diff --git a/backend/headless/fcbh/model_base.py b/backend/headless/fcbh/model_base.py index f3f708f..86525d9 100644 --- a/backend/headless/fcbh/model_base.py +++ b/backend/headless/fcbh/model_base.py @@ -4,6 +4,7 @@ from fcbh.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugmen from fcbh.ldm.modules.diffusionmodules.util import make_beta_schedule from fcbh.ldm.modules.diffusionmodules.openaimodel import Timestep import fcbh.model_management +import fcbh.conds import numpy as np from enum import Enum from . import utils @@ -49,7 +50,7 @@ class BaseModel(torch.nn.Module): self.register_buffer('alphas_cumprod', torch.tensor(alphas_cumprod, dtype=torch.float32)) self.register_buffer('alphas_cumprod_prev', torch.tensor(alphas_cumprod_prev, dtype=torch.float32)) - def apply_model(self, x, t, c_concat=None, c_crossattn=None, c_adm=None, control=None, transformer_options={}): + def apply_model(self, x, t, c_concat=None, c_crossattn=None, control=None, transformer_options={}, **kwargs): if c_concat is not None: xc = torch.cat([x] + [c_concat], dim=1) else: @@ -59,9 +60,10 @@ class BaseModel(torch.nn.Module): xc = xc.to(dtype) t = t.to(dtype) context = context.to(dtype) - if c_adm is not None: - c_adm = c_adm.to(dtype) - return self.diffusion_model(xc, t, context=context, y=c_adm, control=control, transformer_options=transformer_options).float() + extra_conds = {} + for o in kwargs: + extra_conds[o] = kwargs[o].to(dtype) + return self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float() def get_dtype(self): return self.diffusion_model.dtype @@ -72,7 +74,8 @@ class BaseModel(torch.nn.Module): def encode_adm(self, **kwargs): return None - def cond_concat(self, **kwargs): + def extra_conds(self, **kwargs): + out = {} if self.inpaint_model: concat_keys = ("mask", "masked_image") cond_concat = [] @@ -101,8 +104,12 @@ class BaseModel(torch.nn.Module): cond_concat.append(torch.ones_like(noise)[:,:1]) elif ck == "masked_image": cond_concat.append(blank_inpaint_image_like(noise)) - return cond_concat - return None + data = torch.cat(cond_concat, dim=1) + out['c_concat'] = fcbh.conds.CONDNoiseShape(data) + adm = self.encode_adm(**kwargs) + if adm is not None: + out['y'] = fcbh.conds.CONDRegular(adm) + return out def load_model_weights(self, sd, unet_prefix=""): to_load = {} diff --git a/backend/headless/fcbh/sample.py b/backend/headless/fcbh/sample.py index b6e0fdd..5594616 100644 --- a/backend/headless/fcbh/sample.py +++ b/backend/headless/fcbh/sample.py @@ -1,6 +1,7 @@ import torch import fcbh.model_management import fcbh.samplers +import fcbh.conds import fcbh.utils import math import numpy as np @@ -33,22 +34,24 @@ def prepare_mask(noise_mask, shape, device): noise_mask = noise_mask.to(device) return noise_mask -def broadcast_cond(cond, batch, device): - """broadcasts conditioning to the batch size""" - copy = [] - for p in cond: - t = fcbh.utils.repeat_to_batch_size(p[0], batch) - t = t.to(device) - copy += [[t] + p[1:]] - return copy - def get_models_from_cond(cond, model_type): models = [] for c in cond: - if model_type in c[1]: - models += [c[1][model_type]] + if model_type in c: + models += [c[model_type]] return models +def convert_cond(cond): + out = [] + for c in cond: + temp = c[1].copy() + model_conds = temp.get("model_conds", {}) + if c[0] is not None: + model_conds["c_crossattn"] = fcbh.conds.CONDCrossAttn(c[0]) + temp["model_conds"] = model_conds + out.append(temp) + return out + def get_additional_models(positive, negative, dtype): """loads additional models in positive and negative conditioning""" control_nets = set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control")) @@ -72,6 +75,8 @@ def cleanup_additional_models(models): def prepare_sampling(model, noise_shape, positive, negative, noise_mask): device = model.load_device + positive = convert_cond(positive) + negative = convert_cond(negative) if noise_mask is not None: noise_mask = prepare_mask(noise_mask, noise_shape, device) @@ -81,9 +86,7 @@ def prepare_sampling(model, noise_shape, positive, negative, noise_mask): fcbh.model_management.load_models_gpu([model] + models, fcbh.model_management.batch_area_memory(noise_shape[0] * noise_shape[2] * noise_shape[3]) + inference_memory) real_model = model.model - positive_copy = broadcast_cond(positive, noise_shape[0], device) - negative_copy = broadcast_cond(negative, noise_shape[0], device) - return real_model, positive_copy, negative_copy, noise_mask, models + return real_model, positive, negative, noise_mask, models def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, noise_mask=None, sigmas=None, callback=None, disable_pbar=False, seed=None): diff --git a/backend/headless/fcbh/samplers.py b/backend/headless/fcbh/samplers.py index fe41499..91050a4 100644 --- a/backend/headless/fcbh/samplers.py +++ b/backend/headless/fcbh/samplers.py @@ -2,47 +2,44 @@ from .k_diffusion import sampling as k_diffusion_sampling from .k_diffusion import external as k_diffusion_external from .extra_samplers import uni_pc import torch +import enum from fcbh import model_management from .ldm.models.diffusion.ddim import DDIMSampler from .ldm.modules.diffusionmodules.util import make_ddim_timesteps import math from fcbh import model_base import fcbh.utils +import fcbh.conds -def lcm(a, b): #TODO: eventually replace by math.lcm (added in python3.9) - return abs(a*b) // math.gcd(a, b) #The main sampling function shared by all the samplers #Returns predicted noise def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None): - def get_area_and_mult(cond, x_in, timestep_in): + def get_area_and_mult(conds, x_in, timestep_in): area = (x_in.shape[2], x_in.shape[3], 0, 0) strength = 1.0 - if 'timestep_start' in cond[1]: - timestep_start = cond[1]['timestep_start'] + + if 'timestep_start' in conds: + timestep_start = conds['timestep_start'] if timestep_in[0] > timestep_start: return None - if 'timestep_end' in cond[1]: - timestep_end = cond[1]['timestep_end'] + if 'timestep_end' in conds: + timestep_end = conds['timestep_end'] if timestep_in[0] < timestep_end: return None - if 'area' in cond[1]: - area = cond[1]['area'] - if 'strength' in cond[1]: - strength = cond[1]['strength'] - - adm_cond = None - if 'adm_encoded' in cond[1]: - adm_cond = cond[1]['adm_encoded'] + if 'area' in conds: + area = conds['area'] + if 'strength' in conds: + strength = conds['strength'] input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] - if 'mask' in cond[1]: + if 'mask' in conds: # Scale the mask to the size of the input # The mask should have been resized as we began the sampling process mask_strength = 1.0 - if "mask_strength" in cond[1]: - mask_strength = cond[1]["mask_strength"] - mask = cond[1]['mask'] + if "mask_strength" in conds: + mask_strength = conds["mask_strength"] + mask = conds['mask'] assert(mask.shape[1] == x_in.shape[2]) assert(mask.shape[2] == x_in.shape[3]) mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength @@ -51,7 +48,7 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, mod mask = torch.ones_like(input_x) mult = mask * strength - if 'mask' not in cond[1]: + if 'mask' not in conds: rr = 8 if area[2] != 0: for t in range(rr): @@ -67,27 +64,17 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, mod mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1)) conditionning = {} - conditionning['c_crossattn'] = cond[0] - - if 'concat' in cond[1]: - cond_concat_in = cond[1]['concat'] - if cond_concat_in is not None and len(cond_concat_in) > 0: - cropped = [] - for x in cond_concat_in: - cr = x[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] - cropped.append(cr) - conditionning['c_concat'] = torch.cat(cropped, dim=1) - - if adm_cond is not None: - conditionning['c_adm'] = adm_cond + model_conds = conds["model_conds"] + for c in model_conds: + conditionning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area) control = None - if 'control' in cond[1]: - control = cond[1]['control'] + if 'control' in conds: + control = conds['control'] patches = None - if 'gligen' in cond[1]: - gligen = cond[1]['gligen'] + if 'gligen' in conds: + gligen = conds['gligen'] patches = {} gligen_type = gligen[0] gligen_model = gligen[1] @@ -105,22 +92,8 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, mod return True if c1.keys() != c2.keys(): return False - if 'c_crossattn' in c1: - s1 = c1['c_crossattn'].shape - s2 = c2['c_crossattn'].shape - if s1 != s2: - if s1[0] != s2[0] or s1[2] != s2[2]: #these 2 cases should not happen - return False - - mult_min = lcm(s1[1], s2[1]) - diff = mult_min // min(s1[1], s2[1]) - if diff > 4: #arbitrary limit on the padding because it's probably going to impact performance negatively if it's too much - return False - if 'c_concat' in c1: - if c1['c_concat'].shape != c2['c_concat'].shape: - return False - if 'c_adm' in c1: - if c1['c_adm'].shape != c2['c_adm'].shape: + for k in c1: + if not c1[k].can_concat(c2[k]): return False return True @@ -149,31 +122,19 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, mod c_concat = [] c_adm = [] crossattn_max_len = 0 - for x in c_list: - if 'c_crossattn' in x: - c = x['c_crossattn'] - if crossattn_max_len == 0: - crossattn_max_len = c.shape[1] - else: - crossattn_max_len = lcm(crossattn_max_len, c.shape[1]) - c_crossattn.append(c) - if 'c_concat' in x: - c_concat.append(x['c_concat']) - if 'c_adm' in x: - c_adm.append(x['c_adm']) - out = {} - c_crossattn_out = [] - for c in c_crossattn: - if c.shape[1] < crossattn_max_len: - c = c.repeat(1, crossattn_max_len // c.shape[1], 1) #padding with repeat doesn't change result - c_crossattn_out.append(c) - if len(c_crossattn_out) > 0: - out['c_crossattn'] = torch.cat(c_crossattn_out) - if len(c_concat) > 0: - out['c_concat'] = torch.cat(c_concat) - if len(c_adm) > 0: - out['c_adm'] = torch.cat(c_adm) + temp = {} + for x in c_list: + for k in x: + cur = temp.get(k, []) + cur.append(x[k]) + temp[k] = cur + + out = {} + for k in temp: + conds = temp[k] + out[k] = conds[0].concat(conds[1:]) + return out def calc_cond_uncond_batch(model_function, cond, uncond, x_in, timestep, max_total_area, model_options): @@ -389,19 +350,19 @@ def resolve_areas_and_cond_masks(conditions, h, w, device): # While we're doing this, we can also resolve the mask device and scaling for performance reasons for i in range(len(conditions)): c = conditions[i] - if 'area' in c[1]: - area = c[1]['area'] + if 'area' in c: + area = c['area'] if area[0] == "percentage": - modified = c[1].copy() + modified = c.copy() area = (max(1, round(area[1] * h)), max(1, round(area[2] * w)), round(area[3] * h), round(area[4] * w)) modified['area'] = area - c = [c[0], modified] + c = modified conditions[i] = c - if 'mask' in c[1]: - mask = c[1]['mask'] + if 'mask' in c: + mask = c['mask'] mask = mask.to(device=device) - modified = c[1].copy() + modified = c.copy() if len(mask.shape) == 2: mask = mask.unsqueeze(0) if mask.shape[1] != h or mask.shape[2] != w: @@ -422,37 +383,39 @@ def resolve_areas_and_cond_masks(conditions, h, w, device): modified['area'] = area modified['mask'] = mask - conditions[i] = [c[0], modified] + conditions[i] = modified def create_cond_with_same_area_if_none(conds, c): - if 'area' not in c[1]: + if 'area' not in c: return - c_area = c[1]['area'] + c_area = c['area'] smallest = None for x in conds: - if 'area' in x[1]: - a = x[1]['area'] + if 'area' in x: + a = x['area'] if c_area[2] >= a[2] and c_area[3] >= a[3]: if a[0] + a[2] >= c_area[0] + c_area[2]: if a[1] + a[3] >= c_area[1] + c_area[3]: if smallest is None: smallest = x - elif 'area' not in smallest[1]: + elif 'area' not in smallest: smallest = x else: - if smallest[1]['area'][0] * smallest[1]['area'][1] > a[0] * a[1]: + if smallest['area'][0] * smallest['area'][1] > a[0] * a[1]: smallest = x else: if smallest is None: smallest = x if smallest is None: return - if 'area' in smallest[1]: - if smallest[1]['area'] == c_area: + if 'area' in smallest: + if smallest['area'] == c_area: return - n = c[1].copy() - conds += [[smallest[0], n]] + + out = c.copy() + out['model_conds'] = smallest['model_conds'].copy() #TODO: which fields should be copied? + conds += [out] def calculate_start_end_timesteps(model, conds): for t in range(len(conds)): @@ -460,18 +423,18 @@ def calculate_start_end_timesteps(model, conds): timestep_start = None timestep_end = None - if 'start_percent' in x[1]: - timestep_start = model.sigma_to_t(model.t_to_sigma(torch.tensor(x[1]['start_percent'] * 999.0))) - if 'end_percent' in x[1]: - timestep_end = model.sigma_to_t(model.t_to_sigma(torch.tensor(x[1]['end_percent'] * 999.0))) + if 'start_percent' in x: + timestep_start = model.sigma_to_t(model.t_to_sigma(torch.tensor(x['start_percent'] * 999.0))) + if 'end_percent' in x: + timestep_end = model.sigma_to_t(model.t_to_sigma(torch.tensor(x['end_percent'] * 999.0))) if (timestep_start is not None) or (timestep_end is not None): - n = x[1].copy() + n = x.copy() if (timestep_start is not None): n['timestep_start'] = timestep_start if (timestep_end is not None): n['timestep_end'] = timestep_end - conds[t] = [x[0], n] + conds[t] = n def pre_run_control(model, conds): for t in range(len(conds)): @@ -480,8 +443,8 @@ def pre_run_control(model, conds): timestep_start = None timestep_end = None percent_to_timestep_function = lambda a: model.sigma_to_t(model.t_to_sigma(torch.tensor(a) * 999.0)) - if 'control' in x[1]: - x[1]['control'].pre_run(model.inner_model.inner_model, percent_to_timestep_function) + if 'control' in x: + x['control'].pre_run(model.inner_model.inner_model, percent_to_timestep_function) def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func): cond_cnets = [] @@ -490,16 +453,16 @@ def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func): uncond_other = [] for t in range(len(conds)): x = conds[t] - if 'area' not in x[1]: - if name in x[1] and x[1][name] is not None: - cond_cnets.append(x[1][name]) + if 'area' not in x: + if name in x and x[name] is not None: + cond_cnets.append(x[name]) else: cond_other.append((x, t)) for t in range(len(uncond)): x = uncond[t] - if 'area' not in x[1]: - if name in x[1] and x[1][name] is not None: - uncond_cnets.append(x[1][name]) + if 'area' not in x: + if name in x and x[name] is not None: + uncond_cnets.append(x[name]) else: uncond_other.append((x, t)) @@ -509,47 +472,35 @@ def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func): for x in range(len(cond_cnets)): temp = uncond_other[x % len(uncond_other)] o = temp[0] - if name in o[1] and o[1][name] is not None: - n = o[1].copy() + if name in o and o[name] is not None: + n = o.copy() n[name] = uncond_fill_func(cond_cnets, x) - uncond += [[o[0], n]] + uncond += [n] else: - n = o[1].copy() + n = o.copy() n[name] = uncond_fill_func(cond_cnets, x) - uncond[temp[1]] = [o[0], n] + uncond[temp[1]] = n -def encode_adm(model, conds, batch_size, width, height, device, prompt_type): +def encode_model_conds(model_function, conds, noise, device, prompt_type, **kwargs): for t in range(len(conds)): x = conds[t] - adm_out = None - if 'adm' in x[1]: - adm_out = x[1]["adm"] - else: - params = x[1].copy() - params["width"] = params.get("width", width * 8) - params["height"] = params.get("height", height * 8) - params["prompt_type"] = params.get("prompt_type", prompt_type) - adm_out = model.encode_adm(device=device, **params) - - if adm_out is not None: - x[1] = x[1].copy() - x[1]["adm_encoded"] = fcbh.utils.repeat_to_batch_size(adm_out, batch_size).to(device) - - return conds - -def encode_cond(model_function, key, conds, device, **kwargs): - for t in range(len(conds)): - x = conds[t] - params = x[1].copy() + params = x.copy() params["device"] = device + params["noise"] = noise + params["width"] = params.get("width", noise.shape[3] * 8) + params["height"] = params.get("height", noise.shape[2] * 8) + params["prompt_type"] = params.get("prompt_type", prompt_type) for k in kwargs: if k not in params: params[k] = kwargs[k] out = model_function(**params) - if out is not None: - x[1] = x[1].copy() - x[1][key] = out + x = x.copy() + model_conds = x['model_conds'].copy() + for k in out: + model_conds[k] = out[k] + x['model_conds'] = model_conds + conds[t] = x return conds class Sampler: @@ -667,19 +618,15 @@ def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model pre_run_control(model_wrap, negative + positive) - apply_empty_x_to_equal_area(list(filter(lambda c: c[1].get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x]) + apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x]) apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x]) if latent_image is not None: latent_image = model.process_latent_in(latent_image) - if model.is_adm(): - positive = encode_adm(model, positive, noise.shape[0], noise.shape[3], noise.shape[2], device, "positive") - negative = encode_adm(model, negative, noise.shape[0], noise.shape[3], noise.shape[2], device, "negative") - - if hasattr(model, 'cond_concat'): - positive = encode_cond(model.cond_concat, "concat", positive, device, noise=noise, latent_image=latent_image, denoise_mask=denoise_mask) - negative = encode_cond(model.cond_concat, "concat", negative, device, noise=noise, latent_image=latent_image, denoise_mask=denoise_mask) + if hasattr(model, 'extra_conds'): + positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask) + negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask) extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed} diff --git a/fooocus_version.py b/fooocus_version.py index 0eab2d2..5f62795 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.739' +version = '2.1.740' diff --git a/modules/async_worker.py b/modules/async_worker.py index 820bdc3..5fc925e 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -174,7 +174,6 @@ def worker(): loras += [(inpaint_patch_model_path, 1.0)] print(f'[Inpaint] Current inpaint model is {inpaint_patch_model_path}') goals.append('inpaint') - sampler_name = 'dpmpp_2m_sde_gpu' # only support the patched dpmpp_2m_sde_gpu if current_tab == 'ip' or \ advanced_parameters.mixing_image_prompt_and_inpaint or \ advanced_parameters.mixing_image_prompt_and_vary_upscale: diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 8557ac2..b32f811 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -342,7 +342,7 @@ def process_diffusion(positive_cond, negative_cond, steps, switch, width, height sigma_max = float(sigma_max.cpu().numpy()) print(f'[Sampler] sigma_min = {sigma_min}, sigma_max = {sigma_max}') - modules.patch.globalBrownianTreeNoiseSampler = BrownianTreeNoiseSampler( + modules.patch.BrownianTreeNoiseSamplerPatched.global_init( empty_latent['samples'].to(fcbh.model_management.get_torch_device()), sigma_min, sigma_max, seed=image_seed, cpu=False) diff --git a/modules/patch.py b/modules/patch.py index 8303c22..98b56d5 100644 --- a/modules/patch.py +++ b/modules/patch.py @@ -23,9 +23,10 @@ import args_manager import modules.advanced_parameters as advanced_parameters import warnings import safetensors.torch +import modules.constants as constants from fcbh.k_diffusion import utils -from fcbh.k_diffusion.sampling import trange +from fcbh.k_diffusion.sampling import BatchedBrownianTree from fcbh.ldm.modules.diffusionmodules.openaimodel import timestep_embedding, forward_timestep_embed @@ -280,68 +281,58 @@ def encode_token_weights_patched_with_a1111_method(self, token_weight_pairs): return torch.cat(output, dim=-2).cpu(), first_pooled.cpu() -globalBrownianTreeNoiseSampler = None - - -@torch.no_grad() -def sample_dpmpp_fooocus_2m_sde_inpaint_seamless(model, x, sigmas, extra_args=None, callback=None, - disable=None, eta=1., s_noise=1., **kwargs): - print('[Sampler] Fooocus sampler is activated.') - - seed = extra_args.get("seed", None) - assert isinstance(seed, int) - - energy_generator = torch.Generator(device='cpu') - energy_generator.manual_seed(seed + 1) # avoid bad results by using different seeds. - - def get_energy(): - return torch.randn(x.size(), dtype=x.dtype, generator=energy_generator, device="cpu").to(x) - - extra_args = {} if extra_args is None else extra_args - s_in = x.new_ones([x.shape[0]]) - - old_denoised, h_last, h = None, None, None - - latent_processor = model.inner_model.inner_model.inner_model.process_latent_in - inpaint_latent = None - inpaint_mask = None - +def patched_KSamplerX0Inpaint_forward(self, x, sigma, uncond, cond, cond_scale, denoise_mask, model_options={}, seed=None): if inpaint_worker.current_task is not None: + if getattr(self, 'energy_generator', None) is None: + # avoid bad results by using different seeds. + self.energy_generator = torch.Generator(device='cpu').manual_seed((seed + 1) % constants.MAX_SEED) + + latent_processor = self.inner_model.inner_model.inner_model.process_latent_in inpaint_latent = latent_processor(inpaint_worker.current_task.latent).to(x) inpaint_mask = inpaint_worker.current_task.latent_mask.to(x) + energy_sigma = sigma.reshape([sigma.shape[0]] + [1] * (len(x.shape) - 1)) + current_energy = torch.randn(x.size(), dtype=x.dtype, generator=self.energy_generator, device="cpu").to(x) * energy_sigma + x = x * inpaint_mask + (inpaint_latent + current_energy) * (1.0 - inpaint_mask) - def blend_latent(a, b, w): - return a * w + b * (1 - w) + out = self.inner_model(x, sigma, + cond=cond, + uncond=uncond, + cond_scale=cond_scale, + model_options=model_options, + seed=seed) - for i in trange(len(sigmas) - 1, disable=disable): - if inpaint_latent is None: - denoised = model(x, sigmas[i] * s_in, **extra_args) - else: - energy = get_energy() * sigmas[i] + inpaint_latent - x_prime = blend_latent(x, energy, inpaint_mask) - denoised = model(x_prime, sigmas[i] * s_in, **extra_args) - denoised = blend_latent(denoised, inpaint_latent, inpaint_mask) - if callback is not None: - callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) - if sigmas[i + 1] == 0: - x = denoised - else: - t, s = -sigmas[i].log(), -sigmas[i + 1].log() - h = s - t - eta_h = eta * h + out = out * inpaint_mask + inpaint_latent * (1.0 - inpaint_mask) + else: + out = self.inner_model(x, sigma, + cond=cond, + uncond=uncond, + cond_scale=cond_scale, + model_options=model_options, + seed=seed) + return out - x = sigmas[i + 1] / sigmas[i] * (-eta_h).exp() * x + (-h - eta_h).expm1().neg() * denoised - if old_denoised is not None: - r = h_last / h - x = x + 0.5 * (-h - eta_h).expm1().neg() * (1 / r) * (denoised - old_denoised) - x = x + globalBrownianTreeNoiseSampler(sigmas[i], sigmas[i + 1]) * sigmas[i + 1] * ( - -2 * eta_h).expm1().neg().sqrt() * s_noise +class BrownianTreeNoiseSamplerPatched: + transform = None + tree = None - old_denoised = denoised - h_last = h + @staticmethod + def global_init(x, sigma_min, sigma_max, seed=None, transform=lambda x: x, cpu=False): + t0, t1 = transform(torch.as_tensor(sigma_min)), transform(torch.as_tensor(sigma_max)) - return x + BrownianTreeNoiseSamplerPatched.transform = transform + BrownianTreeNoiseSamplerPatched.tree = BatchedBrownianTree(x, t0, t1, seed, cpu=cpu) + + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def __call__(sigma, sigma_next): + transform = BrownianTreeNoiseSamplerPatched.transform + tree = BrownianTreeNoiseSamplerPatched.tree + + t0, t1 = transform(torch.as_tensor(sigma)), transform(torch.as_tensor(sigma_next)) + return tree(t0, t1) / (t1 - t0).abs().sqrt() def timed_adm(y, timesteps): @@ -523,10 +514,11 @@ def patch_all(): fcbh.model_patcher.ModelPatcher.calculate_weight = calculate_weight_patched fcbh.cldm.cldm.ControlNet.forward = patched_cldm_forward fcbh.ldm.modules.diffusionmodules.openaimodel.UNetModel.forward = patched_unet_forward - fcbh.k_diffusion.sampling.sample_dpmpp_2m_sde_gpu = sample_dpmpp_fooocus_2m_sde_inpaint_seamless fcbh.k_diffusion.external.DiscreteEpsDDPMDenoiser.forward = patched_discrete_eps_ddpm_denoiser_forward fcbh.model_base.SDXL.encode_adm = sdxl_encode_adm_patched fcbh.sd1_clip.ClipTokenWeightEncoder.encode_token_weights = encode_token_weights_patched_with_a1111_method + fcbh.samplers.KSamplerX0Inpaint.forward = patched_KSamplerX0Inpaint_forward + fcbh.k_diffusion.sampling.BrownianTreeNoiseSampler = BrownianTreeNoiseSamplerPatched warnings.filterwarnings(action='ignore', module='torchsde') diff --git a/modules/sample_hijack.py b/modules/sample_hijack.py index bf7ea09..30e47b6 100644 --- a/modules/sample_hijack.py +++ b/modules/sample_hijack.py @@ -3,10 +3,10 @@ import fcbh.samplers import fcbh.model_management from fcbh.model_base import SDXLRefiner, SDXL +from fcbh.conds import CONDRegular from fcbh.sample import get_additional_models, get_models_from_cond, cleanup_additional_models from fcbh.samplers import resolve_areas_and_cond_masks, wrap_model, calculate_start_end_timesteps, \ - create_cond_with_same_area_if_none, pre_run_control, apply_empty_x_to_equal_area, encode_adm, \ - encode_cond + create_cond_with_same_area_if_none, pre_run_control, apply_empty_x_to_equal_area, encode_model_conds current_refiner = None @@ -15,15 +15,13 @@ refiner_switch_step = -1 @torch.no_grad() @torch.inference_mode() -def clip_separate(cond, target_model=None, target_clip=None): - c, p = cond[0] +def clip_separate_inner(c, p, target_model=None, target_clip=None): if target_model is None or isinstance(target_model, SDXLRefiner): c = c[..., -1280:].clone() - p = {"pooled_output": p["pooled_output"].clone()} elif isinstance(target_model, SDXL): c = c.clone() - p = {"pooled_output": p["pooled_output"].clone()} else: + p = None c = c[..., :768].clone() final_layer_norm = target_clip.cond_stage_model.clip_l.transformer.text_model.final_layer_norm @@ -43,9 +41,42 @@ def clip_separate(cond, target_model=None, target_clip=None): final_layer_norm.to(device=final_layer_norm_origin_device, dtype=final_layer_norm_origin_dtype) c = c.to(device=c_origin_device, dtype=c_origin_dtype) + return c, p - p = {} - return [[c, p]] + +@torch.no_grad() +@torch.inference_mode() +def clip_separate(cond, target_model=None, target_clip=None): + results = [] + + for c, px in cond: + p = px.get('pooled_output', None) + c, p = clip_separate_inner(c, p, target_model=target_model, target_clip=target_clip) + p = {} if p is None else {'pooled_output': p.clone()} + results.append([c, p]) + + return results + + +@torch.no_grad() +@torch.inference_mode() +def clip_separate_after_preparation(cond, target_model=None, target_clip=None): + results = [] + + for x in cond: + p = x.get('pooled_output', None) + c = x['model_conds']['c_crossattn'].cond + + c, p = clip_separate_inner(c, p, target_model=target_model, target_clip=target_clip) + + result = {'model_conds': {'c_crossattn': CONDRegular(c)}} + + if p is not None: + result['pooled_output'] = p.clone() + + results.append(result) + + return results @torch.no_grad() @@ -73,31 +104,24 @@ def sample_hacked(model, noise, positive, negative, cfg, device, sampler, sigmas # pre_run_control(model_wrap, negative + positive) pre_run_control(model_wrap, positive) # negative is not necessary in Fooocus, 0.5s faster. - apply_empty_x_to_equal_area(list(filter(lambda c: c[1].get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x]) + apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x]) apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x]) if latent_image is not None: latent_image = model.process_latent_in(latent_image) - if model.is_adm(): - positive = encode_adm(model, positive, noise.shape[0], noise.shape[3], noise.shape[2], device, "positive") - negative = encode_adm(model, negative, noise.shape[0], noise.shape[3], noise.shape[2], device, "negative") - - if hasattr(model, 'cond_concat'): - positive = encode_cond(model.cond_concat, "concat", positive, device, noise=noise, latent_image=latent_image, denoise_mask=denoise_mask) - negative = encode_cond(model.cond_concat, "concat", negative, device, noise=noise, latent_image=latent_image, denoise_mask=denoise_mask) + if hasattr(model, 'extra_conds'): + positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask) + negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask) extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed} - if current_refiner is not None and current_refiner.model.is_adm(): - positive_refiner = clip_separate(positive, target_model=current_refiner.model) - negative_refiner = clip_separate(negative, target_model=current_refiner.model) + if current_refiner is not None and hasattr(current_refiner.model, 'extra_conds'): + positive_refiner = clip_separate_after_preparation(positive, target_model=current_refiner.model) + negative_refiner = clip_separate_after_preparation(negative, target_model=current_refiner.model) - positive_refiner = encode_adm(current_refiner.model, positive_refiner, noise.shape[0], noise.shape[3], noise.shape[2], device, "positive") - negative_refiner = encode_adm(current_refiner.model, negative_refiner, noise.shape[0], noise.shape[3], noise.shape[2], device, "negative") - - positive_refiner[0][1]['adm_encoded'].to(positive[0][1]['adm_encoded']) - negative_refiner[0][1]['adm_encoded'].to(negative[0][1]['adm_encoded']) + positive_refiner = encode_model_conds(current_refiner.model.extra_conds, positive_refiner, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask) + negative_refiner = encode_model_conds(current_refiner.model.extra_conds, negative_refiner, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask) def refiner_switch(): cleanup_additional_models(set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control"))) diff --git a/webui.py b/webui.py index 180825d..ee0975a 100644 --- a/webui.py +++ b/webui.py @@ -148,9 +148,9 @@ with shared.gradio_root: with gr.TabItem(label='Inpaint or Outpaint (beta)') as inpaint_tab: inpaint_input_image = grh.Image(label='Drag above image to here', source='upload', type='numpy', tool='sketch', height=500, brush_color="#FFFFFF", elem_id='inpaint_canvas') - gr.HTML('Outpaint Expansion (\U0001F4D4 Document):') + gr.HTML('Outpaint Expansion Direction:') outpaint_selections = gr.CheckboxGroup(choices=['Left', 'Right', 'Top', 'Bottom'], value=[], label='Outpaint', show_label=False, container=False) - gr.HTML('* \"Inpaint or Outpaint\" is powered by the sampler \"DPMPP Fooocus Seamless 2M SDE Karras Inpaint Sampler\" (beta)') + gr.HTML('* Powered by Fooocus Inpaint Engine (beta) \U0001F4D4 Document') switch_js = "(x) => {if(x){setTimeout(() => window.scrollTo({ top: 850, behavior: 'smooth' }), 50);}else{setTimeout(() => window.scrollTo({ top: 0, behavior: 'smooth' }), 50);} return x}" down_js = "() => {setTimeout(() => window.scrollTo({ top: 850, behavior: 'smooth' }), 50);}" From a6ac4604cda93f2ab91a5606c6fed33565636bbf Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 25 Oct 2023 11:18:16 -0700 Subject: [PATCH 31/78] remove unused codes --- modules/default_pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index b32f811..8a4d255 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -11,7 +11,6 @@ import fooocus_extras.vae_interpose as vae_interpose from fcbh.model_base import SDXL, SDXLRefiner from modules.expansion import FooocusExpansion from modules.sample_hijack import clip_separate -from fcbh.k_diffusion.sampling import BrownianTreeNoiseSampler xl_base: core.StableDiffusionModel = None From 01b1e98d378e7ceed08171c0397b2e5d89ea0047 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 25 Oct 2023 12:06:05 -0700 Subject: [PATCH 32/78] Update Styles Update Styles --- fooocus_version.py | 2 +- modules/path.py | 21 ++++++++++++--------- presets/anime.json | 4 ++-- presets/realistic.json | 4 ++-- presets/sdxl.json | 24 ------------------------ sdxl_styles/sdxl_styles_fooocus.json | 16 +++++++++++++--- update_log.md | 13 +++++++++++++ webui.py | 8 ++++---- 8 files changed, 47 insertions(+), 45 deletions(-) delete mode 100644 presets/sdxl.json diff --git a/fooocus_version.py b/fooocus_version.py index 5f62795..daaf61a 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.740' +version = '2.1.741' diff --git a/modules/path.py b/modules/path.py index 90860ff..cd79c6d 100644 --- a/modules/path.py +++ b/modules/path.py @@ -9,6 +9,7 @@ from modules.util import get_files_from_folder config_path = "user_path_config.txt" config_dict = {} +visited_keys = [] try: if os.path.exists(config_path): @@ -37,7 +38,8 @@ if preset is not None: def get_dir_or_set_default(key, default_value): - global config_dict + global config_dict, visited_keys + visited_keys.append(key) v = config_dict.get(key, None) if isinstance(v, str) and os.path.exists(v) and os.path.isdir(v): return v @@ -62,7 +64,8 @@ temp_outputs_path = get_dir_or_set_default('temp_outputs_path', '../outputs/') def get_config_item_or_set_default(key, default_value, validator, disable_empty_as_none=False): - global config_dict + global config_dict, visited_keys + visited_keys.append(key) if key not in config_dict: config_dict[key] = default_value return default_value @@ -115,17 +118,17 @@ default_scheduler = get_config_item_or_set_default( ) default_styles = get_config_item_or_set_default( key='default_styles', - default_value=['Fooocus V2', 'Default (Slightly Cinematic)'], + default_value=['Fooocus V2', 'Fooocus Enhance', 'Fooocus Sharp'], validator=lambda x: isinstance(x, list) and all(y in modules.sdxl_styles.legal_style_names for y in x) ) -default_negative_prompt = get_config_item_or_set_default( - key='default_negative_prompt', - default_value='low quality, bad hands, bad eyes, cropped, missing fingers, extra digit', +default_prompt_negative = get_config_item_or_set_default( + key='default_prompt_negative', + default_value='', validator=lambda x: isinstance(x, str), disable_empty_as_none=True ) -default_positive_prompt = get_config_item_or_set_default( - key='default_positive_prompt', +default_prompt = get_config_item_or_set_default( + key='default_prompt', default_value='', validator=lambda x: isinstance(x, str), disable_empty_as_none=True @@ -177,7 +180,7 @@ default_aspect_ratio = get_config_item_or_set_default( if preset is None: # Do not overwrite user config if preset is applied. with open(config_path, "w", encoding="utf-8") as json_file: - json.dump(config_dict, json_file, indent=4) + json.dump({k: config_dict[k] for k in visited_keys}, json_file, indent=4) os.makedirs(temp_outputs_path, exist_ok=True) diff --git a/presets/anime.json b/presets/anime.json index a75ca18..e368c91 100644 --- a/presets/anime.json +++ b/presets/anime.json @@ -14,8 +14,8 @@ "SAI Enhance", "SAI Fantasy Art" ], - "default_negative_prompt": "(embedding:unaestheticXLv31:0.8), low quality, watermark", - "default_positive_prompt": "1girl, ", + "default_prompt_negative": "(embedding:unaestheticXLv31:0.8), low quality, watermark", + "default_prompt": "1girl, ", "checkpoint_downloads": { "bluePencilXL_v050.safetensors": "https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/bluePencilXL_v050.safetensors", "DreamShaper_8_pruned.safetensors": "https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/DreamShaper_8_pruned.safetensors" diff --git a/presets/realistic.json b/presets/realistic.json index 14b1979..caf78a9 100644 --- a/presets/realistic.json +++ b/presets/realistic.json @@ -11,8 +11,8 @@ "Fooocus Photograph", "Fooocus Negative" ], - "default_negative_prompt": "unrealistic, saturated, high contrast, big nose, painting, drawing, sketch, cartoon, anime, manga, render, CG, 3d, watermark, signature, label", - "default_positive_prompt": "", + "default_prompt_negative": "unrealistic, saturated, high contrast, big nose, painting, drawing, sketch, cartoon, anime, manga, render, CG, 3d, watermark, signature, label", + "default_prompt": "", "checkpoint_downloads": { "realisticStockPhoto_v10.safetensors": "https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/realisticStockPhoto_v10.safetensors" }, diff --git a/presets/sdxl.json b/presets/sdxl.json deleted file mode 100644 index 141ca04..0000000 --- a/presets/sdxl.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "default_model": "sd_xl_base_1.0_0.9vae.safetensors", - "default_refiner": "sd_xl_refiner_1.0_0.9vae.safetensors", - "default_lora": "sd_xl_offset_example-lora_1.0.safetensors", - "default_lora_weight": 0.5, - "default_cfg_scale": 7.0, - "default_sampler": "dpmpp_2m_sde_gpu", - "default_scheduler": "karras", - "default_styles": [ - "Fooocus V2", - "Default (Slightly Cinematic)" - ], - "default_negative_prompt": "low quality, bad hands, bad eyes, cropped, missing fingers, extra digit", - "default_positive_prompt": "", - "checkpoint_downloads": { - "sd_xl_base_1.0_0.9vae.safetensors": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0_0.9vae.safetensors", - "sd_xl_refiner_1.0_0.9vae.safetensors": "https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0_0.9vae.safetensors" - }, - "embeddings_downloads": {}, - "default_aspect_ratio": "1152*896", - "lora_downloads": { - "sd_xl_offset_example-lora_1.0.safetensors": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_offset_example-lora_1.0.safetensors" - } -} diff --git a/sdxl_styles/sdxl_styles_fooocus.json b/sdxl_styles/sdxl_styles_fooocus.json index ae9150e..b6e816d 100644 --- a/sdxl_styles/sdxl_styles_fooocus.json +++ b/sdxl_styles/sdxl_styles_fooocus.json @@ -1,8 +1,13 @@ [ { - "name": "Default (Slightly Cinematic)", - "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy", - "negative_prompt": "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured" + "name": "Fooocus Enhance", + "prompt": "{prompt} . (perfect real extremely details), award-winning, breathtaking, amazing fine detail, dramatic lighting, best quality", + "negative_prompt": "(worst quality, low quality, normal quality, lowres, low details, oversaturated, undersaturated, overexposed, underexposed, grayscale, bw, bad photo, bad photography, bad art:1.4), (watermark, signature, text font, username, error, logo, words, letters, digits, autograph, trademark, name:1.2), (blur, blurry, grainy), morbid, ugly, asymmetrical, mutated malformed, mutilated, poorly lit, bad shadow, draft, cropped, out of frame, cut off, censored, jpeg artifacts, out of focus, glitch, duplicate, (airbrushed, cartoon, anime, semi-realistic, cgi, render, blender, digital art, manga, amateur:1.3), (3D ,3D Game, 3D Game Scene, 3D Character:1.1), (bad hands, bad anatomy, bad body, bad face, bad teeth, bad arms, bad legs, deformities:1.3)" + }, + { + "name": "Fooocus Sharp", + "prompt": "cinematic still {prompt} . sharp focus, emotional, harmonious, vignette, 4k epic detailed photograph shot on kodak detailed cinematic hbo dark moody, 35mm photo, high budget, cinemascope, moody, epic, gorgeous, film grain, grainy", + "negative_prompt": "blurry, anime, cartoon, blured background, graphic, bokeh, background blur, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured" }, { "name": "Fooocus Masterpiece", @@ -18,5 +23,10 @@ "name": "Fooocus Negative", "prompt": "", "negative_prompt": "deformed, bad anatomy, disfigured, poorly drawn face, mutated, extra limb, ugly, poorly drawn hands, missing limb, floating limbs, disconnected limbs, disconnected head, malformed hands, long neck, mutated hands and fingers, bad hands, missing fingers, cropped, worst quality, low quality, mutation, poorly drawn, huge calf, bad hands, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, missing fingers, fused fingers, abnormal eye proportion, Abnormal hands, abnormal legs, abnormal feet, abnormal fingers, drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly, anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch" + }, + { + "name": "Fooocus Cinematic", + "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy", + "negative_prompt": "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured" } ] diff --git a/update_log.md b/update_log.md index a90e927..df7e83a 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,16 @@ +# 2.1.741 + +Style Updates: + +* "Default (Slightly Cinematic)" as renamed to "Fooocus Cinematic". +* "Default (Slightly Cinematic)" is canceled from default style selections. +* Added "Fooocus Sharp". This style combines many CivitAI prompts that reduces SDXL blurry and improves sharpness in a relatively natural way. +* Added "Fooocus Enhance". This style mainly use the very popular [default negative prompts from JuggernautXL](https://civitai.com/models/133005) and some other enhancing words. JuggernautXL's negative prompt has been proved to be very effective in many recent image posts on CivitAI to improve JuggernautXL and many other models. +* "Fooocus Sharp" and "Fooocus Enhance" and "Fooocus V2" becomes the new default set of styles. +* Removed the default text in the "negative prompt" input area since it is not necessary now. +* You can reproduce previous results by using "Fooocus Cinematic". +* "Fooocus Sharp" and "Fooocus Enhance" may undergo minor revision in future updates. + # 2.1.739 * Added support for authentication in --share mode (via auth.json). diff --git a/webui.py b/webui.py index ee0975a..ffa6de5 100644 --- a/webui.py +++ b/webui.py @@ -68,9 +68,9 @@ with shared.gradio_root: prompt = gr.Textbox(show_label=False, placeholder="Type prompt here.", container=False, autofocus=True, elem_classes='type_row', lines=1024) - if isinstance(modules.path.default_positive_prompt, str) \ - and modules.path.default_positive_prompt != '': - shared.gradio_root.load(lambda: modules.path.default_positive_prompt, outputs=prompt) + default_prompt = modules.path.default_prompt + if isinstance(default_prompt, str) and default_prompt != '': + shared.gradio_root.load(lambda: default_prompt, outputs=prompt) with gr.Column(scale=3, min_width=0): generate_button = gr.Button(label="Generate", value="Generate", elem_classes='type_row', elem_id='generate_button', visible=True) @@ -193,7 +193,7 @@ with shared.gradio_root: image_number = gr.Slider(label='Image Number', minimum=1, maximum=32, step=1, value=modules.path.default_image_number) negative_prompt = gr.Textbox(label='Negative Prompt', show_label=True, placeholder="Type prompt here.", info='Describing what you do not want to see.', lines=2, - value=modules.path.default_negative_prompt) + value=modules.path.default_prompt_negative) seed_random = gr.Checkbox(label='Random', value=True) image_seed = gr.Textbox(label='Seed', value=0, max_lines=1, visible=False) # workaround for https://github.com/gradio-app/gradio/issues/5354 From 37ec436a3e71b9e37e880f2eba8f2c3c7656dbdf Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 25 Oct 2023 15:37:18 -0700 Subject: [PATCH 33/78] Fooocus GitHub Bot Commit This commit is generated by a GitHub bot of Fooocus --- backend/headless/fcbh/extra_samplers/uni_pc.py | 2 +- fooocus_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/headless/fcbh/extra_samplers/uni_pc.py b/backend/headless/fcbh/extra_samplers/uni_pc.py index 58e030d..9d5f0c6 100644 --- a/backend/headless/fcbh/extra_samplers/uni_pc.py +++ b/backend/headless/fcbh/extra_samplers/uni_pc.py @@ -881,7 +881,7 @@ def sample_unipc(model, noise, image, sigmas, sampling_function, max_denoise, ex model_kwargs=extra_args, ) - order = min(3, len(timesteps) - 1) + order = min(3, len(timesteps) - 2) uni_pc = UniPC(model_fn, ns, predict_x0=True, thresholding=False, noise_mask=noise_mask, masked_image=image, noise=noise, variant=variant) x = uni_pc.sample(img, timesteps=timesteps, skip_type="time_uniform", method="multistep", order=order, lower_order_final=True, callback=callback, disable_pbar=disable) x /= ns.marginal_alpha(timesteps[-1]) diff --git a/fooocus_version.py b/fooocus_version.py index daaf61a..831e5c9 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.741' +version = '2.1.742' From 959f965b77ff38d33390e8ccbaa691d9fb6c3736 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 25 Oct 2023 17:03:15 -0700 Subject: [PATCH 34/78] improve gpt2 improve gpt2 --- fooocus_version.py | 2 +- modules/expansion.py | 33 ++++++++++++++++++++++++++++++--- update_log.md | 4 ++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 831e5c9..be90b2f 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.742' +version = '2.1.743' diff --git a/modules/expansion.py b/modules/expansion.py index f65089f..2366054 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -1,7 +1,7 @@ import torch - import fcbh.model_management as model_management +from transformers.generation.logits_process import LogitsProcessorList from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed from modules.path import fooocus_expansion_path from fcbh.model_patcher import ModelPatcher @@ -16,6 +16,14 @@ fooocus_magic_split = [ ] dangrous_patterns = '[]【】()（）|:：' +black_list = ['art', 'digital', 'Ġpaint', 'painting', 'drawing', 'draw', 'drawn', + 'concept', 'illustration', 'illustrated', 'illustrate', + 'face', 'eye', 'eyes', 'hand', 'hands', + 'monster', 'artistic', 'oil', 'brush', + 'artwork', 'artworks'] + +black_list += ['Ġ' + k for k in black_list] + def safe_str(x): x = str(x) @@ -33,6 +41,15 @@ def remove_pattern(x, pattern): class FooocusExpansion: def __init__(self): self.tokenizer = AutoTokenizer.from_pretrained(fooocus_expansion_path) + self.vocab = self.tokenizer.vocab + self.logits_bias = torch.zeros((1, len(self.vocab)), dtype=torch.float32) + self.logits_bias[0, self.tokenizer.eos_token_id] = - 16.0 + # test_198 = self.tokenizer('\n', return_tensors="pt") + self.logits_bias[0, 198] = - 1024.0 + for k, v in self.vocab.items(): + if k in black_list: + self.logits_bias[0, v] = - 1024.0 + self.model = AutoModelForCausalLM.from_pretrained(fooocus_expansion_path) self.model.eval() @@ -52,6 +69,10 @@ class FooocusExpansion: self.patcher = ModelPatcher(self.model, load_device=load_device, offload_device=offload_device) print(f'Fooocus Expansion engine loaded for {load_device}, use_fp16 = {use_fp16}.') + def logits_processor(self, input_ids, scores): + self.logits_bias = self.logits_bias.to(scores) + return scores + self.logits_bias + def __call__(self, prompt, seed): if self.patcher.current_device != self.patcher.load_device: print('Fooocus Expansion loaded by itself.') @@ -66,12 +87,18 @@ class FooocusExpansion: tokenized_kwargs.data['input_ids'] = tokenized_kwargs.data['input_ids'].to(self.patcher.load_device) tokenized_kwargs.data['attention_mask'] = tokenized_kwargs.data['attention_mask'].to(self.patcher.load_device) + current_token_length = int(tokenized_kwargs.data['input_ids'].shape[1]) + max_token_length = 77 + 77 * int(float(current_token_length) / 77.0) + + logits_processor = LogitsProcessorList([self.logits_processor]) + # https://huggingface.co/blog/introducing-csearch # https://huggingface.co/docs/transformers/generation_strategies features = self.model.generate(**tokenized_kwargs, num_beams=1, - max_new_tokens=256, - do_sample=True) + max_new_tokens=max_token_length - current_token_length, + do_sample=True, + logits_processor=logits_processor) response = self.tokenizer.batch_decode(features, skip_special_tokens=True) result = response[0][len(origin):] diff --git a/update_log.md b/update_log.md index df7e83a..61e13eb 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,7 @@ +# 2.1.743 + +* Improved GPT2 by removing some tokens that may corrupt styles. + # 2.1.741 Style Updates: From 214c2ba04ce7a92a19f2a1cbb2599c7524f19962 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 25 Oct 2023 21:02:55 -0700 Subject: [PATCH 35/78] fix resize --- fooocus_version.py | 2 +- modules/async_worker.py | 8 ++++++-- modules/util.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index be90b2f..7436cd3 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.743' +version = '2.1.744' diff --git a/modules/async_worker.py b/modules/async_worker.py index 5fc925e..9ef54aa 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -31,7 +31,7 @@ def worker(): from modules.private_logger import log from modules.expansion import safe_str from modules.util import join_prompts, remove_empty_str, HWC3, resize_image, \ - get_image_shape_ceil, set_image_shape_ceil, get_shape_ceil + get_image_shape_ceil, set_image_shape_ceil, get_shape_ceil, resample_image from modules.upscaler import perform_upscale try: @@ -329,10 +329,14 @@ def worker(): f = 1.0 shape_ceil = get_shape_ceil(H * f, W * f) + if shape_ceil < 1024: print(f'[Upscale] Image is resized because it is too small.') + uov_input_image = set_image_shape_ceil(uov_input_image, 1024) shape_ceil = 1024 - uov_input_image = set_image_shape_ceil(uov_input_image, shape_ceil) + else: + uov_input_image = resample_image(uov_input_image, width=W * f, height=H * f) + image_is_super_large = shape_ceil > 2800 if 'fast' in uov_method: diff --git a/modules/util.py b/modules/util.py index c5e8d1b..1601f1f 100644 --- a/modules/util.py +++ b/modules/util.py @@ -12,7 +12,7 @@ LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.L def resample_image(im, width, height): im = Image.fromarray(im) - im = im.resize((width, height), resample=LANCZOS) + im = im.resize((int(width), int(height)), resample=LANCZOS) return np.array(im) From 175f3e3040d30645bb1ce7bcd9173613c1552e43 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 08:24:45 -0700 Subject: [PATCH 36/78] update backend + revise styles update backend + revise styles --- backend/headless/fcbh/clip_vision.py | 31 +++++++++---------- .../headless/fcbh/ldm/modules/attention.py | 9 ++++-- .../ldm/modules/sub_quadratic_attention.py | 3 +- fooocus_extras/ip_adapter.py | 18 +++++++++-- fooocus_version.py | 2 +- modules/async_worker.py | 8 ++--- modules/sdxl_styles.py | 6 ++-- sdxl_styles/sdxl_styles_fooocus.json | 6 ++-- 8 files changed, 51 insertions(+), 32 deletions(-) diff --git a/backend/headless/fcbh/clip_vision.py b/backend/headless/fcbh/clip_vision.py index b93b0da..f3c4bb6 100644 --- a/backend/headless/fcbh/clip_vision.py +++ b/backend/headless/fcbh/clip_vision.py @@ -1,5 +1,5 @@ -from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, CLIPImageProcessor, modeling_utils -from .utils import load_torch_file, transformers_convert +from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, modeling_utils +from .utils import load_torch_file, transformers_convert, common_upscale import os import torch import contextlib @@ -7,6 +7,18 @@ import contextlib import fcbh.ops import fcbh.model_patcher import fcbh.model_management +import fcbh.utils + +def clip_preprocess(image, size=224): + mean = torch.tensor([ 0.48145466,0.4578275,0.40821073], device=image.device, dtype=image.dtype) + std = torch.tensor([0.26862954,0.26130258,0.27577711], device=image.device, dtype=image.dtype) + scale = (size / min(image.shape[1], image.shape[2])) + image = torch.nn.functional.interpolate(image.movedim(-1, 1), size=(round(scale * image.shape[1]), round(scale * image.shape[2])), mode="bicubic", antialias=True) + h = (image.shape[2] - size)//2 + w = (image.shape[3] - size)//2 + image = image[:,:,h:h+size,w:w+size] + image = torch.clip((255. * image), 0, 255).round() / 255.0 + return (image - mean.view([3,1,1])) / std.view([3,1,1]) class ClipVisionModel(): def __init__(self, json_config): @@ -23,25 +35,12 @@ class ClipVisionModel(): self.model.to(self.dtype) self.patcher = fcbh.model_patcher.ModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device) - self.processor = CLIPImageProcessor(crop_size=224, - do_center_crop=True, - do_convert_rgb=True, - do_normalize=True, - do_resize=True, - image_mean=[ 0.48145466,0.4578275,0.40821073], - image_std=[0.26862954,0.26130258,0.27577711], - resample=3, #bicubic - size=224) - def load_sd(self, sd): return self.model.load_state_dict(sd, strict=False) def encode_image(self, image): - img = torch.clip((255. * image), 0, 255).round().int() - img = list(map(lambda a: a, img)) - inputs = self.processor(images=img, return_tensors="pt") fcbh.model_management.load_model_gpu(self.patcher) - pixel_values = inputs['pixel_values'].to(self.load_device) + pixel_values = clip_preprocess(image.to(self.load_device)) if self.dtype != torch.float32: precision_scope = torch.autocast diff --git a/backend/headless/fcbh/ldm/modules/attention.py b/backend/headless/fcbh/ldm/modules/attention.py index a0af385..f3e1b6e 100644 --- a/backend/headless/fcbh/ldm/modules/attention.py +++ b/backend/headless/fcbh/ldm/modules/attention.py @@ -222,9 +222,14 @@ def attention_split(q, k, v, heads, mask=None): mem_free_total = model_management.get_free_memory(q.device) + if _ATTN_PRECISION =="fp32": + element_size = 4 + else: + element_size = q.element_size() + gb = 1024 ** 3 - tensor_size = q.shape[0] * q.shape[1] * k.shape[1] * q.element_size() - modifier = 3 if q.element_size() == 2 else 2.5 + tensor_size = q.shape[0] * q.shape[1] * k.shape[1] * element_size + modifier = 3 if element_size == 2 else 2.5 mem_required = tensor_size * modifier steps = 1 diff --git a/backend/headless/fcbh/ldm/modules/sub_quadratic_attention.py b/backend/headless/fcbh/ldm/modules/sub_quadratic_attention.py index 1f07431..11d1dd4 100644 --- a/backend/headless/fcbh/ldm/modules/sub_quadratic_attention.py +++ b/backend/headless/fcbh/ldm/modules/sub_quadratic_attention.py @@ -83,7 +83,8 @@ def _summarize_chunk( ) max_score, _ = torch.max(attn_weights, -1, keepdim=True) max_score = max_score.detach() - torch.exp(attn_weights - max_score, out=attn_weights) + attn_weights -= max_score + torch.exp(attn_weights, out=attn_weights) exp_weights = attn_weights.to(value.dtype) exp_values = torch.bmm(exp_weights, value) max_score = max_score.squeeze(-1) diff --git a/fooocus_extras/ip_adapter.py b/fooocus_extras/ip_adapter.py index 0d2ca01..aeb7de2 100644 --- a/fooocus_extras/ip_adapter.py +++ b/fooocus_extras/ip_adapter.py @@ -7,6 +7,7 @@ import fcbh.ldm.modules.attention as attention from fooocus_extras.resampler import Resampler from fcbh.model_patcher import ModelPatcher +from modules.core import numpy_to_pytorch SD_V12_CHANNELS = [320] * 4 + [640] * 4 + [1280] * 4 + [1280] * 6 + [640] * 6 + [320] * 6 + [1280] * 2 @@ -144,14 +145,27 @@ def load_ip_adapter(clip_vision_path, ip_negative_path, ip_adapter_path): return +@torch.no_grad() +@torch.inference_mode() +def clip_preprocess(image): + mean = torch.tensor([0.48145466, 0.4578275, 0.40821073], device=image.device, dtype=image.dtype).view([1, 3, 1, 1]) + std = torch.tensor([0.26862954, 0.26130258, 0.27577711], device=image.device, dtype=image.dtype).view([1, 3, 1, 1]) + image = image.movedim(-1, 1) + + # https://github.com/tencent-ailab/IP-Adapter/blob/d580c50a291566bbf9fc7ac0f760506607297e6d/README.md?plain=1#L75 + B, C, H, W = image.shape + assert H == 224 and W == 224 + + return (image - mean) / std + + @torch.no_grad() @torch.inference_mode() def preprocess(img): global ip_unconds - inputs = clip_vision.processor(images=img, return_tensors="pt") fcbh.model_management.load_model_gpu(clip_vision.patcher) - pixel_values = inputs['pixel_values'].to(clip_vision.load_device) + pixel_values = clip_preprocess(numpy_to_pytorch(img).to(clip_vision.load_device)) if clip_vision.dtype != torch.float32: precision_scope = torch.autocast diff --git a/fooocus_version.py b/fooocus_version.py index 7436cd3..64de171 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.744' +version = '2.1.745' diff --git a/modules/async_worker.py b/modules/async_worker.py index 9ef54aa..26cd986 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -210,8 +210,8 @@ def worker(): if not skip_prompt_processing: - prompts = remove_empty_str([safe_str(p) for p in prompt.split('\n')], default='') - negative_prompts = remove_empty_str([safe_str(p) for p in negative_prompt.split('\n')], default='') + prompts = remove_empty_str([safe_str(p) for p in prompt.splitlines()], default='') + negative_prompts = remove_empty_str([safe_str(p) for p in negative_prompt.splitlines()], default='') prompt = prompts[0] negative_prompt = negative_prompts[0] @@ -239,8 +239,8 @@ def worker(): if use_style: for s in style_selections: p, n = apply_style(s, positive=task_prompt) - positive_basic_workloads.append(p) - negative_basic_workloads.append(n) + positive_basic_workloads += p + negative_basic_workloads += n else: positive_basic_workloads.append(task_prompt) diff --git a/modules/sdxl_styles.py b/modules/sdxl_styles.py index 6979aa9..14a4ff1 100644 --- a/modules/sdxl_styles.py +++ b/modules/sdxl_styles.py @@ -40,7 +40,9 @@ for styles_file in styles_files: try: with open(os.path.join(styles_path, styles_file), encoding='utf-8') as f: for entry in json.load(f): - name, prompt, negative_prompt = normalize_key(entry['name']), entry['prompt'], entry['negative_prompt'] + name = normalize_key(entry['name']) + prompt = entry['prompt'] if 'prompt' in entry else '' + negative_prompt = entry['negative_prompt'] if 'negative_prompt' in entry else '' styles[name] = (prompt, negative_prompt) except Exception as e: print(str(e)) @@ -53,7 +55,7 @@ legal_style_names = [fooocus_expansion] + style_keys def apply_style(style, positive): p, n = styles[style] - return p.replace('{prompt}', positive), n + return p.replace('{prompt}', positive).splitlines(), n.splitlines() def apply_wildcards(wildcard_text, rng, directory=wildcards_path): diff --git a/sdxl_styles/sdxl_styles_fooocus.json b/sdxl_styles/sdxl_styles_fooocus.json index b6e816d..68c1db2 100644 --- a/sdxl_styles/sdxl_styles_fooocus.json +++ b/sdxl_styles/sdxl_styles_fooocus.json @@ -1,13 +1,12 @@ [ { "name": "Fooocus Enhance", - "prompt": "{prompt} . (perfect real extremely details), award-winning, breathtaking, amazing fine detail, dramatic lighting, best quality", "negative_prompt": "(worst quality, low quality, normal quality, lowres, low details, oversaturated, undersaturated, overexposed, underexposed, grayscale, bw, bad photo, bad photography, bad art:1.4), (watermark, signature, text font, username, error, logo, words, letters, digits, autograph, trademark, name:1.2), (blur, blurry, grainy), morbid, ugly, asymmetrical, mutated malformed, mutilated, poorly lit, bad shadow, draft, cropped, out of frame, cut off, censored, jpeg artifacts, out of focus, glitch, duplicate, (airbrushed, cartoon, anime, semi-realistic, cgi, render, blender, digital art, manga, amateur:1.3), (3D ,3D Game, 3D Game Scene, 3D Character:1.1), (bad hands, bad anatomy, bad body, bad face, bad teeth, bad arms, bad legs, deformities:1.3)" }, { "name": "Fooocus Sharp", - "prompt": "cinematic still {prompt} . sharp focus, emotional, harmonious, vignette, 4k epic detailed photograph shot on kodak detailed cinematic hbo dark moody, 35mm photo, high budget, cinemascope, moody, epic, gorgeous, film grain, grainy", - "negative_prompt": "blurry, anime, cartoon, blured background, graphic, bokeh, background blur, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured" + "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, 4k epic detailed, shot on kodak, 35mm photo, sharp focus, cinemascope, moody, epic, gorgeous, film grain, grainy", + "negative_prompt": "anime, cartoon, graphic, (blur, blurry, bokeh), text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured" }, { "name": "Fooocus Masterpiece", @@ -21,7 +20,6 @@ }, { "name": "Fooocus Negative", - "prompt": "", "negative_prompt": "deformed, bad anatomy, disfigured, poorly drawn face, mutated, extra limb, ugly, poorly drawn hands, missing limb, floating limbs, disconnected limbs, disconnected head, malformed hands, long neck, mutated hands and fingers, bad hands, missing fingers, cropped, worst quality, low quality, mutation, poorly drawn, huge calf, bad hands, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, missing fingers, fused fingers, abnormal eye proportion, Abnormal hands, abnormal legs, abnormal feet, abnormal fingers, drawing, painting, crayon, sketch, graphite, impressionist, noisy, blurry, soft, deformed, ugly, anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch" }, { From d7ee8145799cf66fc7ff13abf8e6055ddda934ed Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 08:51:53 -0700 Subject: [PATCH 37/78] disable expansion when empty since it is not meaningful and influences image prompt --- fooocus_version.py | 2 +- modules/expansion.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index 64de171..a86d5dc 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.745' +version = '2.1.746' diff --git a/modules/expansion.py b/modules/expansion.py index 2366054..5a2fbaf 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -74,6 +74,9 @@ class FooocusExpansion: return scores + self.logits_bias def __call__(self, prompt, seed): + if prompt == '': + return '' + if self.patcher.current_device != self.patcher.load_device: print('Fooocus Expansion loaded by itself.') model_management.load_model_gpu(self.patcher) From 823fa924d30954e9d885841f3b8b26d4b57f1408 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 08:59:49 -0700 Subject: [PATCH 38/78] disable expansion when empty since it is not meaningful and influences image prompt disable expansion when empty since it is not meaningful and influences image prompt --- fooocus_version.py | 2 +- modules/async_worker.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index a86d5dc..199193a 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.746' +version = '2.1.747' diff --git a/modules/async_worker.py b/modules/async_worker.py index 26cd986..2832ca8 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -216,6 +216,10 @@ def worker(): prompt = prompts[0] negative_prompt = negative_prompts[0] + if prompt == '': + # disable expansion when empty since it is not meaningful and influences image prompt + use_expansion = False + extra_positive_prompts = prompts[1:] if len(prompts) > 1 else [] extra_negative_prompts = negative_prompts[1:] if len(negative_prompts) > 1 else [] From a16b451fd76c6055b8393a861d8d95193e21d776 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 09:51:10 -0700 Subject: [PATCH 39/78] adjust context length (#795) * adjust context length * Update sdxl_styles_fooocus.json --- fooocus_version.py | 2 +- modules/expansion.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 199193a..023bd3a 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.747' +version = '2.1.748' diff --git a/modules/expansion.py b/modules/expansion.py index 5a2fbaf..a5ea1aa 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -1,4 +1,5 @@ import torch +import math import fcbh.model_management as model_management from transformers.generation.logits_process import LogitsProcessorList @@ -91,7 +92,8 @@ class FooocusExpansion: tokenized_kwargs.data['attention_mask'] = tokenized_kwargs.data['attention_mask'].to(self.patcher.load_device) current_token_length = int(tokenized_kwargs.data['input_ids'].shape[1]) - max_token_length = 77 + 77 * int(float(current_token_length) / 77.0) + max_token_length = 75 * int(math.ceil(float(current_token_length) / 75.0)) + max_new_tokens = max_token_length - current_token_length logits_processor = LogitsProcessorList([self.logits_processor]) @@ -99,7 +101,7 @@ class FooocusExpansion: # https://huggingface.co/docs/transformers/generation_strategies features = self.model.generate(**tokenized_kwargs, num_beams=1, - max_new_tokens=max_token_length - current_token_length, + max_new_tokens=max_new_tokens, do_sample=True, logits_processor=logits_processor) From 9982d34b6c1ea8a7bac26933cae023da75e10f03 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 10:29:30 -0700 Subject: [PATCH 40/78] safer deep copy --- modules/async_worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/async_worker.py b/modules/async_worker.py index 2832ca8..9b942c7 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -243,8 +243,8 @@ def worker(): if use_style: for s in style_selections: p, n = apply_style(s, positive=task_prompt) - positive_basic_workloads += p - negative_basic_workloads += n + positive_basic_workloads = positive_basic_workloads + p + negative_basic_workloads = negative_basic_workloads + n else: positive_basic_workloads.append(task_prompt) From f6eee6252041ce71155998071f14ed5f23cfdfa1 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 10:52:05 -0700 Subject: [PATCH 41/78] fix missing text --- fooocus_version.py | 2 +- sdxl_styles/sdxl_styles_fooocus.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 023bd3a..7a78c32 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.748' +version = '2.1.749' diff --git a/sdxl_styles/sdxl_styles_fooocus.json b/sdxl_styles/sdxl_styles_fooocus.json index 68c1db2..81d6442 100644 --- a/sdxl_styles/sdxl_styles_fooocus.json +++ b/sdxl_styles/sdxl_styles_fooocus.json @@ -5,7 +5,7 @@ }, { "name": "Fooocus Sharp", - "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, 4k epic detailed, shot on kodak, 35mm photo, sharp focus, cinemascope, moody, epic, gorgeous, film grain, grainy", + "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, 4k epic detailed, shot on kodak, 35mm photo, sharp focus, high budget, cinemascope, moody, epic, gorgeous, film grain, grainy", "negative_prompt": "anime, cartoon, graphic, (blur, blurry, bokeh), text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured" }, { From 4607316c2f8771cae76440d26fc8c186f9136f6f Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 17:17:05 -0700 Subject: [PATCH 42/78] get each image during generating get each image during generating --- fooocus_version.py | 2 +- modules/async_worker.py | 36 +++++++++++++++++++++++++----------- update_log.md | 4 ++++ webui.py | 16 ++++++++++++++-- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 7a78c32..200eb84 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.749' +version = '2.1.750' diff --git a/modules/async_worker.py b/modules/async_worker.py index 9b942c7..ab8b655 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -3,10 +3,11 @@ import threading buffer = [] outputs = [] +global_results = [] def worker(): - global buffer, outputs + global buffer, outputs, global_results import traceback import numpy as np @@ -47,6 +48,20 @@ def worker(): print(f'[Fooocus] {text}') outputs.append(['preview', (number, text, None)]) + def yield_result(imgs, do_not_show_finished_images=False): + global global_results + + if not isinstance(imgs, list): + imgs = [imgs] + + global_results = global_results + imgs + + if do_not_show_finished_images: + return + + outputs.append(['results', global_results]) + return + @torch.no_grad() @torch.inference_mode() def handler(args): @@ -356,7 +371,7 @@ def worker(): if direct_return: d = [('Upscale (Fast)', '2x')] log(uov_input_image, d, single_line_number=1) - outputs.append(['results', [uov_input_image]]) + yield_result(uov_input_image, do_not_show_finished_images=True) return tiled = True @@ -408,7 +423,7 @@ def worker(): pipeline.final_unet.model.diffusion_model.in_inpaint = True if advanced_parameters.debugging_cn_preprocessor: - outputs.append(['results', inpaint_worker.current_task.visualize_mask_processing()]) + yield_result(inpaint_worker.current_task.visualize_mask_processing(), do_not_show_finished_images=True) return progressbar(13, 'VAE Inpaint encoding ...') @@ -454,7 +469,7 @@ def worker(): cn_img = HWC3(cn_img) task[0] = core.numpy_to_pytorch(cn_img) if advanced_parameters.debugging_cn_preprocessor: - outputs.append(['results', [cn_img]]) + yield_result(cn_img, do_not_show_finished_images=True) return for task in cn_tasks[flags.cn_cpds]: cn_img, cn_stop, cn_weight = task @@ -463,7 +478,7 @@ def worker(): cn_img = HWC3(cn_img) task[0] = core.numpy_to_pytorch(cn_img) if advanced_parameters.debugging_cn_preprocessor: - outputs.append(['results', [cn_img]]) + yield_result(cn_img, do_not_show_finished_images=True) return for task in cn_tasks[flags.cn_ip]: cn_img, cn_stop, cn_weight = task @@ -474,7 +489,7 @@ def worker(): task[0] = ip_adapter.preprocess(cn_img) if advanced_parameters.debugging_cn_preprocessor: - outputs.append(['results', [cn_img]]) + yield_result(cn_img, do_not_show_finished_images=True) return if len(cn_tasks[flags.cn_ip]) > 0: @@ -490,7 +505,6 @@ def worker(): advanced_parameters.freeu_s2 ) - results = [] all_steps = steps * image_number preparation_time = time.perf_counter() - execution_start_time @@ -566,7 +580,7 @@ def worker(): d.append((f'LoRA [{n}] weight', w)) log(x, d, single_line_number=3) - results += imgs + yield_result(imgs, do_not_show_finished_images=len(tasks) == 1) except fcbh.model_management.InterruptProcessingException as e: if shared.last_stop == 'skip': print('User skipped') @@ -578,8 +592,6 @@ def worker(): execution_time = time.perf_counter() - execution_start_time print(f'Generating and saving time: {execution_time:.2f} seconds') - outputs.append(['results', results]) - pipeline.prepare_text_encoder(async_call=True) return @@ -591,7 +603,9 @@ def worker(): handler(task) except: traceback.print_exc() - outputs.append(['results', []]) + if len(buffer) == 0: + outputs.append(['finish', global_results]) + global_results = [] pass diff --git a/update_log.md b/update_log.md index 61e13eb..dbfe5f8 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,7 @@ +# 2.1.750 + +* New UI: now you can get each image during generating. + # 2.1.743 * Improved GPT2 by removing some tokens that may corrupt styles. diff --git a/webui.py b/webui.py index ffa6de5..b2f7f5c 100644 --- a/webui.py +++ b/webui.py @@ -22,10 +22,13 @@ from modules.auth import auth_enabled, check_auth def generate_clicked(*args): + # outputs=[progress_html, progress_window, progress_gallery, gallery] + execution_start_time = time.perf_counter() yield gr.update(visible=True, value=modules.html.make_progress_html(1, 'Initializing ...')), \ gr.update(visible=True, value=None), \ + gr.update(visible=False, value=None), \ gr.update(visible=False) worker.buffer.append(list(args)) @@ -39,9 +42,16 @@ def generate_clicked(*args): percentage, title, image = product yield gr.update(visible=True, value=modules.html.make_progress_html(percentage, title)), \ gr.update(visible=True, value=image) if image is not None else gr.update(), \ + gr.update(), \ gr.update(visible=False) if flag == 'results': + yield gr.update(visible=True), \ + gr.update(visible=True), \ + gr.update(visible=True, value=product), \ + gr.update(visible=False) + if flag == 'finish': yield gr.update(visible=False), \ + gr.update(visible=False), \ gr.update(visible=False), \ gr.update(visible=True, value=product) finished = True @@ -60,7 +70,9 @@ shared.gradio_root = gr.Blocks( with shared.gradio_root: with gr.Row(): with gr.Column(scale=2): - progress_window = grh.Image(label='Preview', show_label=True, height=640, visible=False) + with gr.Row(): + progress_window = grh.Image(label='Preview', show_label=True, height=640, visible=False) + progress_gallery = gr.Gallery(label='Finished Images', show_label=True, object_fit='contain', height=640, visible=False) progress_html = gr.HTML(value=modules.html.make_progress_html(32, 'Progress 32%'), visible=False, elem_id='progress-bar', elem_classes='progress-bar') gallery = gr.Gallery(label='Gallery', show_label=False, object_fit='contain', height=745, visible=True, elem_classes='resizable_area') with gr.Row(elem_classes='type_row'): @@ -356,7 +368,7 @@ with shared.gradio_root: generate_button.click(lambda: (gr.update(visible=True, interactive=True), gr.update(visible=True, interactive=True), gr.update(visible=False), []), outputs=[stop_button, skip_button, generate_button, gallery]) \ .then(fn=refresh_seed, inputs=[seed_random, image_seed], outputs=image_seed) \ .then(advanced_parameters.set_all_advanced_parameters, inputs=adps) \ - .then(fn=generate_clicked, inputs=ctrls, outputs=[progress_html, progress_window, gallery]) \ + .then(fn=generate_clicked, inputs=ctrls, outputs=[progress_html, progress_window, progress_gallery, gallery]) \ .then(lambda: (gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)), outputs=[generate_button, stop_button, skip_button]) \ .then(fn=None, _js='playNotification') From 5cbefa821fd4a7dad67b73009dfcf6aced60c97a Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 18:06:25 -0700 Subject: [PATCH 43/78] minor fix --- fooocus_version.py | 2 +- webui.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index 200eb84..3743d33 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.750' +version = '2.1.751' diff --git a/webui.py b/webui.py index b2f7f5c..9143583 100644 --- a/webui.py +++ b/webui.py @@ -26,6 +26,8 @@ def generate_clicked(*args): execution_start_time = time.perf_counter() + worker.outputs = [] + yield gr.update(visible=True, value=modules.html.make_progress_html(1, 'Initializing ...')), \ gr.update(visible=True, value=None), \ gr.update(visible=False, value=None), \ From f07f7693d3180ed1ceb7db6d2c998b9c56f56162 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 19:42:50 -0700 Subject: [PATCH 44/78] fix refiner --- fooocus_version.py | 2 +- modules/async_worker.py | 7 +++---- modules/path.py | 5 +++++ presets/anime.json | 1 + webui.py | 5 ++++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 3743d33..68d233e 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.751' +version = '2.1.752' diff --git a/modules/async_worker.py b/modules/async_worker.py index ab8b655..3d2bfb5 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -80,6 +80,7 @@ def worker(): guidance_scale = args.pop() base_model_name = args.pop() refiner_model_name = args.pop() + refiner_switch = args.pop() loras = [(args.pop(), args.pop()) for _ in range(5)] input_image_checkbox = args.pop() current_tab = args.pop() @@ -147,10 +148,8 @@ def worker(): if performance_selection == 'Speed': steps = 30 - switch = 20 else: steps = 60 - switch = 40 sampler_name = advanced_parameters.sampler_name scheduler_name = advanced_parameters.scheduler_name @@ -171,10 +170,8 @@ def worker(): else: if performance_selection == 'Speed': steps = 18 - switch = 12 else: steps = 36 - switch = 24 progressbar(1, 'Downloading upscale models ...') modules.path.downloading_upscale_model() if (current_tab == 'inpaint' or (current_tab == 'ip' and advanced_parameters.mixing_image_prompt_and_inpaint))\ @@ -206,6 +203,8 @@ def worker(): pipeline.refresh_controlnets([controlnet_canny_path, controlnet_cpds_path]) ip_adapter.load_ip_adapter(clip_vision_path, ip_negative_path, ip_adapter_path) + switch = int(round(steps * refiner_switch)) + if advanced_parameters.overwrite_step > 0: steps = advanced_parameters.overwrite_step diff --git a/modules/path.py b/modules/path.py index cd79c6d..b3bd263 100644 --- a/modules/path.py +++ b/modules/path.py @@ -91,6 +91,11 @@ default_refiner_model_name = get_config_item_or_set_default( default_value='sd_xl_refiner_1.0_0.9vae.safetensors', validator=lambda x: isinstance(x, str) ) +default_refiner_switch = get_config_item_or_set_default( + key='default_refiner_switch', + default_value=0.8, + validator=lambda x: isinstance(x, float) +) default_lora_name = get_config_item_or_set_default( key='default_lora', default_value='sd_xl_offset_example-lora_1.0.safetensors', diff --git a/presets/anime.json b/presets/anime.json index e368c91..c7f84cd 100644 --- a/presets/anime.json +++ b/presets/anime.json @@ -2,6 +2,7 @@ "default_model": "bluePencilXL_v050.safetensors", "default_refiner": "DreamShaper_8_pruned.safetensors", "default_lora": "sd_xl_offset_example-lora_1.0.safetensors", + "default_refiner_switch": 0.667, "default_lora_weight": 0.5, "default_cfg_scale": 7.0, "default_sampler": "dpmpp_2m_sde_gpu", diff --git a/webui.py b/webui.py index 9143583..91c389d 100644 --- a/webui.py +++ b/webui.py @@ -253,6 +253,9 @@ with shared.gradio_root: info='Higher value means image and texture are sharper.') guidance_scale = gr.Slider(label='Guidance Scale', minimum=1.0, maximum=30.0, step=0.01, value=modules.path.default_cfg_scale, info='Higher value means style is cleaner, vivider, and more artistic.') + refiner_switch = gr.Slider(label='Refiner Switch At', minimum=0.0, maximum=1.0, step=0.0001, + info='When to switch from base model to the refiner (if refiner is used).', + value=modules.path.default_refiner_switch) gr.HTML('\U0001F4D4 Document') dev_mode = gr.Checkbox(label='Developer Debug Mode', value=False, container=False) @@ -361,7 +364,7 @@ with shared.gradio_root: performance_selection, aspect_ratios_selection, image_number, image_seed, sharpness, guidance_scale ] - ctrls += [base_model, refiner_model] + lora_ctrls + ctrls += [base_model, refiner_model, refiner_switch] + lora_ctrls ctrls += [input_image_checkbox, current_tab] ctrls += [uov_method, uov_input_image] ctrls += [outpaint_selections, inpaint_input_image] From 3445c0d7d11bf8ef05f66202e188a5c46419c2d2 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Thu, 26 Oct 2023 22:33:38 -0700 Subject: [PATCH 45/78] Announcement --- update_log.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/update_log.md b/update_log.md index dbfe5f8..4d7d807 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,5 @@ +**(2023 Oct 26) Hi all, the feature updating of Fooocus will (really, really, this time) be paused for about two or three weeks because we really have some other workloads. Thanks for the passion of you all (and we in fact have kept updating even after last pausing announcement a week ago, because of many great feedbacks) - see you soon and we will come back in mid November. However, you may still see updates if other collaborators are fixing bugs or solving problems.** + # 2.1.750 * New UI: now you can get each image during generating. From 56c8a342ec96f50fe09710be266167f3c01cbca0 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 28 Oct 2023 14:49:18 -0700 Subject: [PATCH 46/78] Fooocus GitHub Bot Commit This commit is generated by a GitHub bot of Fooocus --- backend/headless/fcbh/cldm/cldm.py | 47 ++++---- .../modules/diffusionmodules/openaimodel.py | 42 +++---- backend/headless/fcbh/lora.py | 6 +- backend/headless/fcbh/model_detection.py | 112 ++++++++++++++---- backend/headless/fcbh/sd.py | 8 +- backend/headless/fcbh/sd1_clip.py | 49 +++++++- backend/headless/fcbh/sd2_clip.py | 12 +- backend/headless/fcbh/sdxl_clip.py | 29 ++--- backend/headless/fcbh/supported_models.py | 27 ++++- backend/headless/fcbh/utils.py | 30 ++--- fooocus_version.py | 2 +- 11 files changed, 234 insertions(+), 130 deletions(-) diff --git a/backend/headless/fcbh/cldm/cldm.py b/backend/headless/fcbh/cldm/cldm.py index b177e92..d464a46 100644 --- a/backend/headless/fcbh/cldm/cldm.py +++ b/backend/headless/fcbh/cldm/cldm.py @@ -27,7 +27,6 @@ class ControlNet(nn.Module): model_channels, hint_channels, num_res_blocks, - attention_resolutions, dropout=0, channel_mult=(1, 2, 4, 8), conv_resample=True, @@ -52,6 +51,7 @@ class ControlNet(nn.Module): use_linear_in_transformer=False, adm_in_channels=None, transformer_depth_middle=None, + transformer_depth_output=None, device=None, operations=fcbh.ops, ): @@ -79,10 +79,7 @@ class ControlNet(nn.Module): self.image_size = image_size self.in_channels = in_channels self.model_channels = model_channels - if isinstance(transformer_depth, int): - transformer_depth = len(channel_mult) * [transformer_depth] - if transformer_depth_middle is None: - transformer_depth_middle = transformer_depth[-1] + if isinstance(num_res_blocks, int): self.num_res_blocks = len(channel_mult) * [num_res_blocks] else: @@ -90,18 +87,16 @@ class ControlNet(nn.Module): raise ValueError("provide num_res_blocks either as an int (globally constant) or " "as a list/tuple (per-level) with the same length as channel_mult") self.num_res_blocks = num_res_blocks + if disable_self_attentions is not None: # should be a list of booleans, indicating whether to disable self-attention in TransformerBlocks or not assert len(disable_self_attentions) == len(channel_mult) if num_attention_blocks is not None: assert len(num_attention_blocks) == len(self.num_res_blocks) assert all(map(lambda i: self.num_res_blocks[i] >= num_attention_blocks[i], range(len(num_attention_blocks)))) - print(f"Constructor of UNetModel received num_attention_blocks={num_attention_blocks}. " - f"This option has LESS priority than attention_resolutions {attention_resolutions}, " - f"i.e., in cases where num_attention_blocks[i] > 0 but 2**i not in attention_resolutions, " - f"attention will still not be set.") - self.attention_resolutions = attention_resolutions + transformer_depth = transformer_depth[:] + self.dropout = dropout self.channel_mult = channel_mult self.conv_resample = conv_resample @@ -180,11 +175,14 @@ class ControlNet(nn.Module): dims=dims, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, - operations=operations + dtype=self.dtype, + device=device, + operations=operations, ) ] ch = mult * model_channels - if ds in attention_resolutions: + num_transformers = transformer_depth.pop(0) + if num_transformers > 0: if num_head_channels == -1: dim_head = ch // num_heads else: @@ -201,9 +199,9 @@ class ControlNet(nn.Module): if not exists(num_attention_blocks) or nr < num_attention_blocks[level]: layers.append( SpatialTransformer( - ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim, + ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim, disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint, operations=operations + use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations ) ) self.input_blocks.append(TimestepEmbedSequential(*layers)) @@ -223,11 +221,13 @@ class ControlNet(nn.Module): use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, down=True, + dtype=self.dtype, + device=device, operations=operations ) if resblock_updown else Downsample( - ch, conv_resample, dims=dims, out_channels=out_ch, operations=operations + ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype, device=device, operations=operations ) ) ) @@ -245,7 +245,7 @@ class ControlNet(nn.Module): if legacy: #num_heads = 1 dim_head = ch // num_heads if use_spatial_transformer else num_head_channels - self.middle_block = TimestepEmbedSequential( + mid_block = [ ResBlock( ch, time_embed_dim, @@ -253,12 +253,15 @@ class ControlNet(nn.Module): dims=dims, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype, + device=device, operations=operations - ), - SpatialTransformer( # always uses a self-attn + )] + if transformer_depth_middle >= 0: + mid_block += [SpatialTransformer( # always uses a self-attn ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim, disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer, - use_checkpoint=use_checkpoint, operations=operations + use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations ), ResBlock( ch, @@ -267,9 +270,11 @@ class ControlNet(nn.Module): dims=dims, use_checkpoint=use_checkpoint, use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype, + device=device, operations=operations - ), - ) + )] + self.middle_block = TimestepEmbedSequential(*mid_block) self.middle_block_out = self.make_zero_conv(ch, operations=operations) self._feature_size += ch diff --git a/backend/headless/fcbh/ldm/modules/diffusionmodules/openaimodel.py b/backend/headless/fcbh/ldm/modules/diffusionmodules/openaimodel.py index d8ec0a6..9c7cfb8 100644 --- a/backend/headless/fcbh/ldm/modules/diffusionmodules/openaimodel.py +++ b/backend/headless/fcbh/ldm/modules/diffusionmodules/openaimodel.py @@ -259,10 +259,6 @@ class UNetModel(nn.Module): :param model_channels: base channel count for the model. :param out_channels: channels in the output Tensor. :param num_res_blocks: number of residual blocks per downsample. - :param attention_resolutions: a collection of downsample rates at which - attention will take place. May be a set, list, or tuple. - For example, if this contains 4, then at 4x downsampling, attention - will be used. :param dropout: the dropout probability. :param channel_mult: channel multiplier for each level of the UNet. :param conv_resample: if True, use learned convolutions for upsampling and @@ -289,7 +285,6 @@ class UNetModel(nn.Module): model_channels, out_channels, num_res_blocks, - attention_resolutions, dropout=0, channel_mult=(1, 2, 4, 8), conv_resample=True, @@ -314,6 +309,7 @@ class UNetModel(nn.Module): use_linear_in_transformer=False, adm_in_channels=None, transformer_depth_middle=None, + transformer_depth_output=None, device=None, operations=fcbh.ops, ): @@ -341,10 +337,7 @@ class UNetModel(nn.Module): self.in_channels = in_channels self.model_channels = model_channels self.out_channels = out_channels - if isinstance(transformer_depth, int): - transformer_depth = len(channel_mult) * [transformer_depth] - if transformer_depth_middle is None: - transformer_depth_middle = transformer_depth[-1] + if isinstance(num_res_blocks, int): self.num_res_blocks = len(channel_mult) * [num_res_blocks] else: @@ -352,18 +345,16 @@ class UNetModel(nn.Module): raise ValueError("provide num_res_blocks either as an int (globally constant) or " "as a list/tuple (per-level) with the same length as channel_mult") self.num_res_blocks = num_res_blocks + if disable_self_attentions is not None: # should be a list of booleans, indicating whether to disable self-attention in TransformerBlocks or not assert len(disable_self_attentions) == len(channel_mult) if num_attention_blocks is not None: assert len(num_attention_blocks) == len(self.num_res_blocks) - assert all(map(lambda i: self.num_res_blocks[i] >= num_attention_blocks[i], range(len(num_attention_blocks)))) - print(f"Constructor of UNetModel received num_attention_blocks={num_attention_blocks}. " - f"This option has LESS priority than attention_resolutions {attention_resolutions}, " - f"i.e., in cases where num_attention_blocks[i] > 0 but 2**i not in attention_resolutions, " - f"attention will still not be set.") - self.attention_resolutions = attention_resolutions + transformer_depth = transformer_depth[:] + transformer_depth_output = transformer_depth_output[:] + self.dropout = dropout self.channel_mult = channel_mult self.conv_resample = conv_resample @@ -428,7 +419,8 @@ class UNetModel(nn.Module): ) ] ch = mult * model_channels - if ds in attention_resolutions: + num_transformers = transformer_depth.pop(0) + if num_transformers > 0: if num_head_channels == -1: dim_head = ch // num_heads else: @@ -444,7 +436,7 @@ class UNetModel(nn.Module): if not exists(num_attention_blocks) or nr < num_attention_blocks[level]: layers.append(SpatialTransformer( - ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim, + ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim, disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations ) @@ -488,7 +480,7 @@ class UNetModel(nn.Module): if legacy: #num_heads = 1 dim_head = ch // num_heads if use_spatial_transformer else num_head_channels - self.middle_block = TimestepEmbedSequential( + mid_block = [ ResBlock( ch, time_embed_dim, @@ -499,8 +491,9 @@ class UNetModel(nn.Module): dtype=self.dtype, device=device, operations=operations - ), - SpatialTransformer( # always uses a self-attn + )] + if transformer_depth_middle >= 0: + mid_block += [SpatialTransformer( # always uses a self-attn ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim, disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer, use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations @@ -515,8 +508,8 @@ class UNetModel(nn.Module): dtype=self.dtype, device=device, operations=operations - ), - ) + )] + self.middle_block = TimestepEmbedSequential(*mid_block) self._feature_size += ch self.output_blocks = nn.ModuleList([]) @@ -538,7 +531,8 @@ class UNetModel(nn.Module): ) ] ch = model_channels * mult - if ds in attention_resolutions: + num_transformers = transformer_depth_output.pop() + if num_transformers > 0: if num_head_channels == -1: dim_head = ch // num_heads else: @@ -555,7 +549,7 @@ class UNetModel(nn.Module): if not exists(num_attention_blocks) or i < num_attention_blocks[level]: layers.append( SpatialTransformer( - ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim, + ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim, disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations ) diff --git a/backend/headless/fcbh/lora.py b/backend/headless/fcbh/lora.py index 4c1c568..3bec26b 100644 --- a/backend/headless/fcbh/lora.py +++ b/backend/headless/fcbh/lora.py @@ -141,9 +141,9 @@ def model_lora_keys_clip(model, key_map={}): text_model_lora_key = "lora_te_text_model_encoder_layers_{}_{}" clip_l_present = False - for b in range(32): + for b in range(32): #TODO: clean up for c in LORA_CLIP_MAP: - k = "transformer.text_model.encoder.layers.{}.{}.weight".format(b, c) + k = "clip_h.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c) if k in sdk: lora_key = text_model_lora_key.format(b, LORA_CLIP_MAP[c]) key_map[lora_key] = k @@ -154,6 +154,8 @@ def model_lora_keys_clip(model, key_map={}): k = "clip_l.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c) if k in sdk: + lora_key = text_model_lora_key.format(b, LORA_CLIP_MAP[c]) + key_map[lora_key] = k lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base key_map[lora_key] = k clip_l_present = True diff --git a/backend/headless/fcbh/model_detection.py b/backend/headless/fcbh/model_detection.py index cc3d10e..5385127 100644 --- a/backend/headless/fcbh/model_detection.py +++ b/backend/headless/fcbh/model_detection.py @@ -14,6 +14,19 @@ def count_blocks(state_dict_keys, prefix_string): count += 1 return count +def calculate_transformer_depth(prefix, state_dict_keys, state_dict): + context_dim = None + use_linear_in_transformer = False + + transformer_prefix = prefix + "1.transformer_blocks." + transformer_keys = sorted(list(filter(lambda a: a.startswith(transformer_prefix), state_dict_keys))) + if len(transformer_keys) > 0: + last_transformer_depth = count_blocks(state_dict_keys, transformer_prefix + '{}') + context_dim = state_dict['{}0.attn2.to_k.weight'.format(transformer_prefix)].shape[1] + use_linear_in_transformer = len(state_dict['{}1.proj_in.weight'.format(prefix)].shape) == 2 + return last_transformer_depth, context_dim, use_linear_in_transformer + return None + def detect_unet_config(state_dict, key_prefix, dtype): state_dict_keys = list(state_dict.keys()) @@ -40,6 +53,7 @@ def detect_unet_config(state_dict, key_prefix, dtype): channel_mult = [] attention_resolutions = [] transformer_depth = [] + transformer_depth_output = [] context_dim = None use_linear_in_transformer = False @@ -48,60 +62,67 @@ def detect_unet_config(state_dict, key_prefix, dtype): count = 0 last_res_blocks = 0 - last_transformer_depth = 0 last_channel_mult = 0 - while True: + input_block_count = count_blocks(state_dict_keys, '{}input_blocks'.format(key_prefix) + '.{}.') + for count in range(input_block_count): prefix = '{}input_blocks.{}.'.format(key_prefix, count) + prefix_output = '{}output_blocks.{}.'.format(key_prefix, input_block_count - count - 1) + block_keys = sorted(list(filter(lambda a: a.startswith(prefix), state_dict_keys))) if len(block_keys) == 0: break + block_keys_output = sorted(list(filter(lambda a: a.startswith(prefix_output), state_dict_keys))) + if "{}0.op.weight".format(prefix) in block_keys: #new layer - if last_transformer_depth > 0: - attention_resolutions.append(current_res) - transformer_depth.append(last_transformer_depth) num_res_blocks.append(last_res_blocks) channel_mult.append(last_channel_mult) current_res *= 2 last_res_blocks = 0 - last_transformer_depth = 0 last_channel_mult = 0 + out = calculate_transformer_depth(prefix_output, state_dict_keys, state_dict) + if out is not None: + transformer_depth_output.append(out[0]) + else: + transformer_depth_output.append(0) else: res_block_prefix = "{}0.in_layers.0.weight".format(prefix) if res_block_prefix in block_keys: last_res_blocks += 1 last_channel_mult = state_dict["{}0.out_layers.3.weight".format(prefix)].shape[0] // model_channels - transformer_prefix = prefix + "1.transformer_blocks." - transformer_keys = sorted(list(filter(lambda a: a.startswith(transformer_prefix), state_dict_keys))) - if len(transformer_keys) > 0: - last_transformer_depth = count_blocks(state_dict_keys, transformer_prefix + '{}') - if context_dim is None: - context_dim = state_dict['{}0.attn2.to_k.weight'.format(transformer_prefix)].shape[1] - use_linear_in_transformer = len(state_dict['{}1.proj_in.weight'.format(prefix)].shape) == 2 + out = calculate_transformer_depth(prefix, state_dict_keys, state_dict) + if out is not None: + transformer_depth.append(out[0]) + if context_dim is None: + context_dim = out[1] + use_linear_in_transformer = out[2] + else: + transformer_depth.append(0) + + res_block_prefix = "{}0.in_layers.0.weight".format(prefix_output) + if res_block_prefix in block_keys_output: + out = calculate_transformer_depth(prefix_output, state_dict_keys, state_dict) + if out is not None: + transformer_depth_output.append(out[0]) + else: + transformer_depth_output.append(0) - count += 1 - if last_transformer_depth > 0: - attention_resolutions.append(current_res) - transformer_depth.append(last_transformer_depth) num_res_blocks.append(last_res_blocks) channel_mult.append(last_channel_mult) - transformer_depth_middle = count_blocks(state_dict_keys, '{}middle_block.1.transformer_blocks.'.format(key_prefix) + '{}') - - if len(set(num_res_blocks)) == 1: - num_res_blocks = num_res_blocks[0] - - if len(set(transformer_depth)) == 1: - transformer_depth = transformer_depth[0] + if "{}middle_block.1.proj_in.weight".format(key_prefix) in state_dict_keys: + transformer_depth_middle = count_blocks(state_dict_keys, '{}middle_block.1.transformer_blocks.'.format(key_prefix) + '{}') + else: + transformer_depth_middle = -1 unet_config["in_channels"] = in_channels unet_config["model_channels"] = model_channels unet_config["num_res_blocks"] = num_res_blocks - unet_config["attention_resolutions"] = attention_resolutions unet_config["transformer_depth"] = transformer_depth + unet_config["transformer_depth_output"] = transformer_depth_output unet_config["channel_mult"] = channel_mult unet_config["transformer_depth_middle"] = transformer_depth_middle unet_config['use_linear_in_transformer'] = use_linear_in_transformer @@ -124,6 +145,45 @@ def model_config_from_unet(state_dict, unet_key_prefix, dtype, use_base_if_no_ma else: return model_config +def convert_config(unet_config): + new_config = unet_config.copy() + num_res_blocks = new_config.get("num_res_blocks", None) + channel_mult = new_config.get("channel_mult", None) + + if isinstance(num_res_blocks, int): + num_res_blocks = len(channel_mult) * [num_res_blocks] + + if "attention_resolutions" in new_config: + attention_resolutions = new_config.pop("attention_resolutions") + transformer_depth = new_config.get("transformer_depth", None) + transformer_depth_middle = new_config.get("transformer_depth_middle", None) + + if isinstance(transformer_depth, int): + transformer_depth = len(channel_mult) * [transformer_depth] + if transformer_depth_middle is None: + transformer_depth_middle = transformer_depth[-1] + t_in = [] + t_out = [] + s = 1 + for i in range(len(num_res_blocks)): + res = num_res_blocks[i] + d = 0 + if s in attention_resolutions: + d = transformer_depth[i] + + t_in += [d] * res + t_out += [d] * (res + 1) + s *= 2 + transformer_depth = t_in + transformer_depth_output = t_out + new_config["transformer_depth"] = t_in + new_config["transformer_depth_output"] = t_out + new_config["transformer_depth_middle"] = transformer_depth_middle + + new_config["num_res_blocks"] = num_res_blocks + return new_config + + def unet_config_from_diffusers_unet(state_dict, dtype): match = {} attention_resolutions = [] @@ -200,7 +260,7 @@ def unet_config_from_diffusers_unet(state_dict, dtype): matches = False break if matches: - return unet_config + return convert_config(unet_config) return None def model_config_from_diffusers_unet(state_dict, dtype): diff --git a/backend/headless/fcbh/sd.py b/backend/headless/fcbh/sd.py index 5f1f0c6..0982446 100644 --- a/backend/headless/fcbh/sd.py +++ b/backend/headless/fcbh/sd.py @@ -360,7 +360,7 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_cl from . import latent_formats model_config.latent_format = latent_formats.SD15(scale_factor=scale_factor) - model_config.unet_config = unet_config + model_config.unet_config = model_detection.convert_config(unet_config) if config['model']["target"].endswith("ImageEmbeddingConditionedLatentDiffusion"): model = model_base.SD21UNCLIP(model_config, noise_aug_config["params"], model_type=model_type) @@ -388,11 +388,13 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_cl if clip_config["target"].endswith("FrozenOpenCLIPEmbedder"): clip_target.clip = sd2_clip.SD2ClipModel clip_target.tokenizer = sd2_clip.SD2Tokenizer + clip = CLIP(clip_target, embedding_directory=embedding_directory) + w.cond_stage_model = clip.cond_stage_model.clip_h elif clip_config["target"].endswith("FrozenCLIPEmbedder"): clip_target.clip = sd1_clip.SD1ClipModel clip_target.tokenizer = sd1_clip.SD1Tokenizer - clip = CLIP(clip_target, embedding_directory=embedding_directory) - w.cond_stage_model = clip.cond_stage_model + clip = CLIP(clip_target, embedding_directory=embedding_directory) + w.cond_stage_model = clip.cond_stage_model.clip_l load_clip_weights(w, state_dict) return (fcbh.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device), clip, vae) diff --git a/backend/headless/fcbh/sd1_clip.py b/backend/headless/fcbh/sd1_clip.py index 45382b0..56beb81 100644 --- a/backend/headless/fcbh/sd1_clip.py +++ b/backend/headless/fcbh/sd1_clip.py @@ -35,7 +35,7 @@ class ClipTokenWeightEncoder: return z_empty.cpu(), first_pooled.cpu() return torch.cat(output, dim=-2).cpu(), first_pooled.cpu() -class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder): +class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder): """Uses the CLIP transformer encoder for text (from huggingface)""" LAYERS = [ "last", @@ -278,7 +278,13 @@ def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=No valid_file = None for embed_dir in embedding_directory: - embed_path = os.path.join(embed_dir, embedding_name) + embed_path = os.path.abspath(os.path.join(embed_dir, embedding_name)) + embed_dir = os.path.abspath(embed_dir) + try: + if os.path.commonpath((embed_dir, embed_path)) != embed_dir: + continue + except: + continue if not os.path.isfile(embed_path): extensions = ['.safetensors', '.pt', '.bin'] for x in extensions: @@ -336,7 +342,7 @@ def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=No embed_out = next(iter(values)) return embed_out -class SD1Tokenizer: +class SDTokenizer: def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l'): if tokenizer_path is None: tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer") @@ -448,3 +454,40 @@ class SD1Tokenizer: def untokenize(self, token_weight_pair): return list(map(lambda a: (a, self.inv_vocab[a[0]]), token_weight_pair)) + + +class SD1Tokenizer: + def __init__(self, embedding_directory=None, clip_name="l", tokenizer=SDTokenizer): + self.clip_name = clip_name + self.clip = "clip_{}".format(self.clip_name) + setattr(self, self.clip, tokenizer(embedding_directory=embedding_directory)) + + def tokenize_with_weights(self, text:str, return_word_ids=False): + out = {} + out[self.clip_name] = getattr(self, self.clip).tokenize_with_weights(text, return_word_ids) + return out + + def untokenize(self, token_weight_pair): + return getattr(self, self.clip).untokenize(token_weight_pair) + + +class SD1ClipModel(torch.nn.Module): + def __init__(self, device="cpu", dtype=None, clip_name="l", clip_model=SDClipModel, **kwargs): + super().__init__() + self.clip_name = clip_name + self.clip = "clip_{}".format(self.clip_name) + setattr(self, self.clip, clip_model(device=device, dtype=dtype, **kwargs)) + + def clip_layer(self, layer_idx): + getattr(self, self.clip).clip_layer(layer_idx) + + def reset_clip_layer(self): + getattr(self, self.clip).reset_clip_layer() + + def encode_token_weights(self, token_weight_pairs): + token_weight_pairs = token_weight_pairs[self.clip_name] + out, pooled = getattr(self, self.clip).encode_token_weights(token_weight_pairs) + return out, pooled + + def load_sd(self, sd): + return getattr(self, self.clip).load_sd(sd) diff --git a/backend/headless/fcbh/sd2_clip.py b/backend/headless/fcbh/sd2_clip.py index e5cac64..052fe9b 100644 --- a/backend/headless/fcbh/sd2_clip.py +++ b/backend/headless/fcbh/sd2_clip.py @@ -2,7 +2,7 @@ from fcbh import sd1_clip import torch import os -class SD2ClipModel(sd1_clip.SD1ClipModel): +class SD2ClipHModel(sd1_clip.SDClipModel): def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None): if layer == "penultimate": layer="hidden" @@ -12,6 +12,14 @@ class SD2ClipModel(sd1_clip.SD1ClipModel): super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype) self.empty_tokens = [[49406] + [49407] + [0] * 75] -class SD2Tokenizer(sd1_clip.SD1Tokenizer): +class SD2ClipHTokenizer(sd1_clip.SDTokenizer): def __init__(self, tokenizer_path=None, embedding_directory=None): super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1024) + +class SD2Tokenizer(sd1_clip.SD1Tokenizer): + def __init__(self, embedding_directory=None): + super().__init__(embedding_directory=embedding_directory, clip_name="h", tokenizer=SD2ClipHTokenizer) + +class SD2ClipModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, **kwargs): + super().__init__(device=device, dtype=dtype, clip_name="h", clip_model=SD2ClipHModel, **kwargs) diff --git a/backend/headless/fcbh/sdxl_clip.py b/backend/headless/fcbh/sdxl_clip.py index 2064ba4..b05005c 100644 --- a/backend/headless/fcbh/sdxl_clip.py +++ b/backend/headless/fcbh/sdxl_clip.py @@ -2,7 +2,7 @@ from fcbh import sd1_clip import torch import os -class SDXLClipG(sd1_clip.SD1ClipModel): +class SDXLClipG(sd1_clip.SDClipModel): def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None): if layer == "penultimate": layer="hidden" @@ -16,14 +16,14 @@ class SDXLClipG(sd1_clip.SD1ClipModel): def load_sd(self, sd): return super().load_sd(sd) -class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer): +class SDXLClipGTokenizer(sd1_clip.SDTokenizer): def __init__(self, tokenizer_path=None, embedding_directory=None): super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280, embedding_key='clip_g') -class SDXLTokenizer(sd1_clip.SD1Tokenizer): +class SDXLTokenizer: def __init__(self, embedding_directory=None): - self.clip_l = sd1_clip.SD1Tokenizer(embedding_directory=embedding_directory) + self.clip_l = sd1_clip.SDTokenizer(embedding_directory=embedding_directory) self.clip_g = SDXLClipGTokenizer(embedding_directory=embedding_directory) def tokenize_with_weights(self, text:str, return_word_ids=False): @@ -38,7 +38,7 @@ class SDXLTokenizer(sd1_clip.SD1Tokenizer): class SDXLClipModel(torch.nn.Module): def __init__(self, device="cpu", dtype=None): super().__init__() - self.clip_l = sd1_clip.SD1ClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype) + self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype) self.clip_l.layer_norm_hidden_state = False self.clip_g = SDXLClipG(device=device, dtype=dtype) @@ -63,21 +63,6 @@ class SDXLClipModel(torch.nn.Module): else: return self.clip_l.load_sd(sd) -class SDXLRefinerClipModel(torch.nn.Module): +class SDXLRefinerClipModel(sd1_clip.SD1ClipModel): def __init__(self, device="cpu", dtype=None): - super().__init__() - self.clip_g = SDXLClipG(device=device, dtype=dtype) - - def clip_layer(self, layer_idx): - self.clip_g.clip_layer(layer_idx) - - def reset_clip_layer(self): - self.clip_g.reset_clip_layer() - - def encode_token_weights(self, token_weight_pairs): - token_weight_pairs_g = token_weight_pairs["g"] - g_out, g_pooled = self.clip_g.encode_token_weights(token_weight_pairs_g) - return g_out, g_pooled - - def load_sd(self, sd): - return self.clip_g.load_sd(sd) + super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=SDXLClipG) diff --git a/backend/headless/fcbh/supported_models.py b/backend/headless/fcbh/supported_models.py index bb8ae21..fdd4ea4 100644 --- a/backend/headless/fcbh/supported_models.py +++ b/backend/headless/fcbh/supported_models.py @@ -38,8 +38,15 @@ class SD15(supported_models_base.BASE): if ids.dtype == torch.float32: state_dict['cond_stage_model.transformer.text_model.embeddings.position_ids'] = ids.round() + replace_prefix = {} + replace_prefix["cond_stage_model."] = "cond_stage_model.clip_l." + state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix) return state_dict + def process_clip_state_dict_for_saving(self, state_dict): + replace_prefix = {"clip_l.": "cond_stage_model."} + return utils.state_dict_prefix_replace(state_dict, replace_prefix) + def clip_target(self): return supported_models_base.ClipTarget(sd1_clip.SD1Tokenizer, sd1_clip.SD1ClipModel) @@ -62,12 +69,12 @@ class SD20(supported_models_base.BASE): return model_base.ModelType.EPS def process_clip_state_dict(self, state_dict): - state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.transformer.text_model.", 24) + state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.clip_h.transformer.text_model.", 24) return state_dict def process_clip_state_dict_for_saving(self, state_dict): replace_prefix = {} - replace_prefix[""] = "cond_stage_model.model." + replace_prefix["clip_h"] = "cond_stage_model.model" state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix) state_dict = diffusers_convert.convert_text_enc_state_dict_v20(state_dict) return state_dict @@ -104,7 +111,7 @@ class SDXLRefiner(supported_models_base.BASE): "use_linear_in_transformer": True, "context_dim": 1280, "adm_in_channels": 2560, - "transformer_depth": [0, 4, 4, 0], + "transformer_depth": [0, 0, 4, 4, 4, 4, 0, 0], } latent_format = latent_formats.SDXL @@ -139,7 +146,7 @@ class SDXL(supported_models_base.BASE): unet_config = { "model_channels": 320, "use_linear_in_transformer": True, - "transformer_depth": [0, 2, 10], + "transformer_depth": [0, 0, 2, 2, 10, 10], "context_dim": 2048, "adm_in_channels": 2816 } @@ -165,6 +172,7 @@ class SDXL(supported_models_base.BASE): replace_prefix["conditioner.embedders.0.transformer.text_model"] = "cond_stage_model.clip_l.transformer.text_model" state_dict = utils.transformers_convert(state_dict, "conditioner.embedders.1.model.", "cond_stage_model.clip_g.transformer.text_model.", 32) keys_to_replace["conditioner.embedders.1.model.text_projection"] = "cond_stage_model.clip_g.text_projection" + keys_to_replace["conditioner.embedders.1.model.text_projection.weight"] = "cond_stage_model.clip_g.text_projection" keys_to_replace["conditioner.embedders.1.model.logit_scale"] = "cond_stage_model.clip_g.logit_scale" state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix) @@ -189,5 +197,14 @@ class SDXL(supported_models_base.BASE): def clip_target(self): return supported_models_base.ClipTarget(sdxl_clip.SDXLTokenizer, sdxl_clip.SDXLClipModel) +class SSD1B(SDXL): + unet_config = { + "model_channels": 320, + "use_linear_in_transformer": True, + "transformer_depth": [0, 0, 2, 2, 4, 4], + "context_dim": 2048, + "adm_in_channels": 2816 + } -models = [SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL] + +models = [SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B] diff --git a/backend/headless/fcbh/utils.py b/backend/headless/fcbh/utils.py index 2f50c82..5a694b1 100644 --- a/backend/headless/fcbh/utils.py +++ b/backend/headless/fcbh/utils.py @@ -170,25 +170,12 @@ UNET_MAP_BASIC = { def unet_to_diffusers(unet_config): num_res_blocks = unet_config["num_res_blocks"] - attention_resolutions = unet_config["attention_resolutions"] channel_mult = unet_config["channel_mult"] - transformer_depth = unet_config["transformer_depth"] + transformer_depth = unet_config["transformer_depth"][:] + transformer_depth_output = unet_config["transformer_depth_output"][:] num_blocks = len(channel_mult) - if isinstance(num_res_blocks, int): - num_res_blocks = [num_res_blocks] * num_blocks - if isinstance(transformer_depth, int): - transformer_depth = [transformer_depth] * num_blocks - transformers_per_layer = [] - res = 1 - for i in range(num_blocks): - transformers = 0 - if res in attention_resolutions: - transformers = transformer_depth[i] - transformers_per_layer.append(transformers) - res *= 2 - - transformers_mid = unet_config.get("transformer_depth_middle", transformer_depth[-1]) + transformers_mid = unet_config.get("transformer_depth_middle", None) diffusers_unet_map = {} for x in range(num_blocks): @@ -196,10 +183,11 @@ def unet_to_diffusers(unet_config): for i in range(num_res_blocks[x]): for b in UNET_MAP_RESNET: diffusers_unet_map["down_blocks.{}.resnets.{}.{}".format(x, i, UNET_MAP_RESNET[b])] = "input_blocks.{}.0.{}".format(n, b) - if transformers_per_layer[x] > 0: + num_transformers = transformer_depth.pop(0) + if num_transformers > 0: for b in UNET_MAP_ATTENTIONS: diffusers_unet_map["down_blocks.{}.attentions.{}.{}".format(x, i, b)] = "input_blocks.{}.1.{}".format(n, b) - for t in range(transformers_per_layer[x]): + for t in range(num_transformers): for b in TRANSFORMER_BLOCKS: diffusers_unet_map["down_blocks.{}.attentions.{}.transformer_blocks.{}.{}".format(x, i, t, b)] = "input_blocks.{}.1.transformer_blocks.{}.{}".format(n, t, b) n += 1 @@ -218,7 +206,6 @@ def unet_to_diffusers(unet_config): diffusers_unet_map["mid_block.resnets.{}.{}".format(i, UNET_MAP_RESNET[b])] = "middle_block.{}.{}".format(n, b) num_res_blocks = list(reversed(num_res_blocks)) - transformers_per_layer = list(reversed(transformers_per_layer)) for x in range(num_blocks): n = (num_res_blocks[x] + 1) * x l = num_res_blocks[x] + 1 @@ -227,11 +214,12 @@ def unet_to_diffusers(unet_config): for b in UNET_MAP_RESNET: diffusers_unet_map["up_blocks.{}.resnets.{}.{}".format(x, i, UNET_MAP_RESNET[b])] = "output_blocks.{}.0.{}".format(n, b) c += 1 - if transformers_per_layer[x] > 0: + num_transformers = transformer_depth_output.pop() + if num_transformers > 0: c += 1 for b in UNET_MAP_ATTENTIONS: diffusers_unet_map["up_blocks.{}.attentions.{}.{}".format(x, i, b)] = "output_blocks.{}.1.{}".format(n, b) - for t in range(transformers_per_layer[x]): + for t in range(num_transformers): for b in TRANSFORMER_BLOCKS: diffusers_unet_map["up_blocks.{}.attentions.{}.transformer_blocks.{}.{}".format(x, i, t, b)] = "output_blocks.{}.1.transformer_blocks.{}.{}".format(n, t, b) if i == l - 1: diff --git a/fooocus_version.py b/fooocus_version.py index 68d233e..d534bf7 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.752' +version = '2.1.753' From 76120e045e996ce2aaf208ddf9e01f1069e1a273 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 28 Oct 2023 16:20:11 -0700 Subject: [PATCH 47/78] update model list Note that this only influence new users with new downloads Previous users will not be forced to download new files, because this is not friendly and should be avoided. delete user_path_config.txt to receive this new list --- colab_fix.txt | 1 - fooocus_colab.ipynb | 3 +-- fooocus_version.py | 2 +- modules/path.py | 19 +++++++++++-------- readme.md | 22 ++++++++++++++++++---- webui.py | 2 +- 6 files changed, 32 insertions(+), 17 deletions(-) delete mode 100644 colab_fix.txt diff --git a/colab_fix.txt b/colab_fix.txt deleted file mode 100644 index 7b2445c..0000000 --- a/colab_fix.txt +++ /dev/null @@ -1 +0,0 @@ -{"default_refiner": ""} \ No newline at end of file diff --git a/fooocus_colab.ipynb b/fooocus_colab.ipynb index 469573c..205dac5 100644 --- a/fooocus_colab.ipynb +++ b/fooocus_colab.ipynb @@ -12,8 +12,7 @@ "%cd /content\n", "!git clone https://github.com/lllyasviel/Fooocus.git\n", "%cd /content/Fooocus\n", - "!cp colab_fix.txt user_path_config.txt\n", - "!python entry_with_update.py --preset realistic --share\n" + "!python entry_with_update.py --share\n" ] } ], diff --git a/fooocus_version.py b/fooocus_version.py index d534bf7..a1baba2 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.753' +version = '2.1.754' diff --git a/modules/path.py b/modules/path.py index b3bd263..0722468 100644 --- a/modules/path.py +++ b/modules/path.py @@ -83,12 +83,12 @@ def get_config_item_or_set_default(key, default_value, validator, disable_empty_ default_base_model_name = get_config_item_or_set_default( key='default_model', - default_value='sd_xl_base_1.0_0.9vae.safetensors', + default_value='juggernautXL_version6Rundiffusion.safetensors', validator=lambda x: isinstance(x, str) ) default_refiner_model_name = get_config_item_or_set_default( key='default_refiner', - default_value='sd_xl_refiner_1.0_0.9vae.safetensors', + default_value='None', validator=lambda x: isinstance(x, str) ) default_refiner_switch = get_config_item_or_set_default( @@ -103,12 +103,17 @@ default_lora_name = get_config_item_or_set_default( ) default_lora_weight = get_config_item_or_set_default( key='default_lora_weight', - default_value=0.5, + default_value=0.1, validator=lambda x: isinstance(x, float) ) default_cfg_scale = get_config_item_or_set_default( key='default_cfg_scale', - default_value=7.0, + default_value=4.0, + validator=lambda x: isinstance(x, float) +) +default_sample_sharpness = get_config_item_or_set_default( + key='default_sample_sharpness', + default_value=2, validator=lambda x: isinstance(x, float) ) default_sampler = get_config_item_or_set_default( @@ -151,10 +156,8 @@ default_image_number = get_config_item_or_set_default( checkpoint_downloads = get_config_item_or_set_default( key='checkpoint_downloads', default_value={ - 'sd_xl_base_1.0_0.9vae.safetensors': - 'https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0_0.9vae.safetensors', - 'sd_xl_refiner_1.0_0.9vae.safetensors': - 'https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0_0.9vae.safetensors' + 'juggernautXL_version6Rundiffusion.safetensors': + 'https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/juggernautXL_version6Rundiffusion.safetensors' }, validator=lambda x: isinstance(x, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in x.items()) ) diff --git a/readme.md b/readme.md index 296e29d..9b09de2 100644 --- a/readme.md +++ b/readme.md @@ -67,9 +67,8 @@ After you download the file, please uncompress it, and then run the "run.bat". In the first time you launch the software, it will automatically download models: -1. It will download [sd_xl_base_1.0_0.9vae.safetensors from here](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0_0.9vae.safetensors) as the file "Fooocus\models\checkpoints\sd_xl_base_1.0_0.9vae.safetensors". -2. It will download [sd_xl_refiner_1.0_0.9vae.safetensors from here](https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0_0.9vae.safetensors) as the file "Fooocus\models\checkpoints\sd_xl_refiner_1.0_0.9vae.safetensors". -3. Note that if you use inpaint, at the first time you inpaint an image, it will download [Fooocus's own inpaint control model from here](https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/inpaint.fooocus.patch) as the file "Fooocus\models\inpaint\inpaint.fooocus.patch" (the size of this file is 1.28GB). +1. It will download [default models](#models) to the folder "Fooocus\models\checkpoints" given different presets. You can download them in advance if you do not want automatic download. +2. Note that if you use inpaint, at the first time you inpaint an image, it will download [Fooocus's own inpaint control model from here](https://huggingface.co/lllyasviel/fooocus_inpaint/resolve/main/inpaint.fooocus.patch) as the file "Fooocus\models\inpaint\inpaint.fooocus.patch" (the size of this file is 1.28GB). After Fooocus 2.1.60, you will also have `run_anime.bat` and `run_realistic.bat`. They are different model presets (and requires different models, but thet will be automatically downloaded). [Check here for more details](https://github.com/lllyasviel/Fooocus/discussions/679). @@ -122,7 +121,7 @@ If you want to use Anaconda/Miniconda, you can conda activate fooocus pip install pygit2==1.12.2 -Then download the models: download [sd_xl_base_1.0_0.9vae.safetensors from here](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0_0.9vae.safetensors) as the file "Fooocus\models\checkpoints\sd_xl_base_1.0_0.9vae.safetensors", and download [sd_xl_refiner_1.0_0.9vae.safetensors from here](https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0_0.9vae.safetensors) as the file "Fooocus\models\checkpoints\sd_xl_refiner_1.0_0.9vae.safetensors". **Or let Fooocus automatically download the models** using the launcher: +Then download the models: download [default models](#models) to the folder "Fooocus\models\checkpoints". **Or let Fooocus automatically download the models** using the launcher: conda activate fooocus python entry_with_update.py @@ -217,6 +216,21 @@ You can install Fooocus on Apple Mac silicon (M1 or M2) with macOS 'Catalina' or Use `python entry_with_update.py --preset anime` or `python entry_with_update.py --preset realistic` for Fooocus Anime/Realistic Edition. +## Default Models + + +Given different goals, the default models and configs of Fooocus is different: + +| Task | Windows | Linux args | Main Model | Refiner | Config | +| - | - | - | - | - | - | +| General | run.bat | | [juggernautXL v6](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/juggernautXL_version6Rundiffusion.safetensors) | not used | [here](https://github.com/lllyasviel/Fooocus/blob/main/modules/path.py) | +| Realistic | run_realistic.bat | --preset realistic | [realistic_stock_photo](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/realisticStockPhoto_v10.safetensors) | not used | [here](https://github.com/lllyasviel/Fooocus/blob/main/presets/realistic.json) | +| Anime | run_realistic.bat | --preset anime | [bluepencil_v50](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/bluePencilXL_v050.safetensors) | [dreamsharper_v8](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/DreamShaper_8_pruned.safetensors) (SD1.5) | [here](https://github.com/lllyasviel/Fooocus/blob/main/presets/anime.json) | + +Note that the download is **automatic** - you do not need to do anything if the internet connection is okay. However, you can download them manually if you (or move them from somewhere else) have your own preparation. + +Note that if your local parameters are not same with this list, then it means your Fooocus is downloaded from a relatively old version and we do not force users to re-download models. If you want Fooocus to download new models for you, you can delete `Fooocus\user_path_config.txt` and your Fooocus' default model list and configs will be refreshed as the newest version, then all newer models will be downloaded for you. + ## List of "Hidden" Tricks diff --git a/webui.py b/webui.py index 91c389d..f8a964d 100644 --- a/webui.py +++ b/webui.py @@ -249,7 +249,7 @@ with shared.gradio_root: with gr.Row(): model_refresh = gr.Button(label='Refresh', value='\U0001f504 Refresh All Files', variant='secondary', elem_classes='refresh_button') with gr.Tab(label='Advanced'): - sharpness = gr.Slider(label='Sampling Sharpness', minimum=0.0, maximum=30.0, step=0.001, value=2.0, + sharpness = gr.Slider(label='Sampling Sharpness', minimum=0.0, maximum=30.0, step=0.001, value=modules.path.default_sample_sharpness, info='Higher value means image and texture are sharper.') guidance_scale = gr.Slider(label='Guidance Scale', minimum=1.0, maximum=30.0, step=0.01, value=modules.path.default_cfg_scale, info='Higher value means style is cleaner, vivider, and more artistic.') From 809d4a88327ef9c7d9db7bbbab0fe2aba58eff37 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 28 Oct 2023 16:40:41 -0700 Subject: [PATCH 48/78] Update readme.md (#807) --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 9b09de2..bcf5b82 100644 --- a/readme.md +++ b/readme.md @@ -59,7 +59,7 @@ Fooocus also developed many "fooocus-only" features for advanced users to get pe You can directly download Fooocus with: -**[>>> Click here to download <<<](https://github.com/lllyasviel/Fooocus/releases/download/release/Fooocus_win64_2-1-60.7z)** +**[>>> Click here to download <<<](https://github.com/lllyasviel/Fooocus/releases/download/release/Fooocus_win64_2-1-754.7z)** After you download the file, please uncompress it, and then run the "run.bat". From fa3f1709cff359c0a8b1fd71a913bada12bc8966 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 28 Oct 2023 17:07:20 -0700 Subject: [PATCH 49/78] ui --- fooocus_version.py | 2 +- webui.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index a1baba2..aa8214a 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.754' +version = '2.1.755' diff --git a/webui.py b/webui.py index f8a964d..cf75be0 100644 --- a/webui.py +++ b/webui.py @@ -239,6 +239,15 @@ with shared.gradio_root: with gr.Row(): base_model = gr.Dropdown(label='Base Model (SDXL only)', choices=modules.path.model_filenames, value=modules.path.default_base_model_name, show_label=True) refiner_model = gr.Dropdown(label='Refiner (SDXL or SD 1.5)', choices=['None'] + modules.path.model_filenames, value=modules.path.default_refiner_model_name, show_label=True) + + refiner_switch = gr.Slider(label='Refiner Switch At', minimum=0.0, maximum=1.0, step=0.0001, + info='When to switch from the base model to refiner.', + value=modules.path.default_refiner_switch, + visible=modules.path.default_refiner_model_name != 'None') + + refiner_model.change(lambda x: gr.update(visible=x != 'None'), + inputs=refiner_model, outputs=refiner_switch, show_progress=False, queue=False) + with gr.Accordion(label='LoRAs', open=True): lora_ctrls = [] for i in range(5): @@ -253,10 +262,6 @@ with shared.gradio_root: info='Higher value means image and texture are sharper.') guidance_scale = gr.Slider(label='Guidance Scale', minimum=1.0, maximum=30.0, step=0.01, value=modules.path.default_cfg_scale, info='Higher value means style is cleaner, vivider, and more artistic.') - refiner_switch = gr.Slider(label='Refiner Switch At', minimum=0.0, maximum=1.0, step=0.0001, - info='When to switch from base model to the refiner (if refiner is used).', - value=modules.path.default_refiner_switch) - gr.HTML('\U0001F4D4 Document') dev_mode = gr.Checkbox(label='Developer Debug Mode', value=False, container=False) From 48bcc7a5fc79242f0724ac7f1b2eee5615b69060 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 28 Oct 2023 18:31:50 -0700 Subject: [PATCH 50/78] fix error at 0 --- fooocus_version.py | 2 +- webui.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index aa8214a..a0c9508 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.755' +version = '2.1.756' diff --git a/webui.py b/webui.py index cf75be0..206edcc 100644 --- a/webui.py +++ b/webui.py @@ -240,7 +240,7 @@ with shared.gradio_root: base_model = gr.Dropdown(label='Base Model (SDXL only)', choices=modules.path.model_filenames, value=modules.path.default_base_model_name, show_label=True) refiner_model = gr.Dropdown(label='Refiner (SDXL or SD 1.5)', choices=['None'] + modules.path.model_filenames, value=modules.path.default_refiner_model_name, show_label=True) - refiner_switch = gr.Slider(label='Refiner Switch At', minimum=0.0, maximum=1.0, step=0.0001, + refiner_switch = gr.Slider(label='Refiner Switch At', minimum=0.1, maximum=1.0, step=0.0001, info='When to switch from the base model to refiner.', value=modules.path.default_refiner_switch, visible=modules.path.default_refiner_model_name != 'None') From b103f88f1859ed6cf2c2f0ea6c32bdb367111197 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 28 Oct 2023 21:09:25 -0700 Subject: [PATCH 51/78] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index bcf5b82..e09d769 100644 --- a/readme.md +++ b/readme.md @@ -225,7 +225,7 @@ Given different goals, the default models and configs of Fooocus is different: | - | - | - | - | - | - | | General | run.bat | | [juggernautXL v6](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/juggernautXL_version6Rundiffusion.safetensors) | not used | [here](https://github.com/lllyasviel/Fooocus/blob/main/modules/path.py) | | Realistic | run_realistic.bat | --preset realistic | [realistic_stock_photo](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/realisticStockPhoto_v10.safetensors) | not used | [here](https://github.com/lllyasviel/Fooocus/blob/main/presets/realistic.json) | -| Anime | run_realistic.bat | --preset anime | [bluepencil_v50](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/bluePencilXL_v050.safetensors) | [dreamsharper_v8](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/DreamShaper_8_pruned.safetensors) (SD1.5) | [here](https://github.com/lllyasviel/Fooocus/blob/main/presets/anime.json) | +| Anime | run_anime.bat | --preset anime | [bluepencil_v50](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/bluePencilXL_v050.safetensors) | [dreamsharper_v8](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/DreamShaper_8_pruned.safetensors) (SD1.5) | [here](https://github.com/lllyasviel/Fooocus/blob/main/presets/anime.json) | Note that the download is **automatic** - you do not need to do anything if the internet connection is okay. However, you can download them manually if you (or move them from somewhere else) have your own preparation. From 759bfadefacb1d7827843c35cf22f5996508bb2e Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sat, 28 Oct 2023 23:03:35 -0700 Subject: [PATCH 52/78] fix async load order + image wall --- fooocus_version.py | 2 +- modules/async_worker.py | 44 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index a0c9508..6d4b443 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.756' +version = '2.1.757' diff --git a/modules/async_worker.py b/modules/async_worker.py index 3d2bfb5..1db93de 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -10,6 +10,7 @@ def worker(): global buffer, outputs, global_results import traceback + import math import numpy as np import torch import time @@ -62,6 +63,46 @@ def worker(): outputs.append(['results', global_results]) return + def build_image_wall(): + global global_results + + if len(global_results) < 2: + return + + for img in global_results: + if not isinstance(img, np.ndarray): + return + if img.ndim != 3: + return + + H, W, C = global_results[0].shape + + for img in global_results: + Hn, Wn, Cn = img.shape + if H != Hn: + return + if W != Wn: + return + if C != Cn: + return + + cols = float(len(global_results)) ** 0.5 + cols = int(math.ceil(cols)) + rows = float(len(global_results)) / float(cols) + rows = int(math.ceil(rows)) + + wall = np.zeros(shape=(H * rows, W * cols, C), dtype=np.uint8) + + for y in range(rows): + for x in range(cols): + if y * cols + x < len(global_results): + img = global_results[y * cols + x] + wall[y * H:y * H + H, x * W:x * W + W, :] = img + + # must use deep copy otherwise gradio is super laggy. Do not use list.append() . + global_results = global_results + [wall] + return + @torch.no_grad() @torch.inference_mode() def handler(args): @@ -591,7 +632,6 @@ def worker(): execution_time = time.perf_counter() - execution_start_time print(f'Generating and saving time: {execution_time:.2f} seconds') - pipeline.prepare_text_encoder(async_call=True) return while True: @@ -603,8 +643,10 @@ def worker(): except: traceback.print_exc() if len(buffer) == 0: + build_image_wall() outputs.append(['finish', global_results]) global_results = [] + pipeline.prepare_text_encoder(async_call=True) pass From 49b94fa26eb7850bc8bb9573d15f075b6137d6ba Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 29 Oct 2023 17:31:14 +0100 Subject: [PATCH 53/78] use state instead of global for default_image (#812) * use state instead of global for default_image solves issue https://github.com/lllyasviel/Fooocus/issues/716 * fix --------- Co-authored-by: lllyasviel --- fooocus_version.py | 2 +- webui.py | 31 +++++++++---------------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 6d4b443..cd0bf51 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.757' +version = '2.1.758' diff --git a/webui.py b/webui.py index 206edcc..824c8c3 100644 --- a/webui.py +++ b/webui.py @@ -172,31 +172,18 @@ with shared.gradio_root: input_image_checkbox.change(lambda x: gr.update(visible=x), inputs=input_image_checkbox, outputs=image_input_panel, queue=False, _js=switch_js) ip_advanced.change(lambda: None, queue=False, _js=down_js) - current_tab = gr.Textbox(value='uov', visible=False) + current_tab = gr.State(value='uov') + default_image = gr.State(value=None) - default_image = None + lambda_img = lambda x: x['image'] if isinstance(x, dict) else x + uov_input_image.upload(lambda_img, inputs=uov_input_image, outputs=default_image, queue=False) + inpaint_input_image.upload(lambda_img, inputs=inpaint_input_image, outputs=default_image, queue=False) - def update_default_image(x): - global default_image - if isinstance(x, dict): - default_image = x['image'] - else: - default_image = x - return + uov_input_image.clear(lambda: None, outputs=default_image, queue=False) + inpaint_input_image.clear(lambda: None, outputs=default_image, queue=False) - def clear_default_image(): - global default_image - default_image = None - return - - uov_input_image.upload(update_default_image, inputs=uov_input_image, queue=False) - inpaint_input_image.upload(update_default_image, inputs=inpaint_input_image, queue=False) - - uov_input_image.clear(clear_default_image, queue=False) - inpaint_input_image.clear(clear_default_image, queue=False) - - uov_tab.select(lambda: ['uov', default_image], outputs=[current_tab, uov_input_image], queue=False, _js=down_js) - inpaint_tab.select(lambda: ['inpaint', default_image], outputs=[current_tab, inpaint_input_image], queue=False, _js=down_js) + uov_tab.select(lambda x: ['uov', x], inputs=default_image, outputs=[current_tab, uov_input_image], queue=False, _js=down_js) + inpaint_tab.select(lambda x: ['inpaint', x], inputs=default_image, outputs=[current_tab, inpaint_input_image], queue=False, _js=down_js) ip_tab.select(lambda: 'ip', outputs=[current_tab], queue=False, _js=down_js) with gr.Column(scale=1, visible=modules.path.default_advanced_checkbox) as advanced_column: From cb4859fdd8c526af8bf2942c36ac9a80f30540ae Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 11:04:22 -0700 Subject: [PATCH 54/78] Update readme.md --- readme.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index e09d769..468900e 100644 --- a/readme.md +++ b/readme.md @@ -1,9 +1,15 @@
- + + +**Non-cherry-picked** random batch by just typing two words "forest elf" without tweaking any parameters or any strange prompt tags. + +See also **non-cherry-picked** generalization and diversity tests [here](https://github.com/lllyasviel/Fooocus/discussions/808) and [here](https://github.com/lllyasviel/Fooocus/discussions/679) and [here](https://github.com/lllyasviel/Fooocus/discussions/679#realistic). + +In the entire open source community, only Fooocus can achieve this level of **non-cherry-picked** quality. -*(Screenshot of Fooocus Realistic "run_realistic.bat" using default parameters without any manual tweaking)*
+ # Fooocus Fooocus is an image generating software (based on [Gradio](https://www.gradio.app/)). From 932f1333e4231605c1a7966772b36bf9c400aece Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 11:05:26 -0700 Subject: [PATCH 55/78] Update readme.md --- readme.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 468900e..dfbf0fb 100644 --- a/readme.md +++ b/readme.md @@ -1,7 +1,9 @@
-**Non-cherry-picked** random batch by just typing two words "forest elf" without tweaking any parameters or any strange prompt tags. +**Non-cherry-picked** random batch by just typing two words "forest elf", + +without tweaking any parameters, without any strange prompt tags. See also **non-cherry-picked** generalization and diversity tests [here](https://github.com/lllyasviel/Fooocus/discussions/808) and [here](https://github.com/lllyasviel/Fooocus/discussions/679) and [here](https://github.com/lllyasviel/Fooocus/discussions/679#realistic). From dd8bf622635b65c6a2562d06909841304f57c820 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 11:16:45 -0700 Subject: [PATCH 56/78] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index dfbf0fb..d5d02ad 100644 --- a/readme.md +++ b/readme.md @@ -3,7 +3,7 @@ **Non-cherry-picked** random batch by just typing two words "forest elf", -without tweaking any parameters, without any strange prompt tags. +without any parameter tweaking, without any strange prompt tags. See also **non-cherry-picked** generalization and diversity tests [here](https://github.com/lllyasviel/Fooocus/discussions/808) and [here](https://github.com/lllyasviel/Fooocus/discussions/679) and [here](https://github.com/lllyasviel/Fooocus/discussions/679#realistic). From 769bfc7b6dc069e85024d76a9046731e2ebbd168 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 12:38:06 -0700 Subject: [PATCH 57/78] hint --- webui.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webui.py b/webui.py index 824c8c3..5b6aa2b 100644 --- a/webui.py +++ b/webui.py @@ -228,7 +228,10 @@ with shared.gradio_root: refiner_model = gr.Dropdown(label='Refiner (SDXL or SD 1.5)', choices=['None'] + modules.path.model_filenames, value=modules.path.default_refiner_model_name, show_label=True) refiner_switch = gr.Slider(label='Refiner Switch At', minimum=0.1, maximum=1.0, step=0.0001, - info='When to switch from the base model to refiner.', + info='Use 0.4 for SD1.5 realistic models; ' + 'or 0.667 for SD1.5 anime models; ' + 'or 0.8 for XL-refiners; ' + 'or any value for switching two SDXL models.', value=modules.path.default_refiner_switch, visible=modules.path.default_refiner_model_name != 'None') From 634420ed94c216a1f870c2bb4a990d8d632bc551 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 13:10:46 -0700 Subject: [PATCH 58/78] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index d5d02ad..055292c 100644 --- a/readme.md +++ b/readme.md @@ -231,7 +231,7 @@ Given different goals, the default models and configs of Fooocus is different: | Task | Windows | Linux args | Main Model | Refiner | Config | | - | - | - | - | - | - | -| General | run.bat | | [juggernautXL v6](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/juggernautXL_version6Rundiffusion.safetensors) | not used | [here](https://github.com/lllyasviel/Fooocus/blob/main/modules/path.py) | +| General | run.bat | | [juggernautXL v6_RunDiffusion](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/juggernautXL_version6Rundiffusion.safetensors) | not used | [here](https://github.com/lllyasviel/Fooocus/blob/main/modules/path.py) | | Realistic | run_realistic.bat | --preset realistic | [realistic_stock_photo](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/realisticStockPhoto_v10.safetensors) | not used | [here](https://github.com/lllyasviel/Fooocus/blob/main/presets/realistic.json) | | Anime | run_anime.bat | --preset anime | [bluepencil_v50](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/bluePencilXL_v050.safetensors) | [dreamsharper_v8](https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/DreamShaper_8_pruned.safetensors) (SD1.5) | [here](https://github.com/lllyasviel/Fooocus/blob/main/presets/anime.json) | From 867402e3dfb6115ba1e07c708f6995ef47baa075 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 22:03:59 -0700 Subject: [PATCH 59/78] try fix #815 --- fooocus_version.py | 2 +- modules/gradio_hijack.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index cd0bf51..d4df8c0 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.758' +version = '2.1.759' diff --git a/modules/gradio_hijack.py b/modules/gradio_hijack.py index 4fd2db5..181429e 100644 --- a/modules/gradio_hijack.py +++ b/modules/gradio_hijack.py @@ -9,6 +9,9 @@ from typing import Any, Literal import numpy as np import PIL import PIL.ImageOps +import gradio.routes +import importlib + from gradio_client import utils as client_utils from gradio_client.documentation import document, set_documentation_group from gradio_client.serializing import ImgSerializable @@ -461,3 +464,17 @@ def blk_ini(self, *args, **kwargs): Block.__init__ = blk_ini + +gradio.routes.asyncio = importlib.reload(gradio.routes.asyncio) + +if not hasattr(gradio.routes.asyncio, 'original_wait_for'): + gradio.routes.asyncio.original_wait_for = gradio.routes.asyncio.wait_for + + +def patched_wait_for(fut, timeout): + del timeout + return gradio.routes.asyncio.original_wait_for(fut, timeout=65535) + + +gradio.routes.asyncio.wait_for = patched_wait_for + From 6364bb37cf3eeeb60c6f9b57e144e621ab0ea9e1 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 22:26:49 -0700 Subject: [PATCH 60/78] another fix to #815 --- fooocus_version.py | 2 +- webui.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index d4df8c0..0450de0 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.759' +version = '2.1.760' diff --git a/webui.py b/webui.py index 5b6aa2b..2328e2e 100644 --- a/webui.py +++ b/webui.py @@ -41,6 +41,13 @@ def generate_clicked(*args): if len(worker.outputs) > 0: flag, product = worker.outputs.pop(0) if flag == 'preview': + + # help bad internet connection by skipping duplicated preview + if len(worker.outputs) > 0: # if we have the next item + if worker.outputs[0][0] == 'preview': # if the next item is also a preview + print('Skipped one preview for better internet connection.') + continue + percentage, title, image = product yield gr.update(visible=True, value=modules.html.make_progress_html(percentage, title)), \ gr.update(visible=True, value=image) if image is not None else gr.update(), \ From ce2c53f121ca8087a350a356f8c54203b5975972 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 22:36:22 -0700 Subject: [PATCH 61/78] less verbose --- webui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui.py b/webui.py index 2328e2e..00e1ef9 100644 --- a/webui.py +++ b/webui.py @@ -45,7 +45,7 @@ def generate_clicked(*args): # help bad internet connection by skipping duplicated preview if len(worker.outputs) > 0: # if we have the next item if worker.outputs[0][0] == 'preview': # if the next item is also a preview - print('Skipped one preview for better internet connection.') + # print('Skipped one preview for better internet connection.') continue percentage, title, image = product From 51ffcc47e434a902e016300778b1972ebc9784aa Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 29 Oct 2023 22:46:08 -0700 Subject: [PATCH 62/78] Fooocus GitHub Bot Commit This commit is generated by a GitHub bot of Fooocus --- backend/headless/fcbh/cli_args.py | 2 ++ backend/headless/latent_preview.py | 2 +- fooocus_version.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/headless/fcbh/cli_args.py b/backend/headless/fcbh/cli_args.py index 0b07237..85134e9 100644 --- a/backend/headless/fcbh/cli_args.py +++ b/backend/headless/fcbh/cli_args.py @@ -36,6 +36,8 @@ parser = argparse.ArgumentParser() parser.add_argument("--listen", type=str, default="127.0.0.1", metavar="IP", nargs="?", const="0.0.0.0", help="Specify the IP address to listen on (default: 127.0.0.1). If --listen is provided without an argument, it defaults to 0.0.0.0. (listens on all)") parser.add_argument("--port", type=int, default=8188, help="Set the listen port.") parser.add_argument("--enable-cors-header", type=str, default=None, metavar="ORIGIN", nargs="?", const="*", help="Enable CORS (Cross-Origin Resource Sharing) with optional origin or allow all with default '*'.") +parser.add_argument("--max-upload-size", type=float, default=100, help="Set the maximum upload size in MB.") + parser.add_argument("--extra-model-paths-config", type=str, default=None, metavar="PATH", nargs='+', action='append', help="Load one or more extra_model_paths.yaml files.") parser.add_argument("--output-directory", type=str, default=None, help="Set the fcbh_backend output directory.") parser.add_argument("--temp-directory", type=str, default=None, help="Set the fcbh_backend temp directory (default is in the fcbh_backend directory).") diff --git a/backend/headless/latent_preview.py b/backend/headless/latent_preview.py index 5b07078..798c3aa 100644 --- a/backend/headless/latent_preview.py +++ b/backend/headless/latent_preview.py @@ -22,7 +22,7 @@ class TAESDPreviewerImpl(LatentPreviewer): self.taesd = taesd def decode_latent_to_preview(self, x0): - x_sample = self.taesd.decoder(x0)[0].detach() + x_sample = self.taesd.decoder(x0[:1])[0].detach() # x_sample = self.taesd.unscale_latents(x_sample).div(4).add(0.5) # returns value in [-2, 2] x_sample = x_sample.sub(0.5).mul(2) diff --git a/fooocus_version.py b/fooocus_version.py index 0450de0..92e6790 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.760' +version = '2.1.761' From 1d16b942b38d0d23b49375be77a4925f0e7dd215 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 30 Oct 2023 12:26:35 -0700 Subject: [PATCH 63/78] gpt minor fix --- fooocus_version.py | 2 +- modules/expansion.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 92e6790..3539727 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.761' +version = '2.1.762' diff --git a/modules/expansion.py b/modules/expansion.py index a5ea1aa..5f3582f 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -21,7 +21,8 @@ black_list = ['art', 'digital', 'Ġpaint', 'painting', 'drawing', 'draw', 'drawn 'concept', 'illustration', 'illustrated', 'illustrate', 'face', 'eye', 'eyes', 'hand', 'hands', 'monster', 'artistic', 'oil', 'brush', - 'artwork', 'artworks'] + 'artwork', 'artworks', + 'skeletal', 'by', 'By', 'skeleton'] black_list += ['Ġ' + k for k in black_list] From efcd4837ce6ac7aeef60674eb9b05a5e54dbe25a Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 30 Oct 2023 12:34:14 -0700 Subject: [PATCH 64/78] Update expansion.py --- modules/expansion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/expansion.py b/modules/expansion.py index 5f3582f..487bc26 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -19,7 +19,8 @@ dangrous_patterns = '[]【】()（）|:：' black_list = ['art', 'digital', 'Ġpaint', 'painting', 'drawing', 'draw', 'drawn', 'concept', 'illustration', 'illustrated', 'illustrate', - 'face', 'eye', 'eyes', 'hand', 'hands', + 'face', 'eye', 'eyes', 'hand', 'hands', 'head', 'heads', 'leg', 'legs', 'arm', 'arms', + 'shoulder', 'shoulders', 'monster', 'artistic', 'oil', 'brush', 'artwork', 'artworks', 'skeletal', 'by', 'By', 'skeleton'] From c697826f9fd5aa9729ad3ac010d2d76c587f6f7d Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 30 Oct 2023 12:38:13 -0700 Subject: [PATCH 65/78] Update expansion.py --- modules/expansion.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/expansion.py b/modules/expansion.py index 487bc26..c277438 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -17,7 +17,7 @@ fooocus_magic_split = [ ] dangrous_patterns = '[]【】()（）|:：' -black_list = ['art', 'digital', 'Ġpaint', 'painting', 'drawing', 'draw', 'drawn', +black_list = ['art', 'digital', 'paint', 'painting', 'drawing', 'draw', 'drawn', 'concept', 'illustration', 'illustrated', 'illustrate', 'face', 'eye', 'eyes', 'hand', 'hands', 'head', 'heads', 'leg', 'legs', 'arm', 'arms', 'shoulder', 'shoulders', @@ -26,6 +26,10 @@ black_list = ['art', 'digital', 'Ġpaint', 'painting', 'drawing', 'draw', 'drawn 'skeletal', 'by', 'By', 'skeleton'] black_list += ['Ġ' + k for k in black_list] +black_list += [k.upper() for k in black_list] +black_list += [k.capitalize() for k in black_list] +black_list += ['Ġ' + k.upper() for k in black_list] +black_list += ['Ġ' + k.capitalize() for k in black_list] def safe_str(x): From d8616fe8dc3ab0063c22712fec650a423328596a Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 30 Oct 2023 13:50:46 -0700 Subject: [PATCH 66/78] edit gpt list edit gpt list --- fooocus_version.py | 2 +- modules/expansion.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 3539727..36e5c6e 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.762' +version = '2.1.764' diff --git a/modules/expansion.py b/modules/expansion.py index c277438..4c2e9a7 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -17,13 +17,13 @@ fooocus_magic_split = [ ] dangrous_patterns = '[]【】()（）|:：' -black_list = ['art', 'digital', 'paint', 'painting', 'drawing', 'draw', 'drawn', +black_list = ['art', 'digital', 'paint', 'painting', 'painted', 'drawing', 'draw', 'drawn', 'concept', 'illustration', 'illustrated', 'illustrate', 'face', 'eye', 'eyes', 'hand', 'hands', 'head', 'heads', 'leg', 'legs', 'arm', 'arms', - 'shoulder', 'shoulders', + 'shoulder', 'shoulders', 'body', 'facial', 'skin', 'character', 'human', 'portrait', 'cloth' 'monster', 'artistic', 'oil', 'brush', 'artwork', 'artworks', - 'skeletal', 'by', 'By', 'skeleton'] + 'skeletal', 'skeleton', 'a', 'the', 'background'] black_list += ['Ġ' + k for k in black_list] black_list += [k.upper() for k in black_list] @@ -51,8 +51,7 @@ class FooocusExpansion: self.vocab = self.tokenizer.vocab self.logits_bias = torch.zeros((1, len(self.vocab)), dtype=torch.float32) self.logits_bias[0, self.tokenizer.eos_token_id] = - 16.0 - # test_198 = self.tokenizer('\n', return_tensors="pt") - self.logits_bias[0, 198] = - 1024.0 + self.logits_bias[0, 198] = - 1024.0 # test_198 = self.tokenizer('\n', return_tensors="pt") for k, v in self.vocab.items(): if k in black_list: self.logits_bias[0, v] = - 1024.0 From 34bcfa79c0f96adbc6ffec98bf887466f5cd0d29 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 30 Oct 2023 16:40:50 -0700 Subject: [PATCH 67/78] improve gpt2 improve gpt2 --- expansion_experiments.py | 8 ++++++++ fooocus_version.py | 2 +- modules/async_worker.py | 6 +++--- modules/expansion.py | 43 ++++++++++++++++++++++------------------ 4 files changed, 36 insertions(+), 23 deletions(-) create mode 100644 expansion_experiments.py diff --git a/expansion_experiments.py b/expansion_experiments.py new file mode 100644 index 0000000..5c99106 --- /dev/null +++ b/expansion_experiments.py @@ -0,0 +1,8 @@ +from modules.expansion import FooocusExpansion + +expansion = FooocusExpansion() + +text = 'stone' + +for i in range(64): + print(expansion(text, seed=i)) diff --git a/fooocus_version.py b/fooocus_version.py index 36e5c6e..e09972d 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.764' +version = '2.1.766' diff --git a/modules/async_worker.py b/modules/async_worker.py index 1db93de..8b96b80 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -284,7 +284,7 @@ def worker(): progressbar(3, 'Processing prompts ...') tasks = [] for i in range(image_number): - task_seed = (seed + i) % (constants.MAX_SEED + 1) # randint is inclusive, % is not + task_seed = (seed + i) % (constants.MAX_SEED + 1) # randint is inclusive, % is not task_rng = random.Random(task_seed) # may bind to inpaint noise in the future task_prompt = apply_wildcards(prompt, task_rng) @@ -330,9 +330,9 @@ def worker(): for i, t in enumerate(tasks): progressbar(5, f'Preparing Fooocus text #{i + 1} ...') expansion = pipeline.final_expansion(t['task_prompt'], t['task_seed']) - print(f'[Prompt Expansion] New suffix: {expansion}') + print(f'[Prompt Expansion] {expansion}') t['expansion'] = expansion - t['positive'] = copy.deepcopy(t['positive']) + [join_prompts(t['task_prompt'], expansion)] # Deep copy. + t['positive'] = copy.deepcopy(t['positive']) + [expansion] # Deep copy. for i, t in enumerate(tasks): progressbar(7, f'Encoding positive #{i + 1} ...') diff --git a/modules/expansion.py b/modules/expansion.py index 4c2e9a7..74dabd6 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -9,27 +9,33 @@ from fcbh.model_patcher import ModelPatcher # limitation of np.random.seed(), called from transformers.set_seed() SEED_LIMIT_NUMPY = 2**32 +neg_inf = - 8192.0 -fooocus_magic_split = [ - ', extremely', - ', intricate,', +preparation_templates = [ + '{prompt}, extremely detailed, ', + # '{prompt}, intricate, ', ] + dangrous_patterns = '[]【】()（）|:：' black_list = ['art', 'digital', 'paint', 'painting', 'painted', 'drawing', 'draw', 'drawn', 'concept', 'illustration', 'illustrated', 'illustrate', - 'face', 'eye', 'eyes', 'hand', 'hands', 'head', 'heads', 'leg', 'legs', 'arm', 'arms', - 'shoulder', 'shoulders', 'body', 'facial', 'skin', 'character', 'human', 'portrait', 'cloth' - 'monster', 'artistic', 'oil', 'brush', - 'artwork', 'artworks', - 'skeletal', 'skeleton', 'a', 'the', 'background'] + 'face', 'faces', 'eye', 'eyes', 'hand', 'hands', 'head', 'heads', 'leg', 'legs', 'arm', 'arms', + 'shoulder', 'shoulders', 'body', 'facial', 'skin', 'character', 'human', + 'portrait', 'portraits', 'port', 'cloth', + 'monster', 'artistic', 'oil', 'brush', 'ugly', 'ug', + 'artwork', 'artworks', 'pencil', 'line', 'sketch', 'cartoon', 'white', 'black', 'red', + 'skeletal', 'skeleton', 'a', 'the', 'background', 'blur', 'blurred', 'depth', 'no', 'of', + 'catdog', 'cat', 'fur', + 'mugshot', 'selfie', + '!', '!!', '!!!', '!!!!', '!!!!!', '!!!!!!', '!!!!!!!', '-', '(', ')', ':', '”', '"', '.'] -black_list += ['Ġ' + k for k in black_list] -black_list += [k.upper() for k in black_list] -black_list += [k.capitalize() for k in black_list] -black_list += ['Ġ' + k.upper() for k in black_list] -black_list += ['Ġ' + k.capitalize() for k in black_list] +black_list = black_list + [k.upper() for k in black_list] + [k.capitalize() for k in black_list] +black_list.remove('Art') +black_list.remove('ART') + +black_list = black_list + ['Ġ' + k for k in black_list] def safe_str(x): @@ -50,11 +56,9 @@ class FooocusExpansion: self.tokenizer = AutoTokenizer.from_pretrained(fooocus_expansion_path) self.vocab = self.tokenizer.vocab self.logits_bias = torch.zeros((1, len(self.vocab)), dtype=torch.float32) - self.logits_bias[0, self.tokenizer.eos_token_id] = - 16.0 - self.logits_bias[0, 198] = - 1024.0 # test_198 = self.tokenizer('\n', return_tensors="pt") for k, v in self.vocab.items(): if k in black_list: - self.logits_bias[0, v] = - 1024.0 + self.logits_bias[0, v] = neg_inf self.model = AutoModelForCausalLM.from_pretrained(fooocus_expansion_path) self.model.eval() @@ -89,8 +93,8 @@ class FooocusExpansion: seed = int(seed) % SEED_LIMIT_NUMPY set_seed(seed) - origin = safe_str(prompt) - prompt = origin + fooocus_magic_split[seed % len(fooocus_magic_split)] + prompt = safe_str(prompt) + prompt = preparation_templates[seed % len(preparation_templates)].replace('{prompt}', prompt) tokenized_kwargs = self.tokenizer(prompt, return_tensors="pt") tokenized_kwargs.data['input_ids'] = tokenized_kwargs.data['input_ids'].to(self.patcher.load_device) @@ -111,7 +115,8 @@ class FooocusExpansion: logits_processor=logits_processor) response = self.tokenizer.batch_decode(features, skip_special_tokens=True) - result = response[0][len(origin):] + + result = response[0] result = safe_str(result) result = remove_pattern(result, dangrous_patterns) return result From 07b0a37de045372cf23b857801d69988a9dfe917 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 30 Oct 2023 16:54:08 -0700 Subject: [PATCH 68/78] Fooocus GitHub Bot Commit This commit is generated by a GitHub bot of Fooocus --- .../headless/fcbh/ldm/modules/attention.py | 37 ++++++------------- fooocus_version.py | 2 +- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/backend/headless/fcbh/ldm/modules/attention.py b/backend/headless/fcbh/ldm/modules/attention.py index f3e1b6e..c038355 100644 --- a/backend/headless/fcbh/ldm/modules/attention.py +++ b/backend/headless/fcbh/ldm/modules/attention.py @@ -160,32 +160,19 @@ def attention_sub_quad(query, key, value, heads, mask=None): mem_free_total, mem_free_torch = model_management.get_free_memory(query.device, True) - chunk_threshold_bytes = mem_free_torch * 0.5 #Using only this seems to work better on AMD - kv_chunk_size_min = None + kv_chunk_size = None + query_chunk_size = None - #not sure at all about the math here - #TODO: tweak this - if mem_free_total > 8192 * 1024 * 1024 * 1.3: - query_chunk_size_x = 1024 * 4 - elif mem_free_total > 4096 * 1024 * 1024 * 1.3: - query_chunk_size_x = 1024 * 2 - else: - query_chunk_size_x = 1024 - kv_chunk_size_min_x = None - kv_chunk_size_x = (int((chunk_threshold_bytes // (batch_x_heads * bytes_per_token * query_chunk_size_x)) * 2.0) // 1024) * 1024 - if kv_chunk_size_x < 1024: - kv_chunk_size_x = None + for x in [4096, 2048, 1024, 512, 256]: + count = mem_free_total / (batch_x_heads * bytes_per_token * x * 4.0) + if count >= k_tokens: + kv_chunk_size = k_tokens + query_chunk_size = x + break - if chunk_threshold_bytes is not None and qk_matmul_size_bytes <= chunk_threshold_bytes: - # the big matmul fits into our memory limit; do everything in 1 chunk, - # i.e. send it down the unchunked fast-path - query_chunk_size = q_tokens - kv_chunk_size = k_tokens - else: - query_chunk_size = query_chunk_size_x - kv_chunk_size = kv_chunk_size_x - kv_chunk_size_min = kv_chunk_size_min_x + if query_chunk_size is None: + query_chunk_size = 512 hidden_states = efficient_dot_product_attention( query, @@ -229,7 +216,7 @@ def attention_split(q, k, v, heads, mask=None): gb = 1024 ** 3 tensor_size = q.shape[0] * q.shape[1] * k.shape[1] * element_size - modifier = 3 if element_size == 2 else 2.5 + modifier = 3 mem_required = tensor_size * modifier steps = 1 @@ -257,10 +244,10 @@ def attention_split(q, k, v, heads, mask=None): s1 = einsum('b i d, b j d -> b i j', q[:, i:end].float(), k.float()) * scale else: s1 = einsum('b i d, b j d -> b i j', q[:, i:end], k) * scale - first_op_done = True s2 = s1.softmax(dim=-1).to(v.dtype) del s1 + first_op_done = True r1[:, i:end] = einsum('b i j, b j d -> b i d', s2, v) del s2 diff --git a/fooocus_version.py b/fooocus_version.py index e09972d..9360f81 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.766' +version = '2.1.767' From 1b96d3ba0bd1970b4e2682e60e9a0b03f61b54d1 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Mon, 30 Oct 2023 21:17:38 -0700 Subject: [PATCH 69/78] Solve all GPT problems forever --- expansion_experiments.py | 2 +- fooocus_version.py | 2 +- .../fooocus_expansion/positive.txt | 1 + modules/expansion.py | 65 ++++++------------- 4 files changed, 23 insertions(+), 47 deletions(-) create mode 100644 models/prompt_expansion/fooocus_expansion/positive.txt diff --git a/expansion_experiments.py b/expansion_experiments.py index 5c99106..5a2a946 100644 --- a/expansion_experiments.py +++ b/expansion_experiments.py @@ -2,7 +2,7 @@ from modules.expansion import FooocusExpansion expansion = FooocusExpansion() -text = 'stone' +text = 'a handsome man' for i in range(64): print(expansion(text, seed=i)) diff --git a/fooocus_version.py b/fooocus_version.py index 9360f81..b0bb82b 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.767' +version = '2.1.769' diff --git a/models/prompt_expansion/fooocus_expansion/positive.txt b/models/prompt_expansion/fooocus_expansion/positive.txt new file mode 100644 index 0000000..938a0fd --- /dev/null +++ b/models/prompt_expansion/fooocus_expansion/positive.txt @@ -0,0 +1 @@ +accelerated, acclaimed, accomplished, acknowledged, activated, adapted, adjusted, admirable, adored, adorned, advanced, adventurous, advocated, aesthetic, affectionate, affirming, aged, ageless, aimed, airy, alleviated, alluring, altered, ambient, analytical, angelic, animated, appealing, applauded, appreciative, archetypal, ardent, aristocratic, aromatic, arranged, arrayed, arresting, artistic, aspired, assertive, associated, assuaged, assured, astonishing, astounding, astral, atmosphere, attempted, attention-grabbing, attired, attractive, audacious, august, authentic, authoritative, avant-garde, awarded, awe-inspiring, awe-striking, awesome, backed, baked, balanced, baroque, beaming, beaten, beauteous, beautiful, bedazzling, bedecked, beguiling, bejeweled, beloved, best, bestowed, bewitching, blazing, blended, blessed, blissful, blooming, bodacious, bonafide, bonny, borne, boundless, braised, brave, breathtaking, breezy, brewed, bright, brightened, brilliant, broiled, brooded, brought, bubbly, built, buoyant, burning, calm, calmed, canonized, captivating, carried, catchy, celebrated, celestial, certain, championed, changed, charismatic, charmed, charming, chased, chaste, cheered, cheerful, cherished, cherubic, chic, chivalrous, choicest, chosen, chromatic, churned, cinematic, clad, clapped, classic, clear, coached, colorful, colossal, combined, comely, comforted, comforting, commanding, commendable, commended, compatible, complete, complex, complimented, composition, conceived, conferred, confident, congruent, congruous, connected, consecrated, considered, consistent, consoled, consonant, conspicuous, constructed, consummate, contemplated, contemporary, content, conveyed, cooked, cool, coordinated, cosmic, coupled, courageous, courted, coveted, cozy, craved, created, critical, crystal-clear, cuddly, cultivated, cured, curious, current, curvaceous, customized, cute, cutting-edge, dainty, dapper, daring, dashing, daydreamed, dazzling, decked, decorated, decorative, decorous, deep, defended, deific, deified, deliberated, delicate, delightful, delivered, designed, desired, detail, detailed, developed, devout, diaphanous, dignified, directed, discovered, dispatched, displayed, distilled, distinct, distinctive, distinguished, diverse, divine, dramatic, draped, dreamed, dreamlike, dreamy, driven, drop-dead, dynamic, earnest, eased, ecstatic, edified, educated, effective, effervescent, effulgent, elaborate, elated, electrifying, elegant, elevated, elite, embellished, eminent, emphatic, empowering, enchanting, encouraged, encrusted, endearing, endeavored, endorsed, endowed, enduring, energetic, energized, engaging, enlightened, enlivened, enormous, enrapturing, enthralling, enticing, entrancing, envisioned, epic, esteemed, eternal, ethereal, eulogized, euphoric, evocative, evolved, exalted, examining, excellent, exceptional, exciting, exclusive, expansive, expedited, explorative, exposed, expressive, exquisite, extolled, extraordinary, extremely, exuberant, eye-catching, fair, fairy-tale-like, fancy, fantasied, fantastic, fantastical, fascinating, fashionable, fashioned, favorable, favored, fearless, feathery, fermented, fervent, fetched, fetching, fiery, fine, finest, firm, first-class, first-rate, fixed, flamboyant, flaming, flashing, flashy, flattered, flavored, flawless, flimsy, florid, flowery, focused, formal, formed, fortunate, forward-thinking, fostered, found, foxy, fragranced, fresh, fried, fulgent, futuristic, galactic, gallant, garbed, gargantuan, garnished, gemmed, gentle, gentlemanly, genuine, gifted, gigantic, given, glamorous, gleaming, gleamy, glistening, glittering, glittery, glitzy, glorified, glossy, glowing, godly, gorgeous, gossamer, graceful, grand, granted, grateful, gratified, grilled, guarded, guided, hailed, hallowed, handsome, hardy, harmonious, harmonized, hastened, head-turning, heartened, heartfelt, heavenly, heroic, hoisted, holistic, holy, honored, hoped, hopeful, huggable, hushed, hypnotic, ideal, idyllic, illumination, illustrative, imagined, imagistic, immaculate, immense, immortal, impeccable, imposing, impressive, incandescent, infinite, informed, innocent, innovated, innovative, inproportion, inquisitive, inshape, insistent, inspired, inspiring, instructed, integrated, intense, intoxicating, intrepid, intricate, invaluable, invented, investigative, invigorated, invigorating, inviting, iridescent, jaunty, jaw-dropping, jazzy, jeweled, joined, joyful, joyous, jubilant, kingly, kissable, kneaded, knightly, knockout, ladylike, lambent, lasting, laudable, lauded, lavish, leading-edge, led, legendary, lifelike, lifted, light, lightened, likable, limited, linked, lionized, lithe, lively, located, logical, longed, lordly, lovable, loved, lovely, lucent, lucid, lucky, luminous, lush, lustrous, luxurious, luxury, magical, magnificent, majestic, mammoth, marked, marvelous, massive, matched, matchless, matured, meditated, mellow, mentored, merged, merry, mesmerizing, methodical, mind-blowing, modern, modified, modish, molded, monarchical, monstrous, monumental, motivated, motivational, moved, multifaceted, multifarious, mused, mystical, mythical, naive, natty, navigated, neat, new, nifty, noble, notable, noteworthy, novel, nuanced, nubile, nurtured, odorous, offered, opalescent, optimal, optimistic, opulent, orderly, organized, original, originated, ornate, ostentatious, otherworldly, outfitted, outstanding, overjoyed, pacified, painterly, paired, panoramic, paradisiacal, passionate, patterned, peaceful, peerless, peppy, perfect, perfumed, perky, perpetual, persistent, photogenic, pickled, pictorial, picturesque, piloted, pious, placed, planned, pleased, pleasing, plotted, plush, plushy, poached, polished, pondered, posh, positive, praised, praiseworthy, precious, precise, preeminent, premier, premium, prepossessing, presented, preserved, pretty, priceless, prime, princely, pristine, probing, prodigious, profound, progressive, prominent, promoted, pronounced, propelled, proportional, protected, provided, pulchritudinous, pungent, pure, pursued, pushed, quality, queenly, questioning, quickened, quiet, quintessential, racy, radiant, rare, rational, ravishing, real, reborn, recharged, reclaimed, recognized, recovered, redolent, reenergized, refined, reflected, refreshed, refreshing, refulgent, regal, regenerated, reinvigorated, rejuvenated, rejuvenating, related, relaxed, relentless, relieved, remarkable, renewed, representative, rescued, resilient, resolute, respected, resplendent, restored, retrieved, revealed, revered, reverent, revitalized, revitalizing, revived, rewarded, rhythmic, rich, ripened, ritzy, roasted, robed, robust, rococo, romantic, royal, ruminated, sacred, saintly, salient, saluted, sanctified, satisfied, saturated, saved, scenic, scented, schemed, scientific, sculpted, second-to-none, select, seraphic, serene, serious, set, shaped, shapely, shielded, shimmering, shining, shiny, shipshape, show-stopping, shown, showy, sightly, significant, silent, silky, simmered, sincere, singular, situated, sleek, slick, smart, smelly, snazzy, snug, snuggly, solemn, solid, soothed, soothing, sophisticated, sought, soulful, spangled, sparkling, special, sped, spellbinding, spiced, spiffy, spirited, spiritual, splashy, splendid, sprightly, spruce, spunky, stable, stagey, starry, stately, statuesque, steamed, steered, stewed, stimulated, stimulating, stirred, striking, strong, structured, studded, stunning, stupendous, sturdy, stylish, sublime, sumptuous, sunny, superb, superior, superlative, supplied, supported, supreme, sure, surreal, swank, swanky, sweet, sweetened, swish, symbolic, symmetrical, synchronized, systematic, tailored, taking, tangy, tantalizing, targeted, tasteful, taught, tenacious, tender, thankful, theatrical, thought, thrilled, thrilling, tidy, timeless, titanic, toasted, top-drawer, top-notch, top-tier, topnotch, tough, trained, tranquil, tranquilized, transcendent, transformed, translucent, transparent, transported, treasured, trendy, tried, trim, true, twinkling, twinned, unblemished, uncluttered, uncovered, unequaled, unfeigned, unique, united, universal, unmatched, unparalleled, unrivaled, unsullied, unsurpassed, unyielding, up-to-date, upheld, uplifted, uplifting, valiant, valued, varied, velvety, venerated, venturesome, vibrant, vinted, virginal, visualized, vitalized, vivacious, vivid, vivified, voguish, warm, well-built, well-formed, well-kept, well-proportioned, well-turned, whimsical, whisked, whole, winning, winsome, wished, wisp-like, wonderful, wondrous, wooed, worshiped, worshipped, yearned, zesty, zippy, sharp, focus, background diff --git a/modules/expansion.py b/modules/expansion.py index 74dabd6..daa6d5a 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -1,3 +1,4 @@ +import os import torch import math import fcbh.model_management as model_management @@ -9,33 +10,6 @@ from fcbh.model_patcher import ModelPatcher # limitation of np.random.seed(), called from transformers.set_seed() SEED_LIMIT_NUMPY = 2**32 -neg_inf = - 8192.0 - - -preparation_templates = [ - '{prompt}, extremely detailed, ', - # '{prompt}, intricate, ', -] - -dangrous_patterns = '[]【】()（）|:：' - -black_list = ['art', 'digital', 'paint', 'painting', 'painted', 'drawing', 'draw', 'drawn', - 'concept', 'illustration', 'illustrated', 'illustrate', - 'face', 'faces', 'eye', 'eyes', 'hand', 'hands', 'head', 'heads', 'leg', 'legs', 'arm', 'arms', - 'shoulder', 'shoulders', 'body', 'facial', 'skin', 'character', 'human', - 'portrait', 'portraits', 'port', 'cloth', - 'monster', 'artistic', 'oil', 'brush', 'ugly', 'ug', - 'artwork', 'artworks', 'pencil', 'line', 'sketch', 'cartoon', 'white', 'black', 'red', - 'skeletal', 'skeleton', 'a', 'the', 'background', 'blur', 'blurred', 'depth', 'no', 'of', - 'catdog', 'cat', 'fur', - 'mugshot', 'selfie', - '!', '!!', '!!!', '!!!!', '!!!!!', '!!!!!!', '!!!!!!!', '-', '(', ')', ':', '”', '"', '.'] - -black_list = black_list + [k.upper() for k in black_list] + [k.capitalize() for k in black_list] -black_list.remove('Art') -black_list.remove('ART') - -black_list = black_list + ['Ġ' + k for k in black_list] def safe_str(x): @@ -54,11 +28,21 @@ def remove_pattern(x, pattern): class FooocusExpansion: def __init__(self): self.tokenizer = AutoTokenizer.from_pretrained(fooocus_expansion_path) - self.vocab = self.tokenizer.vocab - self.logits_bias = torch.zeros((1, len(self.vocab)), dtype=torch.float32) - for k, v in self.vocab.items(): - if k in black_list: - self.logits_bias[0, v] = neg_inf + + positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'), encoding='utf-8').read() + positive_words = positive_words.lower().replace(' ', '').replace('\n', '').split(',') + + # print(', '.join(sorted(list(set(positive_words))))) + + # t198 = self.tokenizer('\n', return_tensors="np") + # t11 = self.tokenizer(',', return_tensors="np") + # positive_ids = [11, 198, self.tokenizer.eos_token_id] + positive_ids = [11] + + self.bad_words_ids = [] + for k, v in self.tokenizer.vocab.items(): + if k.replace('Ġ', '') not in positive_words and v not in positive_ids: + self.bad_words_ids.append([v]) self.model = AutoModelForCausalLM.from_pretrained(fooocus_expansion_path) self.model.eval() @@ -79,10 +63,6 @@ class FooocusExpansion: self.patcher = ModelPatcher(self.model, load_device=load_device, offload_device=offload_device) print(f'Fooocus Expansion engine loaded for {load_device}, use_fp16 = {use_fp16}.') - def logits_processor(self, input_ids, scores): - self.logits_bias = self.logits_bias.to(scores) - return scores + self.logits_bias - def __call__(self, prompt, seed): if prompt == '': return '' @@ -93,8 +73,7 @@ class FooocusExpansion: seed = int(seed) % SEED_LIMIT_NUMPY set_seed(seed) - prompt = safe_str(prompt) - prompt = preparation_templates[seed % len(preparation_templates)].replace('{prompt}', prompt) + prompt = safe_str(prompt) + ',' tokenized_kwargs = self.tokenizer(prompt, return_tensors="pt") tokenized_kwargs.data['input_ids'] = tokenized_kwargs.data['input_ids'].to(self.patcher.load_device) @@ -104,19 +83,15 @@ class FooocusExpansion: max_token_length = 75 * int(math.ceil(float(current_token_length) / 75.0)) max_new_tokens = max_token_length - current_token_length - logits_processor = LogitsProcessorList([self.logits_processor]) - # https://huggingface.co/blog/introducing-csearch # https://huggingface.co/docs/transformers/generation_strategies features = self.model.generate(**tokenized_kwargs, - num_beams=1, + top_k=100, max_new_tokens=max_new_tokens, do_sample=True, - logits_processor=logits_processor) + bad_words_ids=self.bad_words_ids) response = self.tokenizer.batch_decode(features, skip_special_tokens=True) + result = safe_str(response[0]) - result = response[0] - result = safe_str(result) - result = remove_pattern(result, dangrous_patterns) return result From f30e71ad645d033ab88a8cb8633bf2d1a4c4f1bc Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Tue, 31 Oct 2023 12:32:02 -0700 Subject: [PATCH 70/78] maintain gpt maintain gpt --- fooocus_version.py | 2 +- .../fooocus_expansion/positive.txt | 992 +++++++++++++++++- modules/expansion.py | 8 +- 3 files changed, 996 insertions(+), 6 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index b0bb82b..f7c3cbd 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.769' +version = '2.1.770' diff --git a/models/prompt_expansion/fooocus_expansion/positive.txt b/models/prompt_expansion/fooocus_expansion/positive.txt index 938a0fd..4e35c77 100644 --- a/models/prompt_expansion/fooocus_expansion/positive.txt +++ b/models/prompt_expansion/fooocus_expansion/positive.txt @@ -1 +1,991 @@ -accelerated, acclaimed, accomplished, acknowledged, activated, adapted, adjusted, admirable, adored, adorned, advanced, adventurous, advocated, aesthetic, affectionate, affirming, aged, ageless, aimed, airy, alleviated, alluring, altered, ambient, analytical, angelic, animated, appealing, applauded, appreciative, archetypal, ardent, aristocratic, aromatic, arranged, arrayed, arresting, artistic, aspired, assertive, associated, assuaged, assured, astonishing, astounding, astral, atmosphere, attempted, attention-grabbing, attired, attractive, audacious, august, authentic, authoritative, avant-garde, awarded, awe-inspiring, awe-striking, awesome, backed, baked, balanced, baroque, beaming, beaten, beauteous, beautiful, bedazzling, bedecked, beguiling, bejeweled, beloved, best, bestowed, bewitching, blazing, blended, blessed, blissful, blooming, bodacious, bonafide, bonny, borne, boundless, braised, brave, breathtaking, breezy, brewed, bright, brightened, brilliant, broiled, brooded, brought, bubbly, built, buoyant, burning, calm, calmed, canonized, captivating, carried, catchy, celebrated, celestial, certain, championed, changed, charismatic, charmed, charming, chased, chaste, cheered, cheerful, cherished, cherubic, chic, chivalrous, choicest, chosen, chromatic, churned, cinematic, clad, clapped, classic, clear, coached, colorful, colossal, combined, comely, comforted, comforting, commanding, commendable, commended, compatible, complete, complex, complimented, composition, conceived, conferred, confident, congruent, congruous, connected, consecrated, considered, consistent, consoled, consonant, conspicuous, constructed, consummate, contemplated, contemporary, content, conveyed, cooked, cool, coordinated, cosmic, coupled, courageous, courted, coveted, cozy, craved, created, critical, crystal-clear, cuddly, cultivated, cured, curious, current, curvaceous, customized, cute, cutting-edge, dainty, dapper, daring, dashing, daydreamed, dazzling, decked, decorated, decorative, decorous, deep, defended, deific, deified, deliberated, delicate, delightful, delivered, designed, desired, detail, detailed, developed, devout, diaphanous, dignified, directed, discovered, dispatched, displayed, distilled, distinct, distinctive, distinguished, diverse, divine, dramatic, draped, dreamed, dreamlike, dreamy, driven, drop-dead, dynamic, earnest, eased, ecstatic, edified, educated, effective, effervescent, effulgent, elaborate, elated, electrifying, elegant, elevated, elite, embellished, eminent, emphatic, empowering, enchanting, encouraged, encrusted, endearing, endeavored, endorsed, endowed, enduring, energetic, energized, engaging, enlightened, enlivened, enormous, enrapturing, enthralling, enticing, entrancing, envisioned, epic, esteemed, eternal, ethereal, eulogized, euphoric, evocative, evolved, exalted, examining, excellent, exceptional, exciting, exclusive, expansive, expedited, explorative, exposed, expressive, exquisite, extolled, extraordinary, extremely, exuberant, eye-catching, fair, fairy-tale-like, fancy, fantasied, fantastic, fantastical, fascinating, fashionable, fashioned, favorable, favored, fearless, feathery, fermented, fervent, fetched, fetching, fiery, fine, finest, firm, first-class, first-rate, fixed, flamboyant, flaming, flashing, flashy, flattered, flavored, flawless, flimsy, florid, flowery, focused, formal, formed, fortunate, forward-thinking, fostered, found, foxy, fragranced, fresh, fried, fulgent, futuristic, galactic, gallant, garbed, gargantuan, garnished, gemmed, gentle, gentlemanly, genuine, gifted, gigantic, given, glamorous, gleaming, gleamy, glistening, glittering, glittery, glitzy, glorified, glossy, glowing, godly, gorgeous, gossamer, graceful, grand, granted, grateful, gratified, grilled, guarded, guided, hailed, hallowed, handsome, hardy, harmonious, harmonized, hastened, head-turning, heartened, heartfelt, heavenly, heroic, hoisted, holistic, holy, honored, hoped, hopeful, huggable, hushed, hypnotic, ideal, idyllic, illumination, illustrative, imagined, imagistic, immaculate, immense, immortal, impeccable, imposing, impressive, incandescent, infinite, informed, innocent, innovated, innovative, inproportion, inquisitive, inshape, insistent, inspired, inspiring, instructed, integrated, intense, intoxicating, intrepid, intricate, invaluable, invented, investigative, invigorated, invigorating, inviting, iridescent, jaunty, jaw-dropping, jazzy, jeweled, joined, joyful, joyous, jubilant, kingly, kissable, kneaded, knightly, knockout, ladylike, lambent, lasting, laudable, lauded, lavish, leading-edge, led, legendary, lifelike, lifted, light, lightened, likable, limited, linked, lionized, lithe, lively, located, logical, longed, lordly, lovable, loved, lovely, lucent, lucid, lucky, luminous, lush, lustrous, luxurious, luxury, magical, magnificent, majestic, mammoth, marked, marvelous, massive, matched, matchless, matured, meditated, mellow, mentored, merged, merry, mesmerizing, methodical, mind-blowing, modern, modified, modish, molded, monarchical, monstrous, monumental, motivated, motivational, moved, multifaceted, multifarious, mused, mystical, mythical, naive, natty, navigated, neat, new, nifty, noble, notable, noteworthy, novel, nuanced, nubile, nurtured, odorous, offered, opalescent, optimal, optimistic, opulent, orderly, organized, original, originated, ornate, ostentatious, otherworldly, outfitted, outstanding, overjoyed, pacified, painterly, paired, panoramic, paradisiacal, passionate, patterned, peaceful, peerless, peppy, perfect, perfumed, perky, perpetual, persistent, photogenic, pickled, pictorial, picturesque, piloted, pious, placed, planned, pleased, pleasing, plotted, plush, plushy, poached, polished, pondered, posh, positive, praised, praiseworthy, precious, precise, preeminent, premier, premium, prepossessing, presented, preserved, pretty, priceless, prime, princely, pristine, probing, prodigious, profound, progressive, prominent, promoted, pronounced, propelled, proportional, protected, provided, pulchritudinous, pungent, pure, pursued, pushed, quality, queenly, questioning, quickened, quiet, quintessential, racy, radiant, rare, rational, ravishing, real, reborn, recharged, reclaimed, recognized, recovered, redolent, reenergized, refined, reflected, refreshed, refreshing, refulgent, regal, regenerated, reinvigorated, rejuvenated, rejuvenating, related, relaxed, relentless, relieved, remarkable, renewed, representative, rescued, resilient, resolute, respected, resplendent, restored, retrieved, revealed, revered, reverent, revitalized, revitalizing, revived, rewarded, rhythmic, rich, ripened, ritzy, roasted, robed, robust, rococo, romantic, royal, ruminated, sacred, saintly, salient, saluted, sanctified, satisfied, saturated, saved, scenic, scented, schemed, scientific, sculpted, second-to-none, select, seraphic, serene, serious, set, shaped, shapely, shielded, shimmering, shining, shiny, shipshape, show-stopping, shown, showy, sightly, significant, silent, silky, simmered, sincere, singular, situated, sleek, slick, smart, smelly, snazzy, snug, snuggly, solemn, solid, soothed, soothing, sophisticated, sought, soulful, spangled, sparkling, special, sped, spellbinding, spiced, spiffy, spirited, spiritual, splashy, splendid, sprightly, spruce, spunky, stable, stagey, starry, stately, statuesque, steamed, steered, stewed, stimulated, stimulating, stirred, striking, strong, structured, studded, stunning, stupendous, sturdy, stylish, sublime, sumptuous, sunny, superb, superior, superlative, supplied, supported, supreme, sure, surreal, swank, swanky, sweet, sweetened, swish, symbolic, symmetrical, synchronized, systematic, tailored, taking, tangy, tantalizing, targeted, tasteful, taught, tenacious, tender, thankful, theatrical, thought, thrilled, thrilling, tidy, timeless, titanic, toasted, top-drawer, top-notch, top-tier, topnotch, tough, trained, tranquil, tranquilized, transcendent, transformed, translucent, transparent, transported, treasured, trendy, tried, trim, true, twinkling, twinned, unblemished, uncluttered, uncovered, unequaled, unfeigned, unique, united, universal, unmatched, unparalleled, unrivaled, unsullied, unsurpassed, unyielding, up-to-date, upheld, uplifted, uplifting, valiant, valued, varied, velvety, venerated, venturesome, vibrant, vinted, virginal, visualized, vitalized, vivacious, vivid, vivified, voguish, warm, well-built, well-formed, well-kept, well-proportioned, well-turned, whimsical, whisked, whole, winning, winsome, wished, wisp-like, wonderful, wondrous, wooed, worshiped, worshipped, yearned, zesty, zippy, sharp, focus, background +abloom +accelerated +accepted +accepting +acclaimed +accomplished +acknowledged +activated +adapted +adjusted +admirable +adored +adorned +advanced +adventurous +advocated +aesthetic +affectionate +affirmed +affirming +aged +ageless +agile +aimed +airy +aligned +alive +alleviated +alluring +altered +altruistic +amazing +ambient +amiable +analytical +angelic +animated +appealing +applauded +appreciated +appreciative +archetypal +ardent +aristocratic +aromatic +arranged +arrayed +arresting +artistic +aspirational +aspired +assertive +associated +assuaged +assured +assuredly +astonishing +astounding +astral +atmosphere +attempted +attention-grabbing +attentive +attired +attractive +audacious +august +authentic +authoritative +avant-garde +awarded +awe-inspiring +awe-striking +awesome +backed +background +baked +balance +balanced +baroque +beaming +beaten +beatific +beauteous +beautiful +bedazzling +bedecked +beguiling +bejeweled +beloved +benevolent +best +bestowed +bewitching +blazing +blended +blessed +blissed +blissful +blooming +bodacious +bonafide +bonny +borne +boundless +braised +brave +breathtaking +breezy +brewed +bright +brightened +brilliant +broiled +brooded +brought +bubbly +built +buoyant +buoyed +burning +calm +calmed +canonized +captivating +caring +carried +catchy +celebrated +celestial +certain +championed +changed +channeled +charismatic +charmed +charming +chased +chaste +cheered +cheerful +cherished +cherubic +chic +chivalrous +choicest +chosen +chromatic +churned +cinematic +clad +clapped +classic +clear +clear-minded +coached +color +colors +colorful +colossal +combined +comely +comforted +comforting +commanding +commendable +commended +committed +compassionate +compatible +complete +complex +complimented +composition +comprehensive +conceived +conferred +confident +congruent +congruous +connected +consecrated +considerable +considered +consistent +consoled +consonant +conspicuous +constructed +constructive +consummate +contemplated +contemplative +contemporary +content +contrasted +conveyed +cooked +cool +coordinated +coruscating +cosmic +coupled +courageous +courted +coveted +cozy +craved +created +creative +crisp +critical +crystal-clear +cuddly +cultivated +cured +curious +current +curvaceous +customized +cute +cutting-edge +dainty +dapper +daring +dashing +dauntless +daydreamed +dazzling +dazzlingly +decked +decorated +decorative +decorous +dedicated +deep +defended +definitive +deific +deified +deliberated +delicate +delightful +delivered +demonstrative +dependable +depicted +designed +desired +destined +detail +detailed +determined +developed +devoted +devout +diaphanous +dignified +diligent +directed +discovered +dispatched +displayed +distilled +distinct +distinctive +distinguished +diverse +diversified +divine +dramatic +draped +dreamed +dreamlike +dreamy +driven +drop-dead +dynamic +earnest +eased +ecstatic +edified +edifying +educated +effective +effervescent +effulgent +effusive +elaborate +elated +electrifying +elegant +elevated +elite +eloquent +embellished +emboldened +eminent +emotional +empathetic +empathic +emphatic +empowered +empowering +enchanting +enchantingly +encouraged +encrusted +endearing +endeavored +endorsed +endowed +enduring +energetic +energized +energizing +engaging +enigmatic +enlightened +enlivened +enormous +enraptured +enrapturing +enthralling +enticed +enticing +entrancing +envisioned +epic +equanimous +esteemed +eternal +ethereal +ethically +eulogized +euphonious +euphoric +evocative +evocatively +evolved +exalted +examining +excellent +exceptional +exciting +exclusive +exemplary +exhilarating +expansive +expedited +explorative +exposed +expressive +expressively +exquisite +extemporized +extolled +extradited +extraordinary +extremely +exuberant +eye-catching +fabulous +facilitated +fair +fairy-tale-like +fancy +fantasied +fantastic +fantastical +fascinating +fashionable +fashioned +fathomless +favorable +favored +fearless +feathery +felicitous +fermented +fervent +festive +fetched +fetching +fiery +fine +finest +firm +first-class +first-rate +fixed +flamboyant +flaming +flashing +flashy +flattered +flavored +flawless +flimsy +florid +flourishing +flowery +focus +focused +formal +formed +fortuitous +fortunate +forward-thinking +fostered +fostering +found +foxy +fragranced +free-spirited +fresh +fried +fulfilled +fulgent +futuristic +galactic +gallant +galvanized +garbed +gargantuan +garnished +gemmed +generous +gentle +gentlemanly +genuine +gifted +gigantic +given +glamorous +gleaming +gleamy +glistening +glittering +glittery +glitzy +glorified +glorious +glossy +glowing +godly +gorgeous +gossamer +graceful +gracious +grand +grandiose +granted +grateful +gratified +grilled +guarded +guided +hailed +hallowed +handsome +hardy +harmonious +harmonized +hastened +head-turning +heartened +heartfelt +heartwarming +heavenly +heroic +hoisted +holistic +holy +honored +hoped +hopeful +huggable +hushed +hypnotic +ideal +idealized +idyllic +illuminated +illuminating +illumination +illustrative +illustrious +imaginative +imagined +imagistic +imbued +immaculate +immense +immortal +impeccable +imposing +impressive +incandescent +incisive +indomitable +infinite +inflamed +informed +innocent +innovated +innovative +inproportion +inquisitive +insatiable +inshape +insightful +insistent +inspired +inspiring +instructed +integrated +intense +intoxicating +intrepid +intricate +invaluable +invented +investigative +invigorated +invigorating +invincible +inviting +iridescent +jaunty +jaw-dropping +jazzy +jeweled +joined +joyful +joyous +jubilant +judicious +keen +kind-hearted +kindhearted +kinetic +kingly +kissable +kneaded +knightly +knockout +laced +ladylike +lambent +lasting +laudable +laudatory +lauded +lavish +leading-edge +legendary +lifelike +lifted +light +lightened +lighthearted +likable +limited +linked +lionized +lithe +lively +located +logical +longed +lordly +lovable +loved +lovely +lucent +lucid +lucky +luminous +lush +lustrous +luxurious +luxury +magical +magnificent +majestic +mammoth +manifesting +marked +marvelous +massive +masterful +matched +matchless +matured +meaningful +meditated +meditative +mellifluous +mellow +mentored +merged +merry +mesmerizing +methodical +meticulous +mind-blowing +mindful +miraculous +mirthful +modern +modified +modish +molded +monarchical +monstrous +monumental +motivated +motivational +moved +multifaceted +multifarious +mused +mystical +mythical +naive +natty +navigated +neat +new +nifty +noble +notable +noteworthy +novel +nuanced +nubile +nurtured +nurturing +odorous +offered +opalescent +optimal +optimistic +opulent +orderly +organized +original +originated +ornate +ostentatious +otherworldly +outfitted +outstanding +overjoyed +pacified +painterly +paired +palpable +panoramic +paradisiacal +passionate +patterned +peaceful +peerless +peppy +perfect +perfumed +perky +perpetual +persevering +persistent +photogenic +pickled +pictorial +picturesque +piloted +pious +pivotal +placed +placid +planned +pleased +pleasing +plotted +plush +plushy +poached +polished +pondered +posh +positive +praised +praiseworthy +precious +precise +preeminent +premier +premium +prepossessing +presented +preserved +pretty +priceless +prime +princely +pristine +probing +prodigious +professional +profound +progressive +prominent +promoted +pronounced +propelled +proportional +protected +provided +pulchritudinous +pungent +pure +pursued +pushed +quality +queenly +questioning +quickened +quiet +quintessential +racy +radiant +rare +rational +ravishing +real +reborn +recharged +reclaimed +recognized +recovered +redolent +reenergized +refined +reflected +refreshed +refreshing +refulgent +regal +regenerated +reinvigorated +rejuvenated +rejuvenating +related +relaxed +relentless +relieved +remarkable +renewed +representative +rescued +resilient +resolute +respected +resplendent +restored +retrieved +revealed +revered +reverent +revitalized +revitalizing +revived +rewarded +rhythmic +rich +ripened +ritzy +roasted +robed +robust +rococo +romantic +royal +ruminated +sacred +saintly +salient +saluted +sanctified +satisfied +saturated +saved +scenic +scented +schemed +scientific +sculpted +second-to-none +select +seraphic +serendipitous +serene +serenity +serious +set +shaped +shapely +sharp +shielded +shimmering +shining +shiny +shipshape +show-stopping +shown +showy +sightly +significant +silent +silky +simmered +sincere +singular +situated +sleek +slick +smart +smelly +snazzy +snug +snuggly +solemn +solid +soothed +soothing +sophisticated +sought +soulful +spangled +sparkling +special +spectacular +sped +spellbinding +spiced +spiffy +spirited +spiritual +splashy +splendid +sprightly +spruce +spunky +stable +stagey +starry +stately +statuesque +steamed +steered +stewed +still +stimulated +stimulating +stirred +striking +strong +structured +studded +stunning +stupendous +sturdy +stylish +sublime +sumptuous +sun-kissed +sunny +superb +superior +superlative +supplied +supported +supreme +sure +surreal +swank +swanky +sweet +sweetened +swish +symbolic +symmetry +symmetrical +synchronized +systematic +tailored +taking +tangy +tantalizing +targeted +tasteful +taught +tenacious +tender +thankful +theatrical +thought +thrilled +thrilling +tidy +timeless +titanic +toasted +top-drawer +top-notch +top-tier +topnotch +tough +trained +tranquil +tranquilized +transcendent +transcendental +transformed +translucent +transparent +transported +treasured +trendy +tried +trim +true +twinkling +twinned +unblemished +uncluttered +unconditional +uncovered +unequaled +unfeigned +unique +united +universal +unmatched +unparalleled +unrivaled +unsullied +unsurpassed +unyielding +up-to-date +upheld +uplifted +uplifting +valiant +valued +varied +velvety +venerated +venturesome +vibrant +vinted +virginal +virtuous +visualized +vitalized +vivacious +vivid +vivified +voguish +warm +well-built +well-formed +well-kept +well-proportioned +well-turned +whimsical +whisked +whole +winning +winsome +wished +wisp-like +wonderful +wondrous +wooed +worshiped +worshipped +yearned +zesty +zippy diff --git a/modules/expansion.py b/modules/expansion.py index daa6d5a..ce40b56 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -29,14 +29,14 @@ class FooocusExpansion: def __init__(self): self.tokenizer = AutoTokenizer.from_pretrained(fooocus_expansion_path) - positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'), encoding='utf-8').read() - positive_words = positive_words.lower().replace(' ', '').replace('\n', '').split(',') - - # print(', '.join(sorted(list(set(positive_words))))) + positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'), + encoding='utf-8').read().splitlines() + # new_content = '\n'.join(sorted(list(set(positive_words)))) # t198 = self.tokenizer('\n', return_tensors="np") # t11 = self.tokenizer(',', return_tensors="np") # positive_ids = [11, 198, self.tokenizer.eos_token_id] + positive_ids = [11] self.bad_words_ids = [] From f07e64b357aaf51ff91facff554d9896ca8b171f Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Tue, 31 Oct 2023 13:20:34 -0700 Subject: [PATCH 71/78] remove unused codes --- modules/expansion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/expansion.py b/modules/expansion.py index ce40b56..a8f3d9c 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -3,11 +3,11 @@ import torch import math import fcbh.model_management as model_management -from transformers.generation.logits_process import LogitsProcessorList from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed from modules.path import fooocus_expansion_path from fcbh.model_patcher import ModelPatcher + # limitation of np.random.seed(), called from transformers.set_seed() SEED_LIMIT_NUMPY = 2**32 From 5dc1221c6557df3e7fdf09d4ecf1ffbdf4b3b42b Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Tue, 31 Oct 2023 14:41:18 -0700 Subject: [PATCH 72/78] maintain gpt --- fooocus_version.py | 2 +- .../prompt_expansion/fooocus_expansion/positive.txt | 3 --- modules/expansion.py | 12 +++++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index f7c3cbd..cb54aeb 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.770' +version = '2.1.771' diff --git a/models/prompt_expansion/fooocus_expansion/positive.txt b/models/prompt_expansion/fooocus_expansion/positive.txt index 4e35c77..a713144 100644 --- a/models/prompt_expansion/fooocus_expansion/positive.txt +++ b/models/prompt_expansion/fooocus_expansion/positive.txt @@ -18,8 +18,6 @@ aesthetic affectionate affirmed affirming -aged -ageless agile aimed airy @@ -654,7 +652,6 @@ outfitted outstanding overjoyed pacified -painterly paired palpable panoramic diff --git a/modules/expansion.py b/modules/expansion.py index a8f3d9c..2478410 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -31,18 +31,20 @@ class FooocusExpansion: positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'), encoding='utf-8').read().splitlines() + positive_words = [x.lower() for x in positive_words if x != ''] # new_content = '\n'.join(sorted(list(set(positive_words)))) - # t198 = self.tokenizer('\n', return_tensors="np") - # t11 = self.tokenizer(',', return_tensors="np") - # positive_ids = [11, 198, self.tokenizer.eos_token_id] + # eos = self.tokenizer.eos_token_id - positive_ids = [11] + symbols = '-+,.;?!!!' self.bad_words_ids = [] for k, v in self.tokenizer.vocab.items(): - if k.replace('Ġ', '') not in positive_words and v not in positive_ids: + if k.replace('Ġ', '').lower() not in positive_words and k not in symbols: self.bad_words_ids.append([v]) + else: + # print(k) + pass self.model = AutoModelForCausalLM.from_pretrained(fooocus_expansion_path) self.model.eval() From 01bfa11fd1730c63ee79473ba6a6033220117dd2 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 1 Nov 2023 12:50:12 -0700 Subject: [PATCH 73/78] minor revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now the GPT will try to (1) use more aligned formatting of commas, (2) a bit more dynamic in word choice, (3) avoid duplication like “detail, detail, detail, detail” --- fooocus_version.py | 2 +- modules/expansion.py | 34 ++++++++++++++++++++++------------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index cb54aeb..200610c 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.771' +version = '2.1.772' diff --git a/modules/expansion.py b/modules/expansion.py index 2478410..a536fad 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -3,6 +3,7 @@ import torch import math import fcbh.model_management as model_management +from transformers.generation.logits_process import LogitsProcessorList from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed from modules.path import fooocus_expansion_path from fcbh.model_patcher import ModelPatcher @@ -10,6 +11,7 @@ from fcbh.model_patcher import ModelPatcher # limitation of np.random.seed(), called from transformers.set_seed() SEED_LIMIT_NUMPY = 2**32 +neg_inf = - 8192.0 def safe_str(x): @@ -31,20 +33,17 @@ class FooocusExpansion: positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'), encoding='utf-8').read().splitlines() - positive_words = [x.lower() for x in positive_words if x != ''] + positive_words = ['Ġ' + x for x in positive_words if x != ''] - # new_content = '\n'.join(sorted(list(set(positive_words)))) - # eos = self.tokenizer.eos_token_id + self.logits_bias = torch.zeros((1, len(self.tokenizer.vocab)), dtype=torch.float32) + neg_inf - symbols = '-+,.;?!!!' - - self.bad_words_ids = [] for k, v in self.tokenizer.vocab.items(): - if k.replace('Ġ', '').lower() not in positive_words and k not in symbols: - self.bad_words_ids.append([v]) - else: - # print(k) - pass + if k in positive_words: + self.logits_bias[0, v] = 0 + + # t11 = self.tokenizer(',', return_tensors="np") + # t198 = self.tokenizer('\n', return_tensors="np") + # eos = self.tokenizer.eos_token_id self.model = AutoModelForCausalLM.from_pretrained(fooocus_expansion_path) self.model.eval() @@ -65,6 +64,17 @@ class FooocusExpansion: self.patcher = ModelPatcher(self.model, load_device=load_device, offload_device=offload_device) print(f'Fooocus Expansion engine loaded for {load_device}, use_fp16 = {use_fp16}.') + @torch.no_grad() + @torch.inference_mode() + def logits_processor(self, input_ids, scores): + assert scores.ndim == 2 and scores.shape[0] == 1 + bias = self.logits_bias.to(scores).clone() + bias[0, input_ids[0].to(bias.device).long()] = neg_inf + bias[0, 11] = 0 + return scores + bias + + @torch.no_grad() + @torch.inference_mode() def __call__(self, prompt, seed): if prompt == '': return '' @@ -91,7 +101,7 @@ class FooocusExpansion: top_k=100, max_new_tokens=max_new_tokens, do_sample=True, - bad_words_ids=self.bad_words_ids) + logits_processor=LogitsProcessorList([self.logits_processor])) response = self.tokenizer.batch_decode(features, skip_special_tokens=True) result = safe_str(response[0]) From 7d532b02852730a6d2ad53d33a39e885695d20e2 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 1 Nov 2023 15:43:49 -0700 Subject: [PATCH 74/78] add words to tokenizer thank you, GPT-4 --- fooocus_version.py | 2 +- .../fooocus_expansion/positive.txt | 528 +++--------------- modules/expansion.py | 9 +- 3 files changed, 99 insertions(+), 440 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 200610c..0bbac3c 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.772' +version = '2.1.773' diff --git a/models/prompt_expansion/fooocus_expansion/positive.txt b/models/prompt_expansion/fooocus_expansion/positive.txt index a713144..0a1f7e4 100644 --- a/models/prompt_expansion/fooocus_expansion/positive.txt +++ b/models/prompt_expansion/fooocus_expansion/positive.txt @@ -1,4 +1,4 @@ -abloom +abundant accelerated accepted accepting @@ -9,118 +9,74 @@ activated adapted adjusted admirable -adored +adorable adorned advanced adventurous advocated aesthetic -affectionate affirmed -affirming +affluent agile aimed -airy aligned alive -alleviated -alluring altered -altruistic amazing ambient -amiable +amplified analytical -angelic animated appealing applauded appreciated -appreciative -archetypal ardent -aristocratic aromatic arranged -arrayed arresting +articulate artistic -aspirational -aspired -assertive associated -assuaged assured -assuredly astonishing astounding -astral atmosphere attempted -attention-grabbing attentive -attired attractive -audacious -august authentic authoritative -avant-garde awarded -awe-inspiring -awe-striking awesome backed background baked balance balanced -baroque -beaming +balancing beaten -beatific -beauteous beautiful -bedazzling -bedecked -beguiling -bejeweled beloved +beneficial benevolent best bestowed -bewitching blazing blended blessed -blissed -blissful -blooming -bodacious -bonafide -bonny +boosted borne -boundless -braised brave breathtaking -breezy brewed bright -brightened brilliant -broiled -brooded brought -bubbly built -buoyant -buoyed burning calm calmed -canonized -captivating +candid caring carried catchy @@ -129,66 +85,49 @@ celestial certain championed changed -channeled charismatic -charmed charming chased -chaste cheered cheerful cherished -cherubic chic -chivalrous -choicest chosen -chromatic -churned cinematic clad -clapped classic +classy clear -clear-minded coached +coherent +collected color -colors colorful +colors colossal combined -comely -comforted comforting commanding -commendable -commended committed compassionate compatible complete complex -complimented +complimentary +composed composition comprehensive conceived conferred confident -congruent -congruous connected -consecrated considerable considered consistent -consoled -consonant conspicuous constructed constructive -consummate contemplated -contemplative contemporary content contrasted @@ -196,54 +135,36 @@ conveyed cooked cool coordinated -coruscating -cosmic coupled courageous -courted coveted cozy -craved created creative +credited crisp critical -crystal-clear -cuddly cultivated cured curious current -curvaceous customized cute -cutting-edge -dainty -dapper daring -dashing -dauntless -daydreamed +darling dazzling -dazzlingly -decked decorated decorative -decorous dedicated deep defended definitive -deific -deified -deliberated delicate delightful delivered -demonstrative -dependable depicted designed +desirable desired destined detail @@ -252,9 +173,8 @@ determined developed devoted devout -diaphanous -dignified diligent +direct directed discovered dispatched @@ -264,77 +184,42 @@ distinct distinctive distinguished diverse -diversified divine dramatic draped dreamed -dreamlike -dreamy driven -drop-dead dynamic earnest eased ecstatic -edified -edifying educated effective -effervescent -effulgent -effusive elaborate -elated -electrifying elegant elevated elite -eloquent -embellished -emboldened eminent emotional -empathetic -empathic -emphatic empowered empowering -enchanting -enchantingly +enchanted encouraged -encrusted -endearing -endeavored endorsed endowed enduring energetic -energized -energizing engaging +enhanced enigmatic enlightened -enlivened enormous -enraptured -enrapturing -enthralling -enticed enticing -entrancing envisioned epic -equanimous esteemed eternal -ethereal -ethically -eulogized -euphonious -euphoric -evocative -evocatively +everlasting evolved exalted examining @@ -343,646 +228,413 @@ exceptional exciting exclusive exemplary -exhilarating +exotic expansive -expedited -explorative exposed expressive -expressively exquisite -extemporized -extolled -extradited +extended extraordinary extremely -exuberant -eye-catching fabulous facilitated fair -fairy-tale-like +faithful +famous fancy -fantasied fantastic -fantastical fascinating fashionable fashioned -fathomless favorable favored fearless -feathery -felicitous fermented -fervent +fertile festive -fetched -fetching fiery fine finest firm -first-class -first-rate fixed -flamboyant flaming flashing flashy -flattered flavored flawless -flimsy -florid flourishing -flowery +flowing focus focused formal formed -fortuitous fortunate -forward-thinking -fostered fostering -found -foxy -fragranced -free-spirited +frank fresh fried +friendly +fruitful fulfilled -fulgent +full futuristic -galactic -gallant -galvanized -garbed -gargantuan -garnished -gemmed generous gentle -gentlemanly genuine gifted gigantic -given glamorous -gleaming -gleamy -glistening -glittering -glittery -glitzy -glorified glorious glossy glowing -godly gorgeous -gossamer graceful gracious grand -grandiose granted grateful -gratified +great grilled +grounded +grown guarded guided hailed -hallowed handsome -hardy -harmonious -harmonized -hastened -head-turning -heartened +healing +healthy heartfelt -heartwarming heavenly heroic -hoisted +historic holistic holy +honest honored hoped hopeful -huggable -hushed -hypnotic +iconic ideal -idealized -idyllic illuminated illuminating illumination -illustrative illustrious imaginative imagined -imagistic -imbued -immaculate immense immortal -impeccable imposing impressive -incandescent -incisive -indomitable +improved +incredible infinite -inflamed informed +ingenious innocent -innovated innovative -inproportion -inquisitive -insatiable -inshape insightful -insistent +inspirational inspired inspiring instructed integrated intense -intoxicating -intrepid intricate +intriguing invaluable invented investigative -invigorated -invigorating invincible inviting -iridescent -jaunty -jaw-dropping -jazzy -jeweled +irresistible joined joyful -joyous -jubilant -judicious keen -kind-hearted -kindhearted +kindly kinetic -kingly -kissable -kneaded -knightly knockout laced -ladylike -lambent lasting -laudable -laudatory lauded lavish -leading-edge legendary -lifelike lifted light -lightened -lighthearted -likable limited linked -lionized -lithe lively located logical -longed -lordly -lovable loved lovely -lucent +loving +loyal lucid lucky -luminous lush -lustrous luxurious luxury +magic magical magnificent majestic -mammoth -manifesting marked marvelous massive -masterful matched -matchless matured meaningful -meditated -meditative -mellifluous -mellow -mentored +memorable merged merry -mesmerizing -methodical meticulous -mind-blowing mindful miraculous -mirthful modern modified -modish -molded -monarchical monstrous monumental motivated motivational moved -multifaceted -multifarious -mused +moving mystical mythical naive -natty -navigated neat new +nice nifty noble notable noteworthy novel nuanced -nubile -nurtured -nurturing -odorous offered -opalescent +open optimal optimistic -opulent orderly organized original originated -ornate -ostentatious -otherworldly -outfitted outstanding -overjoyed -pacified +overwhelming paired palpable -panoramic -paradisiacal passionate -patterned peaceful -peerless -peppy perfect -perfumed -perky +perfected perpetual -persevering persistent -photogenic -pickled -pictorial -picturesque -piloted +phenomenal pious pivotal placed -placid planned +pleasant pleased pleasing +plentiful plotted plush -plushy -poached +poetic +poignant polished -pondered -posh positive praised -praiseworthy precious precise -preeminent premier premium -prepossessing presented preserved +prestigious pretty priceless prime -princely pristine probing -prodigious +productive professional profound +progressed progressive prominent promoted pronounced propelled proportional +prosperous protected provided -pulchritudinous -pungent +provocative pure pursued pushed +quaint quality -queenly questioning -quickened quiet -quintessential -racy radiant rare rational -ravishing real reborn -recharged reclaimed recognized recovered -redolent -reenergized refined reflected refreshed refreshing -refulgent -regal -regenerated -reinvigorated -rejuvenated -rejuvenating related relaxed relentless +reliable relieved remarkable renewed +renowned representative rescued resilient -resolute respected -resplendent +respectful restored retrieved revealed +revealing revered -reverent -revitalized -revitalizing revived rewarded -rhythmic rich -ripened -ritzy roasted -robed robust -rococo romantic royal -ruminated sacred -saintly salient -saluted -sanctified satisfied +satisfying saturated saved scenic -scented -schemed scientific -sculpted -second-to-none select -seraphic -serendipitous -serene -serenity +sensational serious set shaped -shapely sharp shielded -shimmering shining shiny -shipshape -show-stopping shown -showy -sightly significant silent -silky -simmered sincere singular situated sleek slick smart -smelly -snazzy snug -snuggly solemn solid -soothed soothing sophisticated sought -soulful -spangled sparkling special spectacular sped -spellbinding -spiced -spiffy spirited spiritual -splashy splendid -sprightly -spruce -spunky +spread stable -stagey -starry -stately -statuesque -steamed -steered -stewed +steady still stimulated stimulating stirred +straightforward striking strong structured -studded stunning -stupendous sturdy stylish sublime -sumptuous -sun-kissed +successful sunny superb superior -superlative supplied supported +supportive supreme sure surreal -swank -swanky sweet -sweetened -swish symbolic symmetry -symmetrical synchronized systematic tailored taking -tangy -tantalizing targeted -tasteful taught -tenacious +tempting tender +terrific thankful theatrical thought +thoughtful thrilled thrilling +thriving tidy timeless -titanic -toasted -top-drawer -top-notch -top-tier -topnotch +touching tough trained tranquil -tranquilized -transcendent -transcendental transformed translucent transparent transported -treasured +tremendous trendy tried trim true -twinkling -twinned -unblemished -uncluttered +trustworthy +unbelievable unconditional uncovered -unequaled -unfeigned +unified unique united universal unmatched unparalleled -unrivaled -unsullied -unsurpassed -unyielding -up-to-date upheld -uplifted -uplifting valiant valued varied -velvety -venerated -venturesome vibrant -vinted -virginal virtuous -visualized -vitalized -vivacious vivid -vivified -voguish warm -well-built -well-formed -well-kept -well-proportioned -well-turned -whimsical -whisked +wealthy whole winning -winsome wished -wisp-like +witty wonderful -wondrous -wooed -worshiped worshipped -yearned -zesty -zippy +worthy diff --git a/modules/expansion.py b/modules/expansion.py index a536fad..e26f722 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -33,13 +33,20 @@ class FooocusExpansion: positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'), encoding='utf-8').read().splitlines() - positive_words = ['Ġ' + x for x in positive_words if x != ''] + positive_words = ['Ġ' + x.lower() for x in positive_words if x != ''] self.logits_bias = torch.zeros((1, len(self.tokenizer.vocab)), dtype=torch.float32) + neg_inf + debug_list = [] for k, v in self.tokenizer.vocab.items(): if k in positive_words: self.logits_bias[0, v] = 0 + debug_list.append(k[1:]) + + print(f'Fooocus V2 Expansion: Vocab with {len(debug_list)} words.') + + # debug_list = '\n'.join(sorted(debug_list)) + # print(debug_list) # t11 = self.tokenizer(',', return_tensors="np") # t198 = self.tokenizer('\n', return_tensors="np") From c62fc6e1bce955084454d3a7f8d07aeb1f3703b4 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 1 Nov 2023 15:55:59 -0700 Subject: [PATCH 75/78] speedup --- fooocus_version.py | 2 +- modules/expansion.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fooocus_version.py b/fooocus_version.py index 0bbac3c..80f1f10 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.1.773' +version = '2.1.774' diff --git a/modules/expansion.py b/modules/expansion.py index e26f722..64d3f07 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -75,9 +75,12 @@ class FooocusExpansion: @torch.inference_mode() def logits_processor(self, input_ids, scores): assert scores.ndim == 2 and scores.shape[0] == 1 - bias = self.logits_bias.to(scores).clone() + self.logits_bias = self.logits_bias.to(scores) + + bias = self.logits_bias.clone() bias[0, input_ids[0].to(bias.device).long()] = neg_inf bias[0, 11] = 0 + return scores + bias @torch.no_grad() From 9ced4fe6510f4c934d0ad47428488b4949ed67ab Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 1 Nov 2023 22:20:30 -0700 Subject: [PATCH 76/78] highly --- models/prompt_expansion/fooocus_expansion/positive.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/models/prompt_expansion/fooocus_expansion/positive.txt b/models/prompt_expansion/fooocus_expansion/positive.txt index 0a1f7e4..ab04072 100644 --- a/models/prompt_expansion/fooocus_expansion/positive.txt +++ b/models/prompt_expansion/fooocus_expansion/positive.txt @@ -306,6 +306,7 @@ healthy heartfelt heavenly heroic +highly historic holistic holy From 32c282daab3ddb0c7cfd12166a2f5e526dc5a420 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 1 Nov 2023 22:24:34 -0700 Subject: [PATCH 77/78] very --- models/prompt_expansion/fooocus_expansion/positive.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/models/prompt_expansion/fooocus_expansion/positive.txt b/models/prompt_expansion/fooocus_expansion/positive.txt index ab04072..9be121d 100644 --- a/models/prompt_expansion/fooocus_expansion/positive.txt +++ b/models/prompt_expansion/fooocus_expansion/positive.txt @@ -627,6 +627,7 @@ upheld valiant valued varied +very vibrant virtuous vivid From 51d87ce00bb67c7afceff8d54551d6c4af1626c6 Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Wed, 1 Nov 2023 22:43:43 -0700 Subject: [PATCH 78/78] license --- modules/expansion.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/expansion.py b/modules/expansion.py index 64d3f07..8f9507c 100644 --- a/modules/expansion.py +++ b/modules/expansion.py @@ -1,3 +1,10 @@ +# Fooocus GPT2 Expansion +# Algorithm created by Lvmin Zhang at 2023, Stanford +# If used inside Fooocus, any use is permitted. +# If used outside Fooocus, only non-commercial use is permitted (CC-By NC 4.0). +# This applies to the word list, vocab, model, and algorithm. + + import os import torch import math