hi,I encountered a problem when using sdxl and controlnet together. The weight of sdxl will be replaced by the weight combined with lora.follow is my code:
import sdkit
from sdkit.generate import generate_images
from sdkit.models import load_model
from sdkit.utils import log
from PIL import Image
context = sdkit.Context()
convert an existing image into an openpose image (or skip these lines if you have a custom openpose image)
from sdkit.filter import apply_filters
input_image = Image.open("../2.png") # set your input image here <---------
canny_image = apply_filters(context, "canny", input_image) # you can apply other filters too, e.g. "canny" etc.
print(canny_image)
load SD
context.model_paths["stable-diffusion"] = "../sdxl10/stable-diffusion-xl-base-1.0/sd_xl_base_1.0_0.9vae.safetensors" # <---- SD model path here
load_model(context, "stable-diffusion")
load controlnet
context.model_paths["controlnet"] = "../sdxl10/controlnet-canny-sdxl-1.0-mid/diffusion_pytorch_model.fp16.safetensors" # <----- Controlnet model path
load_model(context, "controlnet")
generate the image
image = generate_images(
context,
"1boy, muscular, full armor, armor, shoulder plates, angry, looking at viewer, spiked hair, white hair",
seed=42,
width=1024,
height=1024,
control_image=canny_image,
)[0]
save the image
image.save("controlnet_image.jpg")
log.info("Generated images with a Controlnet!")
error code:
10:30:12.705 INFO MainThread Applying canny...
Traceback (most recent call last):
File "/media/alleniver/mydata/diffusion2/web_server/sdkit/examples/009-generate-controlnet.py", line 16, in
canny_image = apply_filters(context, "canny", input_image) # you can apply other filters too, e.g. "canny" etc.
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 39, in apply_filters
return [apply_filter_single_image(context, filters, image, **kwargs) for image in images]
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 39, in
return [apply_filter_single_image(context, filters, image, **kwargs) for image in images]
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 49, in apply_filter_single_image
image = get_filter_module(filter_type).apply(context, image, **kwargs)
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 59, in default_apply
model = context.models[filter_type]
KeyError: 'canny'
same error with 'depth' weights.
hi,I encountered a problem when using sdxl and controlnet together. The weight of sdxl will be replaced by the weight combined with lora.follow is my code:
import sdkit
from sdkit.generate import generate_images
from sdkit.models import load_model
from sdkit.utils import log
from PIL import Image
context = sdkit.Context()
convert an existing image into an openpose image (or skip these lines if you have a custom openpose image)
from sdkit.filter import apply_filters
input_image = Image.open("../2.png") # set your input image here <---------
canny_image = apply_filters(context, "canny", input_image) # you can apply other filters too, e.g. "canny" etc.
print(canny_image)
load SD
context.model_paths["stable-diffusion"] = "../sdxl10/stable-diffusion-xl-base-1.0/sd_xl_base_1.0_0.9vae.safetensors" # <---- SD model path here
load_model(context, "stable-diffusion")
load controlnet
context.model_paths["controlnet"] = "../sdxl10/controlnet-canny-sdxl-1.0-mid/diffusion_pytorch_model.fp16.safetensors" # <----- Controlnet model path
load_model(context, "controlnet")
generate the image
image = generate_images(
context,
"1boy, muscular, full armor, armor, shoulder plates, angry, looking at viewer, spiked hair, white hair",
seed=42,
width=1024,
height=1024,
control_image=canny_image,
)[0]
save the image
image.save("controlnet_image.jpg")
log.info("Generated images with a Controlnet!")
error code:
10:30:12.705 INFO MainThread Applying canny...
Traceback (most recent call last):
File "/media/alleniver/mydata/diffusion2/web_server/sdkit/examples/009-generate-controlnet.py", line 16, in
canny_image = apply_filters(context, "canny", input_image) # you can apply other filters too, e.g. "canny" etc.
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 39, in apply_filters
return [apply_filter_single_image(context, filters, image, **kwargs) for image in images]
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 39, in
return [apply_filter_single_image(context, filters, image, **kwargs) for image in images]
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 49, in apply_filter_single_image
image = get_filter_module(filter_type).apply(context, image, **kwargs)
File "/home/alleniver/anaconda3/envs/edtool/lib/python3.10/site-packages/sdkit/filter/apply_filters.py", line 59, in default_apply
model = context.models[filter_type]
KeyError: 'canny'
same error with 'depth' weights.