From 8c76967f26b482b19c83a65045304fdf4a78189c Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:37:16 -0400 Subject: [PATCH 01/61] adding ray.. --- modules/api/ray.py | 34 ++++++++++++++++++++++++++++++++++ modules/launch_utils.py | 4 +++- webui.py | 11 +++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 modules/api/ray.py diff --git a/modules/api/ray.py b/modules/api/ray.py new file mode 100644 index 00000000000..e6f3038e0c7 --- /dev/null +++ b/modules/api/ray.py @@ -0,0 +1,34 @@ +from ray import serve +import ray +from fastapi import FastAPI +from modules.api.api import Api +from modules.call_queue import queue_lock +from modules import initialize_util + + + +ray.init() + + +NUM_REPLICAS: int = 4 +if NUM_REPLICAS > ray.available_resources()["GPU"]: + print( + "Your cluster does not currently have enough resources to run with these settings. " + "Consider decreasing the number of workers, or decreasing the resources needed " + "per worker. Ignore this if your cluster auto-scales." + ) + + +app = FastAPI() +initialize_util.setup_middleware(app) +api = Api(app, queue_lock) + +@serve.deployment( + ray_actor_options={"num_gpus": 1}, + num_replicas=NUM_REPLICAS, +) +@serve.ingress(api) +class APIIngress(Api): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 6e54d06367c..9faf040ef62 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -428,10 +428,12 @@ def configure_for_tests(): def start(): - print(f"Launching {'API server' if '--nowebui' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}") + print(f"Launching {'API server' if '--nowebui' or '--ray' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}") import webui if '--nowebui' in sys.argv: webui.api_only() + elif '--ray' in sys.argv: + webui.ray_only() else: webui.webui() diff --git a/webui.py b/webui.py index 12328423d0d..061a44ec28e 100644 --- a/webui.py +++ b/webui.py @@ -44,7 +44,18 @@ def api_only(): root_path=f"/{cmd_opts.subpath}" if cmd_opts.subpath else "" ) +def ray_only(): + from modules.ray import APIIngress + entrypoint = APIIngress.bind() + port = 8000 + + # Shutdown any existing Serve replicas, if they're still around. + serve.shutdown() + serve.run(entrypoint, port=port, name="serving_stable_diffusion_template") + print("Done setting up replicas! Now accepting requests...") + + def webui(): from modules.shared_cmd_options import cmd_opts From b6648631552ad9de8d653a29e218785c2c4385de Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 28 Sep 2023 12:42:58 -0400 Subject: [PATCH 02/61] Update cmd_args.py --- modules/cmd_args.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/cmd_args.py b/modules/cmd_args.py index aab62286e24..39a41094929 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -117,3 +117,4 @@ parser.add_argument('--timeout-keep-alive', type=int, default=30, help='set timeout_keep_alive for uvicorn') parser.add_argument("--disable-all-extensions", action='https://github.com/store_true', help="prevent all extensions from running regardless of any other settings", default=False) parser.add_argument("--disable-extra-extensions", action='https://github.com/store_true', help=" prevent all extensions except built-in from running regardless of any other settings", default=False) +parser.add_argument("--ray", action='https://github.com/store_true', help="use api=True to launch the ray API instead of the webui") \ No newline at end of file From 7901912c5094656b6e8b73dbb27f8e9d4b7faad8 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 28 Sep 2023 16:00:04 -0400 Subject: [PATCH 03/61] adding ray --- modules/api/api.py | 6 ++++-- modules/api/ray.py | 12 +++++++++--- modules/shared_items.py | 6 +++++- requirements.txt | 2 ++ requirements_versions.txt | 2 ++ webui-user.sh | 1 + webui.py | 6 +++--- 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/modules/api/api.py b/modules/api/api.py index e6edffe7144..fea1dc4d73d 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -198,7 +198,7 @@ async def http_exception_handler(request: Request, e: HTTPException): class Api: - def __init__(self, app: FastAPI, queue_lock: Lock): + def __init__(self, app: FastAPI): if shared.cmd_opts.api_auth: self.credentials = {} for auth in shared.cmd_opts.api_auth.split(","): @@ -207,8 +207,9 @@ def __init__(self, app: FastAPI, queue_lock: Lock): self.router = APIRouter() self.app = app - self.queue_lock = queue_lock + #self.queue_lock = queue_lock api_middleware(self.app) + print("API initialized") self.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=models.TextToImageResponse) self.add_api_route("/sdapi/v1/img2img", self.img2imgapi, methods=["POST"], response_model=models.ImageToImageResponse) self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ExtrasSingleImageResponse) @@ -252,6 +253,7 @@ def __init__(self, app: FastAPI, queue_lock: Lock): self.default_script_arg_txt2img = [] self.default_script_arg_img2img = [] + def add_api_route(self, path: str, endpoint, **kwargs): if shared.cmd_opts.api_auth: return self.app.add_api_route(path, endpoint, dependencies=[Depends(self.auth)], **kwargs) diff --git a/modules/api/ray.py b/modules/api/ray.py index e6f3038e0c7..d479f006842 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -7,8 +7,8 @@ -ray.init() - +#ray.init() +ray.init("ray://localhost:10001") NUM_REPLICAS: int = 4 if NUM_REPLICAS > ray.available_resources()["GPU"]: @@ -21,7 +21,11 @@ app = FastAPI() initialize_util.setup_middleware(app) -api = Api(app, queue_lock) +#api = Api(app) +api = Api(app) +# Try to serialize the Api class +#pickle.dumps(api) + @serve.deployment( ray_actor_options={"num_gpus": 1}, @@ -32,3 +36,5 @@ class APIIngress(Api): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) + + diff --git a/modules/shared_items.py b/modules/shared_items.py index 84d69c8df43..301b32873d2 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -115,5 +115,9 @@ def sd_model(self, value): modules.sd_models.model_data.set_sd_model(value) + def __reduce__(self): + # The callable is the class itself + # The arguments are the current values of the sd_model property + return (self.__class__, (self.sd_model,)) -sys.modules['modules.shared'].__class__ = Shared +#sys.modules['modules.shared'].__class__ = Shared diff --git a/requirements.txt b/requirements.txt index 80b438455ce..ed2443b2de2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,3 +32,5 @@ torch torchdiffeq torchsde transformers==4.30.2 +ray[default] +ray[serve] diff --git a/requirements_versions.txt b/requirements_versions.txt index f8ae1f385ae..9da032d0c4f 100644 --- a/requirements_versions.txt +++ b/requirements_versions.txt @@ -29,3 +29,5 @@ torch torchdiffeq==0.2.3 torchsde==0.2.5 transformers==4.30.2 +ray +ray[serve] diff --git a/webui-user.sh b/webui-user.sh index 70306c60d5b..8af458ea577 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -11,6 +11,7 @@ # Commandline arguments for webui.py, for example: export COMMANDLINE_ARGS="--medvram --opt-split-attention" #export COMMANDLINE_ARGS="" +export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen '--ray'" # python3 executable #python_cmd="python3" diff --git a/webui.py b/webui.py index 061a44ec28e..a911a53edb8 100644 --- a/webui.py +++ b/webui.py @@ -45,8 +45,8 @@ def api_only(): ) def ray_only(): - from modules.ray import APIIngress - + from modules.api.ray import APIIngress + from ray import serve entrypoint = APIIngress.bind() port = 8000 @@ -55,7 +55,7 @@ def ray_only(): serve.run(entrypoint, port=port, name="serving_stable_diffusion_template") print("Done setting up replicas! Now accepting requests...") - + def webui(): from modules.shared_cmd_options import cmd_opts From 73c28677f642a2f2910dcdbd07093cafb8164b31 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 28 Sep 2023 16:11:08 -0400 Subject: [PATCH 04/61] Update ray.py --- modules/api/ray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index d479f006842..15533b593c4 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -10,7 +10,7 @@ #ray.init() ray.init("ray://localhost:10001") -NUM_REPLICAS: int = 4 +NUM_REPLICAS: int = 1 if NUM_REPLICAS > ray.available_resources()["GPU"]: print( "Your cluster does not currently have enough resources to run with these settings. " From 6d3fca5239fb33f9f924234c7e29de900784b474 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 28 Sep 2023 16:45:20 -0400 Subject: [PATCH 05/61] minor fixes --- modules/api/ray.py | 4 ++-- webui-user.sh | 2 +- webui.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 15533b593c4..3b10ea89d89 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -7,8 +7,8 @@ -#ray.init() -ray.init("ray://localhost:10001") +ray.init() +#ray.init("ray://localhost:10001") NUM_REPLICAS: int = 1 if NUM_REPLICAS > ray.available_resources()["GPU"]: diff --git a/webui-user.sh b/webui-user.sh index 8af458ea577..009ab6e8970 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -11,7 +11,7 @@ # Commandline arguments for webui.py, for example: export COMMANDLINE_ARGS="--medvram --opt-split-attention" #export COMMANDLINE_ARGS="" -export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen '--ray'" +export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen --ray" # python3 executable #python_cmd="python3" diff --git a/webui.py b/webui.py index a911a53edb8..58342579146 100644 --- a/webui.py +++ b/webui.py @@ -47,12 +47,10 @@ def api_only(): def ray_only(): from modules.api.ray import APIIngress from ray import serve - entrypoint = APIIngress.bind() - port = 8000 # Shutdown any existing Serve replicas, if they're still around. serve.shutdown() - serve.run(entrypoint, port=port, name="serving_stable_diffusion_template") + serve.run(APIIngress.bind(), port=8000, name="serving_stable_diffusion_template") print("Done setting up replicas! Now accepting requests...") @@ -169,5 +167,7 @@ def webui(): if cmd_opts.nowebui: api_only() + elif cmd_opts.ray: + ray_only() else: webui() From c0a9a3947d0120ac0281cad1805074fbab95b937 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 29 Sep 2023 10:31:27 -0400 Subject: [PATCH 06/61] adding args to shared.py --- modules/cmd_args.py | 2 +- modules/shared.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 39a41094929..c286f1b4e50 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -117,4 +117,4 @@ parser.add_argument('--timeout-keep-alive', type=int, default=30, help='set timeout_keep_alive for uvicorn') parser.add_argument("--disable-all-extensions", action='https://github.com/store_true', help="prevent all extensions from running regardless of any other settings", default=False) parser.add_argument("--disable-extra-extensions", action='https://github.com/store_true', help=" prevent all extensions except built-in from running regardless of any other settings", default=False) -parser.add_argument("--ray", action='https://github.com/store_true', help="use api=True to launch the ray API instead of the webui") \ No newline at end of file +parser.add_argument("--ray", action='https://github.com/store_true', help="use api=True to launch the ray API instead of the webui") diff --git a/modules/shared.py b/modules/shared.py index 636619391fc..c809a6c79bd 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -85,3 +85,25 @@ refresh_checkpoints = shared_items.refresh_checkpoints list_samplers = shared_items.list_samplers reload_hypernetworks = shared_items.reload_hypernetworks + + +# Add a few dummy flags from possibly Ray server. +# This is a limitation from Ray and will be fixed in future version. +parser.add_argument("--node-ip-address", type=str, default="") +parser.add_argument("--node-manager-port", type=str, default="") +parser.add_argument("--object-store-name", type=str, default="") +parser.add_argument("--raylet-name", type=str, default="") +parser.add_argument("--redis-address", type=str, default="") +parser.add_argument("--temp-dir", type=str, default="") +parser.add_argument("--metrics-agent-port", type=str, default="") +#parser.add_argument("--runtime-env-agent-port", type=str, default="") +parser.add_argument("--logging-rotate-bytes", type=str, default="") +parser.add_argument("--logging-rotate-backup-count", type=str, default="") +parser.add_argument("--runtime-env-agent-port", type=str, default="") +parser.add_argument("--gcs-address", type=str, default="") +parser.add_argument("--session-name", type=str, default="") +#parser.add_argument("--temp-dir", type=str, default="") +parser.add_argument("--webui", type=str, default="") +parser.add_argument("--cluster-id", type=str, default="") +parser.add_argument("--startup-token", type=str, default="") +parser.add_argument("--worker-launch-time-ms", type=str, default="") \ No newline at end of file From 9e514b3a58cef900cc37fde2ac81c12664cfb01d Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 29 Sep 2023 16:35:44 -0400 Subject: [PATCH 07/61] getting stuff right, but circular import happening --- .gitignore | 2 ++ modules/api/ray.py | 19 ++++++++++++------- modules/cmd_args.py | 22 ++++++++++++++++++++++ modules/shared.py | 24 +----------------------- webui-user.sh | 4 ++-- webui.py | 3 ++- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 09734267ff5..841e25d966d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ notification.mp3 /node_modules /package-lock.json /.coverage* + +sd-data \ No newline at end of file diff --git a/modules/api/ray.py b/modules/api/ray.py index 3b10ea89d89..d548a8d06a8 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -4,8 +4,13 @@ from modules.api.api import Api from modules.call_queue import queue_lock from modules import initialize_util +from modules import script_callbacks +from modules import initialize +initialize.imports() + +initialize.check_versions() ray.init() #ray.init("ray://localhost:10001") @@ -18,13 +23,16 @@ "per worker. Ignore this if your cluster auto-scales." ) - +initialize.initialize() app = FastAPI() initialize_util.setup_middleware(app) -#api = Api(app) + + api = Api(app) -# Try to serialize the Api class -#pickle.dumps(api) + +script_callbacks.before_ui_callback() +script_callbacks.app_started_callback(None, app) + @serve.deployment( @@ -35,6 +43,3 @@ class APIIngress(Api): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - - - diff --git a/modules/cmd_args.py b/modules/cmd_args.py index c286f1b4e50..8168fa4765b 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -118,3 +118,25 @@ parser.add_argument("--disable-all-extensions", action='https://github.com/store_true', help="prevent all extensions from running regardless of any other settings", default=False) parser.add_argument("--disable-extra-extensions", action='https://github.com/store_true', help=" prevent all extensions except built-in from running regardless of any other settings", default=False) parser.add_argument("--ray", action='https://github.com/store_true', help="use api=True to launch the ray API instead of the webui") + + +# Add a few dummy flags from possibly Ray server. +# This is a limitation from Ray and will be fixed in future version. +parser.add_argument("--node-ip-address", type=str, default="") +parser.add_argument("--node-manager-port", type=str, default="") +parser.add_argument("--object-store-name", type=str, default="") +parser.add_argument("--raylet-name", type=str, default="") +parser.add_argument("--redis-address", type=str, default="") +parser.add_argument("--temp-dir", type=str, default="") +parser.add_argument("--metrics-agent-port", type=str, default="") +#parser.add_argument("--runtime-env-agent-port", type=str, default="") +parser.add_argument("--logging-rotate-bytes", type=str, default="") +parser.add_argument("--logging-rotate-backup-count", type=str, default="") +parser.add_argument("--runtime-env-agent-port", type=str, default="") +parser.add_argument("--gcs-address", type=str, default="") +parser.add_argument("--session-name", type=str, default="") +#parser.add_argument("--temp-dir", type=str, default="") +parser.add_argument("--webui", type=str, default="") +parser.add_argument("--cluster-id", type=str, default="") +parser.add_argument("--startup-token", type=str, default="") +parser.add_argument("--worker-launch-time-ms", type=str, default="") \ No newline at end of file diff --git a/modules/shared.py b/modules/shared.py index c809a6c79bd..d351c1f1efc 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -84,26 +84,4 @@ list_checkpoint_tiles = shared_items.list_checkpoint_tiles refresh_checkpoints = shared_items.refresh_checkpoints list_samplers = shared_items.list_samplers -reload_hypernetworks = shared_items.reload_hypernetworks - - -# Add a few dummy flags from possibly Ray server. -# This is a limitation from Ray and will be fixed in future version. -parser.add_argument("--node-ip-address", type=str, default="") -parser.add_argument("--node-manager-port", type=str, default="") -parser.add_argument("--object-store-name", type=str, default="") -parser.add_argument("--raylet-name", type=str, default="") -parser.add_argument("--redis-address", type=str, default="") -parser.add_argument("--temp-dir", type=str, default="") -parser.add_argument("--metrics-agent-port", type=str, default="") -#parser.add_argument("--runtime-env-agent-port", type=str, default="") -parser.add_argument("--logging-rotate-bytes", type=str, default="") -parser.add_argument("--logging-rotate-backup-count", type=str, default="") -parser.add_argument("--runtime-env-agent-port", type=str, default="") -parser.add_argument("--gcs-address", type=str, default="") -parser.add_argument("--session-name", type=str, default="") -#parser.add_argument("--temp-dir", type=str, default="") -parser.add_argument("--webui", type=str, default="") -parser.add_argument("--cluster-id", type=str, default="") -parser.add_argument("--startup-token", type=str, default="") -parser.add_argument("--worker-launch-time-ms", type=str, default="") \ No newline at end of file +reload_hypernetworks = shared_items.reload_hypernetworks \ No newline at end of file diff --git a/webui-user.sh b/webui-user.sh index 009ab6e8970..a6fcaf4d327 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -11,7 +11,7 @@ # Commandline arguments for webui.py, for example: export COMMANDLINE_ARGS="--medvram --opt-split-attention" #export COMMANDLINE_ARGS="" -export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen --ray" +export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen --ray" #--ray # python3 executable #python_cmd="python3" @@ -41,7 +41,7 @@ export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --preci #export BLIP_COMMIT_HASH="" # Uncomment to enable accelerated launch -#export ACCELERATE="True" +export ACCELERATE="True" # Uncomment to disable TCMalloc #export NO_TCMALLOC="True" diff --git a/webui.py b/webui.py index 58342579146..6f553107c47 100644 --- a/webui.py +++ b/webui.py @@ -19,7 +19,8 @@ def create_api(app): from modules.api.api import Api from modules.call_queue import queue_lock - api = Api(app, queue_lock) + api = Api(app) + #api = Api(app, queue_lock) return api From 66bc6a51388079e513069c61a7d48e88e6f0ea2a Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 29 Sep 2023 17:10:28 -0400 Subject: [PATCH 08/61] closer.. fixing circular imports --- modules/api/ray.py | 1 + modules/processing.py | 9 ++++++++- modules/sd_samplers_common.py | 6 +++--- modules/sd_samplers_kdiffusion.py | 9 +++++---- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index d548a8d06a8..c13bb193716 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -8,6 +8,7 @@ from modules import initialize + initialize.imports() initialize.check_versions() diff --git a/modules/processing.py b/modules/processing.py index e124e7f0dd2..351727e6b0b 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -19,7 +19,6 @@ from modules import devices, prompt_parser, masking, sd_samplers, lowvram, generation_parameters_copypaste, extra_networks, sd_vae_approx, scripts, sd_samplers_common, sd_unet, errors, rng from modules.rng import slerp # noqa: F401 from modules.sd_hijack import model_hijack -from modules.sd_samplers_common import images_tensor_to_samples, decode_first_stage, approximation_indexes from modules.shared import opts, cmd_opts, state import modules.shared as shared import modules.paths as paths @@ -89,6 +88,7 @@ def create_binary_mask(image): return image def txt2img_image_conditioning(sd_model, x, width, height): + from modules.sd_samplers_common import images_tensor_to_samples, approximation_indexes if sd_model.model.conditioning_key in {'hybrid', 'concat'}: # Inpainting models # The "masked-image" in this case will just be all 0.5 since the entire image is masked. @@ -277,6 +277,7 @@ def txt2img_image_conditioning(self, x, width=None, height=None): return txt2img_image_conditioning(self.sd_model, x, width or self.width, height or self.height) def depth2img_image_conditioning(self, source_image): + from modules.sd_samplers_common import images_tensor_to_samples, approximation_indexes # Use the AddMiDaS helper to Format our source image to suit the MiDaS model transformer = AddMiDaS(model_type="dpt_hybrid") transformed = transformer({"jpg": rearrange(source_image[0], "c h w -> h w c")}) @@ -296,6 +297,7 @@ def depth2img_image_conditioning(self, source_image): return conditioning def edit_image_conditioning(self, source_image): + from modules.sd_samplers_common import images_tensor_to_samples, approximation_indexes conditioning_image = images_tensor_to_samples(source_image*0.5+0.5, approximation_indexes.get(opts.sd_vae_encode_method)) return conditioning_image @@ -588,6 +590,7 @@ class DecodedSamples(list): def decode_latent_batch(model, batch, target_device=None, check_for_nans=False): + from modules.sd_samplers_common import decode_first_stage samples = DecodedSamples() for i in range(batch.shape[0]): @@ -1156,6 +1159,7 @@ def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subs return self.sample_hr_pass(samples, decoded_samples, seeds, subseeds, subseed_strength, prompts) def sample_hr_pass(self, samples, decoded_samples, seeds, subseeds, subseed_strength, prompts): + if shared.state.interrupted: return samples @@ -1189,6 +1193,7 @@ def save_intermediate(image, index): # Avoid making the inpainting conditioning unless necessary as # this does need some extra compute to decode / encode the image again. if getattr(self, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) < 1.0: + from modules.sd_samplers_common import decode_first_stage , approximation_indexes image_conditioning = self.img2img_image_conditioning(decode_first_stage(self.sd_model, samples), samples) else: image_conditioning = self.txt2img_image_conditioning(samples) @@ -1212,6 +1217,7 @@ def save_intermediate(image, index): decoded_samples = decoded_samples.to(shared.device, dtype=devices.dtype_vae) if opts.sd_vae_encode_method != 'Full': + from modules.sd_samplers_common import images_tensor_to_samples self.extra_generation_params['VAE Encoder'] = opts.sd_vae_encode_method samples = images_tensor_to_samples(decoded_samples, approximation_indexes.get(opts.sd_vae_encode_method)) @@ -1387,6 +1393,7 @@ def mask_blur(self, value): self.mask_blur_y = value def init(self, all_prompts, all_seeds, all_subseeds): + from modules.sd_samplers_common import images_tensor_to_samples, approximation_indexes self.image_cfg_scale: float = self.image_cfg_scale if shared.sd_model.cond_stage_key == "edit" else None self.sampler = sd_samplers.create_sampler(self.sampler_name, self.sd_model) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 58efcad2374..fe1173715ae 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -3,7 +3,7 @@ import numpy as np import torch from PIL import Image -from modules import devices, images, sd_vae_approx, sd_samplers, sd_vae_taesd, shared, sd_models +from modules import devices, images, sd_vae_approx, sd_vae_taesd, shared, sd_models from modules.shared import opts, state import k_diffusion.sampling @@ -122,8 +122,8 @@ def store_latent(decoded): def is_sampler_using_eta_noise_seed_delta(p): """returns whether sampler from config will use eta noise seed delta for image creation""" - - sampler_config = sd_samplers.find_sampler_config(p.sampler_name) + from modules.sd_samplers import find_sampler_config + sampler_config = find_sampler_config(p.sampler_name) eta = p.eta diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index 8a8c87e0d01..4ed3bc1c95e 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -1,7 +1,8 @@ import torch import inspect import k_diffusion.sampling -from modules import sd_samplers_common, sd_samplers_extra, sd_samplers_cfg_denoiser +from modules.sd_samplers_common import SamplerData, Sampler, setup_img2img_steps +from modules import sd_samplers_extra, sd_samplers_cfg_denoiser from modules.sd_samplers_cfg_denoiser import CFGDenoiser # noqa: F401 from modules.script_callbacks import ExtraNoiseParams, extra_noise_callback @@ -40,7 +41,7 @@ samplers_data_k_diffusion = [ - sd_samplers_common.SamplerData(label, lambda model, funcname=funcname: KDiffusionSampler(funcname, model), aliases, options) + SamplerData(label, lambda model, funcname=funcname: KDiffusionSampler(funcname, model), aliases, options) for label, funcname, aliases, options in samplers_k_diffusion if callable(funcname) or hasattr(k_diffusion.sampling, funcname) ] @@ -76,7 +77,7 @@ def inner_model(self): return self.model_wrap -class KDiffusionSampler(sd_samplers_common.Sampler): +class KDiffusionSampler(Sampler): def __init__(self, funcname, sd_model, options=None): super().__init__(funcname) @@ -139,7 +140,7 @@ def get_sigmas(self, p, steps): return sigmas def sample_img2img(self, p, x, noise, conditioning, unconditional_conditioning, steps=None, image_conditioning=None): - steps, t_enc = sd_samplers_common.setup_img2img_steps(p, steps) + steps, t_enc = setup_img2img_steps(p, steps) sigmas = self.get_sigmas(p, steps) sigma_sched = sigmas[steps - t_enc - 1:] From 9f0c3382fbe51cd8c260e5cf0be10d34794be530 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Tue, 3 Oct 2023 09:06:17 -0400 Subject: [PATCH 09/61] getting closer i think --- launch.py | 20 +++++++++++++++++++ modules/api/ray.py | 43 +++++++++++++++++++++++++++-------------- modules/launch_utils.py | 6 ++++++ modules/sd_samplers.py | 2 +- webui-user.sh | 2 +- 5 files changed, 56 insertions(+), 17 deletions(-) diff --git a/launch.py b/launch.py index f83820d2534..6f7697b72f3 100644 --- a/launch.py +++ b/launch.py @@ -22,6 +22,7 @@ prepare_environment = launch_utils.prepare_environment configure_for_tests = launch_utils.configure_for_tests start = launch_utils.start +ray = launch_utils.ray def main(): @@ -44,5 +45,24 @@ def main(): start() +def ray_launch(): + if args.dump_sysinfo: + filename = launch_utils.dump_sysinfo() + + print(f"Sysinfo saved as {filename}. Exiting...") + + exit(0) + + launch_utils.startup_timer.record("initial startup") + + with launch_utils.startup_timer.subcategory("prepare environment"): + if not args.skip_prepare_environment: + prepare_environment() + + if args.test_server: + configure_for_tests() + + ray() + if __name__ == "__main__": main() diff --git a/modules/api/ray.py b/modules/api/ray.py index c13bb193716..1c276b18282 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -9,10 +9,6 @@ -initialize.imports() - -initialize.check_versions() - ray.init() #ray.init("ray://localhost:10001") @@ -24,23 +20,40 @@ "per worker. Ignore this if your cluster auto-scales." ) -initialize.initialize() app = FastAPI() -initialize_util.setup_middleware(app) - - -api = Api(app) - -script_callbacks.before_ui_callback() -script_callbacks.app_started_callback(None, app) - - @serve.deployment( ray_actor_options={"num_gpus": 1}, num_replicas=NUM_REPLICAS, ) -@serve.ingress(api) +@serve.ingress(app) class APIIngress(Api): def __init__(self, *args, **kwargs) -> None: + from launch import ray_launch + #from modules import sd_samplers + ray_launch() + #sd_samplers.set_samplers() + initialize.imports() + initialize.check_versions() + + initialize.initialize() + app = FastAPI() + initialize_util.setup_middleware(app) + + + api = Api(app) + + script_callbacks.before_ui_callback() + script_callbacks.app_started_callback(None, app) + + print(f"Startup time: {startup_timer.summary()}.") + api.launch( + server_name="0.0.0.0" if cmd_opts.listen else "127.0.0.1", + port=cmd_opts.port if cmd_opts.port else 7861, + root_path=f"/{cmd_opts.subpath}" if cmd_opts.subpath else "" + ) + super().__init__(*args, **kwargs) + + + diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 9faf040ef62..510d6ac66be 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -438,6 +438,12 @@ def start(): webui.webui() +def ray(): + print(f"Launching {'API server' if '--nowebui' or '--ray' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}") + import webui + webui.ray_only() + + def dump_sysinfo(): from modules import sysinfo import datetime diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 45faae62821..745b2911a7f 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -56,4 +56,4 @@ def visible_sampler_names(): return [x.name for x in samplers if x.name not in samplers_hidden] -set_samplers() +#set_samplers() diff --git a/webui-user.sh b/webui-user.sh index a6fcaf4d327..b7400b74a0e 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -11,7 +11,7 @@ # Commandline arguments for webui.py, for example: export COMMANDLINE_ARGS="--medvram --opt-split-attention" #export COMMANDLINE_ARGS="" -export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen --ray" #--ray +export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen --ray" # python3 executable #python_cmd="python3" From 9d44dc8878299498addb2fdfbfb75e01af6133fc Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:30:57 -0400 Subject: [PATCH 10/61] basic ray ingress launcher up --- launch.py | 22 +-------------------- modules/api/ray.py | 42 +++++++++++++++-------------------------- modules/launch_utils.py | 9 ++------- webui.py | 10 +--------- 4 files changed, 19 insertions(+), 64 deletions(-) diff --git a/launch.py b/launch.py index 6f7697b72f3..410dc05ccff 100644 --- a/launch.py +++ b/launch.py @@ -22,7 +22,7 @@ prepare_environment = launch_utils.prepare_environment configure_for_tests = launch_utils.configure_for_tests start = launch_utils.start -ray = launch_utils.ray + def main(): @@ -44,25 +44,5 @@ def main(): start() - -def ray_launch(): - if args.dump_sysinfo: - filename = launch_utils.dump_sysinfo() - - print(f"Sysinfo saved as {filename}. Exiting...") - - exit(0) - - launch_utils.startup_timer.record("initial startup") - - with launch_utils.startup_timer.subcategory("prepare environment"): - if not args.skip_prepare_environment: - prepare_environment() - - if args.test_server: - configure_for_tests() - - ray() - if __name__ == "__main__": main() diff --git a/modules/api/ray.py b/modules/api/ray.py index 1c276b18282..39401cfcef2 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -2,16 +2,18 @@ import ray from fastapi import FastAPI from modules.api.api import Api -from modules.call_queue import queue_lock + from modules import initialize_util from modules import script_callbacks from modules import initialize - +import time ray.init() #ray.init("ray://localhost:10001") + + NUM_REPLICAS: int = 1 if NUM_REPLICAS > ray.available_resources()["GPU"]: print( @@ -21,39 +23,25 @@ ) app = FastAPI() +initialize_util.setup_middleware(app) +script_callbacks.app_started_callback(None, app) @serve.deployment( ray_actor_options={"num_gpus": 1}, num_replicas=NUM_REPLICAS, ) @serve.ingress(app) -class APIIngress(Api): +class APIIngress: def __init__(self, *args, **kwargs) -> None: - from launch import ray_launch - #from modules import sd_samplers - ray_launch() - #sd_samplers.set_samplers() - initialize.imports() - initialize.check_versions() - - initialize.initialize() - app = FastAPI() - initialize_util.setup_middleware(app) - - - api = Api(app) - - script_callbacks.before_ui_callback() - script_callbacks.app_started_callback(None, app) - - print(f"Startup time: {startup_timer.summary()}.") - api.launch( - server_name="0.0.0.0" if cmd_opts.listen else "127.0.0.1", - port=cmd_opts.port if cmd_opts.port else 7861, - root_path=f"/{cmd_opts.subpath}" if cmd_opts.subpath else "" - ) + pass - super().__init__(*args, **kwargs) +def ray_only(): + # Shutdown any existing Serve replicas, if they're still around. + serve.shutdown() + serve.run(APIIngress.bind(), port=8000, name="serving_stable_diffusion_template") + print("Done setting up replicas! Now accepting requests...") + while True: + time.sleep(1000) diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 510d6ac66be..7f1dcbe9d68 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -430,20 +430,15 @@ def configure_for_tests(): def start(): print(f"Launching {'API server' if '--nowebui' or '--ray' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}") import webui + from modules.api.ray import ray_only if '--nowebui' in sys.argv: webui.api_only() elif '--ray' in sys.argv: - webui.ray_only() + ray_only() else: webui.webui() -def ray(): - print(f"Launching {'API server' if '--nowebui' or '--ray' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}") - import webui - webui.ray_only() - - def dump_sysinfo(): from modules import sysinfo import datetime diff --git a/webui.py b/webui.py index 6f553107c47..224b4e8d00a 100644 --- a/webui.py +++ b/webui.py @@ -45,15 +45,6 @@ def api_only(): root_path=f"/{cmd_opts.subpath}" if cmd_opts.subpath else "" ) -def ray_only(): - from modules.api.ray import APIIngress - from ray import serve - - # Shutdown any existing Serve replicas, if they're still around. - serve.shutdown() - serve.run(APIIngress.bind(), port=8000, name="serving_stable_diffusion_template") - print("Done setting up replicas! Now accepting requests...") - def webui(): from modules.shared_cmd_options import cmd_opts @@ -165,6 +156,7 @@ def webui(): if __name__ == "__main__": from modules.shared_cmd_options import cmd_opts + from modules.api.ray import ray_only if cmd_opts.nowebui: api_only() From ef71f4bad1d81e514e5cf7fc43904634be6b1b2e Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:47:41 -0400 Subject: [PATCH 11/61] Update ray.py --- modules/api/ray.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/api/ray.py b/modules/api/ray.py index 39401cfcef2..422cb516fc6 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -24,6 +24,8 @@ app = FastAPI() initialize_util.setup_middleware(app) +#api = Api(app) +script_callbacks.before_ui_callback() script_callbacks.app_started_callback(None, app) @serve.deployment( From 5cf14f75b34cc80eff528f1a757ea743bbd67515 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Wed, 4 Oct 2023 11:54:15 -0400 Subject: [PATCH 12/61] getting closer.. --- modules/api/ray.py | 34 ++++++++++++++++++++++++++-------- modules/launch_utils.py | 3 ++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 422cb516fc6..6d74933c28e 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -2,13 +2,14 @@ import ray from fastapi import FastAPI from modules.api.api import Api - from modules import initialize_util from modules import script_callbacks from modules import initialize import time +from modules.shared_cmd_options import cmd_opts + ray.init() #ray.init("ray://localhost:10001") @@ -22,11 +23,15 @@ "per worker. Ignore this if your cluster auto-scales." ) +#initialize.initialize() app = FastAPI() -initialize_util.setup_middleware(app) +#initialize_util.setup_middleware(app) #api = Api(app) -script_callbacks.before_ui_callback() -script_callbacks.app_started_callback(None, app) +#app.include_router(api.router) +#script_callbacks.before_ui_callback() +#script_callbacks.app_started_callback(None, app) + + @serve.deployment( ray_actor_options={"num_gpus": 1}, @@ -34,16 +39,29 @@ ) @serve.ingress(app) class APIIngress: - def __init__(self, *args, **kwargs) -> None: + def __init__(self) -> None: pass + def ray_only(): + from fastapi import FastAPI + from modules.shared_cmd_options import cmd_opts + + initialize.initialize() + + app = FastAPI() + initialize_util.setup_middleware(app) + api = Api(app) + app.include_router(api.router) + + from modules import script_callbacks + script_callbacks.before_ui_callback() + script_callbacks.app_started_callback(None, app) + # Shutdown any existing Serve replicas, if they're still around. serve.shutdown() serve.run(APIIngress.bind(), port=8000, name="serving_stable_diffusion_template") print("Done setting up replicas! Now accepting requests...") while True: - time.sleep(1000) - - + time.sleep(1000) \ No newline at end of file diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 7f1dcbe9d68..88123977249 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -430,10 +430,11 @@ def configure_for_tests(): def start(): print(f"Launching {'API server' if '--nowebui' or '--ray' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}") import webui - from modules.api.ray import ray_only + if '--nowebui' in sys.argv: webui.api_only() elif '--ray' in sys.argv: + from modules.api.ray import ray_only ray_only() else: webui.webui() From a664f6eacd8001cf5bb503437000270486487b40 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Wed, 4 Oct 2023 15:38:11 -0400 Subject: [PATCH 13/61] defined my own raypi class --- modules/api/ray.py | 36 +- modules/api/raypi.py | 795 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 810 insertions(+), 21 deletions(-) create mode 100644 modules/api/raypi.py diff --git a/modules/api/ray.py b/modules/api/ray.py index 6d74933c28e..3ac07847ecf 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -1,12 +1,13 @@ from ray import serve import ray from fastapi import FastAPI -from modules.api.api import Api +from modules.api.raypi import Api from modules import initialize_util from modules import script_callbacks from modules import initialize import time +from ray.serve.handle import DeploymentHandle from modules.shared_cmd_options import cmd_opts @@ -24,7 +25,7 @@ ) #initialize.initialize() -app = FastAPI() +#app = FastAPI() #initialize_util.setup_middleware(app) #api = Api(app) #app.include_router(api.router) @@ -32,36 +33,29 @@ #script_callbacks.app_started_callback(None, app) - -@serve.deployment( - ray_actor_options={"num_gpus": 1}, - num_replicas=NUM_REPLICAS, -) -@serve.ingress(app) -class APIIngress: - def __init__(self) -> None: - pass - - - def ray_only(): from fastapi import FastAPI from modules.shared_cmd_options import cmd_opts + from modules import script_callbacks + # Shutdown any existing Serve replicas, if they're still around. + serve.shutdown() + serve.start() initialize.initialize() app = FastAPI() initialize_util.setup_middleware(app) - api = Api(app) - app.include_router(api.router) - from modules import script_callbacks script_callbacks.before_ui_callback() script_callbacks.app_started_callback(None, app) + #Api.deploy() + #api = Api(app) # Create an instance of the Api class + serve.run(Api.bind() , port=8000) #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app + - # Shutdown any existing Serve replicas, if they're still around. - serve.shutdown() - serve.run(APIIngress.bind(), port=8000, name="serving_stable_diffusion_template") print("Done setting up replicas! Now accepting requests...") while True: - time.sleep(1000) \ No newline at end of file + time.sleep(1000) + + + diff --git a/modules/api/raypi.py b/modules/api/raypi.py new file mode 100644 index 00000000000..2a7e51dbbcb --- /dev/null +++ b/modules/api/raypi.py @@ -0,0 +1,795 @@ +import base64 +import io +import os +import time +import datetime +import uvicorn +import ipaddress +import requests +import gradio as gr +from threading import Lock +from io import BytesIO +from fastapi import APIRouter, Depends, FastAPI, Request, Response +from fastapi.security import HTTPBasic, HTTPBasicCredentials +from fastapi.exceptions import HTTPException +from fastapi.responses import JSONResponse +from fastapi.encoders import jsonable_encoder +from secrets import compare_digest + +import modules.shared as shared +from modules import sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing, errors, restart, shared_items +from modules.api import models +from modules.shared import opts +from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images +from modules.textual_inversion.textual_inversion import create_embedding, train_embedding +from modules.textual_inversion.preprocess import preprocess +from modules.hypernetworks.hypernetwork import create_hypernetwork, train_hypernetwork +from PIL import PngImagePlugin,Image +from modules.sd_models import unload_model_weights, reload_model_weights, checkpoint_aliases +from modules.sd_models_config import find_checkpoint_config_near_filename +from modules.realesrgan_model import get_realesrgan_models +from modules import devices +from typing import Dict, List, Any +import piexif +import piexif.helper +from contextlib import closing + +from ray import serve + +app = FastAPI() + +def script_name_to_index(name, scripts): + try: + return [script.title().lower() for script in scripts].index(name.lower()) + except Exception as e: + raise HTTPException(status_code=422, detail=f"Script '{name}' not found") from e + + +def validate_sampler_name(name): + config = sd_samplers.all_samplers_map.get(name, None) + if config is None: + raise HTTPException(status_code=404, detail="Sampler not found") + + return name + + +def setUpscalers(req: dict): + reqDict = vars(req) + reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None) + reqDict['extras_upscaler_2'] = reqDict.pop('upscaler_2', None) + return reqDict + + +def verify_https://github.com/url(url): + """Returns True if the url refers to a global resource.""" + + import socket + from urllib.parse import urlparse + try: + parsed_url = urlparse(url) + domain_name = parsed_url.netloc + host = socket.gethostbyname_ex(domain_name) + for ip in host[2]: + ip_addr = ipaddress.ip_address(ip) + if not ip_addr.is_global: + return False + except Exception: + return False + + return True + + +def decode_base64_to_image(encoding): + if encoding.startswith("http://") or encoding.startswith("https://"): + if not opts.api_enable_requests: + raise HTTPException(status_code=500, detail="Requests not allowed") + + if opts.api_forbid_local_requests and not verify_url(https://github.com/encoding): + raise HTTPException(status_code=500, detail="Request to local resource not allowed") + + headers = {'user-agent': opts.api_useragent} if opts.api_useragent else {} + response = requests.get(encoding, timeout=30, headers=headers) + try: + image = Image.open(BytesIO(response.content)) + return image + except Exception as e: + raise HTTPException(status_code=500, detail="Invalid image url") from e + + if encoding.startswith("data:image/"): + encoding = encoding.split(";")[1].split(",")[1] + try: + image = Image.open(BytesIO(base64.b64decode(encoding))) + return image + except Exception as e: + raise HTTPException(status_code=500, detail="Invalid encoded image") from e + + +def encode_pil_to_base64(image): + with io.BytesIO() as output_bytes: + + if opts.samples_format.lower() == 'png': + use_metadata = False + metadata = PngImagePlugin.PngInfo() + for key, value in image.info.items(): + if isinstance(key, str) and isinstance(value, str): + metadata.add_text(key, value) + use_metadata = True + image.save(output_bytes, format="PNG", pnginfo=(metadata if use_metadata else None), quality=opts.jpeg_quality) + + elif opts.samples_format.lower() in ("jpg", "jpeg", "webp"): + if image.mode == "RGBA": + image = image.convert("RGB") + parameters = image.info.get('parameters', None) + exif_bytes = piexif.dump({ + "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(parameters or "", encoding="unicode") } + }) + if opts.samples_format.lower() in ("jpg", "jpeg"): + image.save(output_bytes, format="JPEG", exif = exif_bytes, quality=opts.jpeg_quality) + else: + image.save(output_bytes, format="WEBP", exif = exif_bytes, quality=opts.jpeg_quality) + + else: + raise HTTPException(status_code=500, detail="Invalid image format") + + bytes_data = output_bytes.getvalue() + + return base64.b64encode(bytes_data) + + +def api_middleware(app: FastAPI): + rich_available = False + try: + if os.environ.get('WEBUI_RICH_EXCEPTIONS', None) is not None: + import anyio # importing just so it can be placed on silent list + import starlette # importing just so it can be placed on silent list + from rich.console import Console + console = Console() + rich_available = True + except Exception: + pass + + @app.middleware("http") + async def log_and_time(req: Request, call_next): + ts = time.time() + res: Response = await call_next(req) + duration = str(round(time.time() - ts, 4)) + res.headers["X-Process-Time"] = duration + endpoint = req.scope.get('path', 'err') + if shared.cmd_opts.api_log and endpoint.startswith('/sdapi'): + print('API {t} {code} {prot}/{ver} {method} {endpoint} {cli} {duration}'.format( + t=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"), + code=res.status_code, + ver=req.scope.get('http_version', '0.0'), + cli=req.scope.get('client', ('0:0.0.0', 0))[0], + prot=req.scope.get('scheme', 'err'), + method=req.scope.get('method', 'err'), + endpoint=endpoint, + duration=duration, + )) + return res + + def handle_exception(request: Request, e: Exception): + err = { + "error": type(e).__name__, + "detail": vars(e).get('detail', ''), + "body": vars(e).get('body', ''), + "errors": str(e), + } + if not isinstance(e, HTTPException): # do not print backtrace on known httpexceptions + message = f"API error: {request.method}: {request.url} {err}" + if rich_available: + print(message) + console.print_exception(show_locals=True, max_frames=2, extra_lines=1, suppress=[anyio, starlette], word_wrap=False, width=min([console.width, 200])) + else: + errors.report(message, exc_info=True) + return JSONResponse(status_code=vars(e).get('status_code', 500), content=jsonable_encoder(err)) + + @app.middleware("http") + async def exception_handling(request: Request, call_next): + try: + return await call_next(request) + except Exception as e: + return handle_exception(request, e) + + @app.exception_handler(Exception) + async def fastapi_exception_handler(request: Request, e: Exception): + return handle_exception(request, e) + + @app.exception_handler(HTTPException) + async def http_exception_handler(request: Request, e: HTTPException): + return handle_exception(request, e) + + from ray import serve + import ray + + +@serve.deployment( + ray_actor_options={"num_gpus": 1}, + autoscaling_config={"min_replicas": 0, "max_replicas": 2}, + route_prefix="/sdapi/v1", + ) +@serve.ingress(app) +class Api: + def __init__(self, app: FastAPI): + if shared.cmd_opts.api_auth: + self.credentials = {} + for auth in shared.cmd_opts.api_auth.split(","): + user, password = auth.split(":") + self.credentials[user] = password + + self.app = app + #self.queue_lock = queue_lock + api_middleware(self.app) + print("API initialized") + + self.default_script_arg_txt2img = [] + self.default_script_arg_img2img = [] + + + + def auth(self, credentials: HTTPBasicCredentials = Depends(HTTPBasic())): + if credentials.username in self.credentials: + if compare_digest(credentials.password, self.credentials[credentials.username]): + return True + + raise HTTPException(status_code=401, detail="Incorrect username or password", headers={"WWW-Authenticate": "Basic"}) + + def get_selectable_script(self, script_name, script_runner): + if script_name is None or script_name == "": + return None, None + + script_idx = script_name_to_index(script_name, script_runner.selectable_scripts) + script = script_runner.selectable_scripts[script_idx] + return script, script_idx + + @app.get("/scripts", response_model=models.ScriptsList) + def get_scripts_list(self): + t2ilist = [script.name for script in scripts.scripts_txt2img.scripts if script.name is not None] + i2ilist = [script.name for script in scripts.scripts_img2img.scripts if script.name is not None] + + return models.ScriptsList(txt2img=t2ilist, img2img=i2ilist) + + @app.get("/script-info", response_model=List[models.ScriptInfo]) + def get_script_info(self): + res = [] + + for script_list in [scripts.scripts_txt2img.scripts, scripts.scripts_img2img.scripts]: + res += [script.api_info for script in script_list if script.api_info is not None] + + return res + + def get_script(self, script_name, script_runner): + if script_name is None or script_name == "": + return None, None + + script_idx = script_name_to_index(script_name, script_runner.scripts) + return script_runner.scripts[script_idx] + + def init_default_script_args(self, script_runner): + #find max idx from the scripts in runner and generate a none array to init script_args + last_arg_index = 1 + for script in script_runner.scripts: + if last_arg_index < script.args_to: + last_arg_index = script.args_to + # None everywhere except position 0 to initialize script args + script_args = [None]*last_arg_index + script_args[0] = 0 + + # get default values + with gr.Blocks(): # will throw errors calling ui function without this + for script in script_runner.scripts: + if script.ui(script.is_img2img): + ui_default_values = [] + for elem in script.ui(script.is_img2img): + ui_default_values.append(elem.value) + script_args[script.args_from:script.args_to] = ui_default_values + return script_args + + def init_script_args(self, request, default_script_args, selectable_scripts, selectable_idx, script_runner): + script_args = default_script_args.copy() + # position 0 in script_arg is the idx+1 of the selectable script that is going to be run when using scripts.scripts_*2img.run() + if selectable_scripts: + script_args[selectable_scripts.args_from:selectable_scripts.args_to] = request.script_args + script_args[0] = selectable_idx + 1 + + # Now check for always on scripts + if request.alwayson_scripts: + for alwayson_script_name in request.alwayson_scripts.keys(): + alwayson_script = self.get_script(alwayson_script_name, script_runner) + if alwayson_script is None: + raise HTTPException(status_code=422, detail=f"always on script {alwayson_script_name} not found") + # Selectable script in always on script param check + if alwayson_script.alwayson is False: + raise HTTPException(status_code=422, detail="Cannot have a selectable script in the always on scripts params") + # always on script with no arg should always run so you don't really need to add them to the requests + if "args" in request.alwayson_scripts[alwayson_script_name]: + # min between arg length in scriptrunner and arg length in the request + for idx in range(0, min((alwayson_script.args_to - alwayson_script.args_from), len(request.alwayson_scripts[alwayson_script_name]["args"]))): + script_args[alwayson_script.args_from + idx] = request.alwayson_scripts[alwayson_script_name]["args"][idx] + return script_args + + @app.post("/txt2img", response_model=models.TextToImageResponse) + def text2imgapi(self, txt2imgreq: models.StableDiffusionTxt2ImgProcessingAPI): + script_runner = scripts.scripts_txt2img + if not script_runner.scripts: + script_runner.initialize_scripts(False) + ui.create_ui() + if not self.default_script_arg_txt2img: + self.default_script_arg_txt2img = self.init_default_script_args(script_runner) + selectable_scripts, selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner) + + populate = txt2imgreq.copy(update={ # Override __init__ params + "sampler_name": validate_sampler_name(txt2imgreq.sampler_name or txt2imgreq.sampler_index), + "do_not_save_samples": not txt2imgreq.save_images, + "do_not_save_grid": not txt2imgreq.save_images, + }) + if populate.sampler_name: + populate.sampler_index = None # prevent a warning later on + + args = vars(populate) + args.pop('script_name', None) + args.pop('script_args', None) # will refeed them to the pipeline directly after initializing them + args.pop('alwayson_scripts', None) + + script_args = self.init_script_args(txt2imgreq, self.default_script_arg_txt2img, selectable_scripts, selectable_script_idx, script_runner) + + send_images = args.pop('send_images', True) + args.pop('save_images', None) + + with self.queue_lock: + with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: + p.is_api = True + p.scripts = script_runner + p.outpath_grids = opts.outdir_txt2img_grids + p.outpath_samples = opts.outdir_txt2img_samples + + try: + shared.state.begin(job="scripts_txt2img") + if selectable_scripts is not None: + p.script_args = script_args + processed = scripts.scripts_txt2img.run(p, *p.script_args) # Need to pass args as list here + else: + p.script_args = tuple(script_args) # Need to pass args as tuple here + processed = process_images(p) + finally: + shared.state.end() + shared.total_tqdm.clear() + + b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else [] + + return models.TextToImageResponse(images=b64images, parameters=vars(txt2imgreq), info=processed.js()) + + @app.post("/img2img", response_model=models.ImageToImageResponse) + def img2imgapi(self, img2imgreq: models.StableDiffusionImg2ImgProcessingAPI): + init_images = img2imgreq.init_images + if init_images is None: + raise HTTPException(status_code=404, detail="Init image not found") + + mask = img2imgreq.mask + if mask: + mask = decode_base64_to_image(mask) + + script_runner = scripts.scripts_img2img + if not script_runner.scripts: + script_runner.initialize_scripts(True) + ui.create_ui() + if not self.default_script_arg_img2img: + self.default_script_arg_img2img = self.init_default_script_args(script_runner) + selectable_scripts, selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner) + + populate = img2imgreq.copy(update={ # Override __init__ params + "sampler_name": validate_sampler_name(img2imgreq.sampler_name or img2imgreq.sampler_index), + "do_not_save_samples": not img2imgreq.save_images, + "do_not_save_grid": not img2imgreq.save_images, + "mask": mask, + }) + if populate.sampler_name: + populate.sampler_index = None # prevent a warning later on + + args = vars(populate) + args.pop('include_init_images', None) # this is meant to be done by "exclude": True in model, but it's for a reason that I cannot determine. + args.pop('script_name', None) + args.pop('script_args', None) # will refeed them to the pipeline directly after initializing them + args.pop('alwayson_scripts', None) + + script_args = self.init_script_args(img2imgreq, self.default_script_arg_img2img, selectable_scripts, selectable_script_idx, script_runner) + + send_images = args.pop('send_images', True) + args.pop('save_images', None) + + with self.queue_lock: + with closing(StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)) as p: + p.init_images = [decode_base64_to_image(x) for x in init_images] + p.is_api = True + p.scripts = script_runner + p.outpath_grids = opts.outdir_img2img_grids + p.outpath_samples = opts.outdir_img2img_samples + + try: + shared.state.begin(job="scripts_img2img") + if selectable_scripts is not None: + p.script_args = script_args + processed = scripts.scripts_img2img.run(p, *p.script_args) # Need to pass args as list here + else: + p.script_args = tuple(script_args) # Need to pass args as tuple here + processed = process_images(p) + finally: + shared.state.end() + shared.total_tqdm.clear() + + b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else [] + + if not img2imgreq.include_init_images: + img2imgreq.init_images = None + img2imgreq.mask = None + + return models.ImageToImageResponse(images=b64images, parameters=vars(img2imgreq), info=processed.js()) + + @app.post("/extra-single-image",response_model=models.ExtrasSingleImageResponse) + def extras_single_image_api(self, req: models.ExtrasSingleImageRequest): + reqDict = setUpscalers(req) + + reqDict['image'] = decode_base64_to_image(reqDict['image']) + + with self.queue_lock: + result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict) + + return models.ExtrasSingleImageResponse(image=encode_pil_to_base64(result[0][0]), html_info=result[1]) + + @app.post("/extra-batch-images",response_model=models.ExtrasBatchImagesResponse) + def extras_batch_images_api(self, req: models.ExtrasBatchImagesRequest): + reqDict = setUpscalers(req) + + image_list = reqDict.pop('imageList', []) + image_folder = [decode_base64_to_image(x.data) for x in image_list] + + with self.queue_lock: + result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict) + + return models.ExtrasBatchImagesResponse(images=list(map(encode_pil_to_base64, result[0])), html_info=result[1]) + + @app.post("/png-info",response_model=models.PNGInfoResponse) + def pnginfoapi(self, req: models.PNGInfoRequest): + if(not req.image.strip()): + return models.PNGInfoResponse(info="") + + image = decode_base64_to_image(req.image.strip()) + if image is None: + return models.PNGInfoResponse(info="") + + geninfo, items = images.read_info_from_image(image) + if geninfo is None: + geninfo = "" + + items = {**{'parameters': geninfo}, **items} + + return models.PNGInfoResponse(info=geninfo, items=items) + + @app.get("/progress",response_model=models.ProgressResponse) + def progressapi(self, req: models.ProgressRequest = Depends()): + # copy from check_progress_call of ui.py + + if shared.state.job_count == 0: + return models.ProgressResponse(progress=0, eta_relative=0, state=shared.state.dict(), textinfo=shared.state.textinfo) + + # avoid dividing zero + progress = 0.01 + + if shared.state.job_count > 0: + progress += shared.state.job_no / shared.state.job_count + if shared.state.sampling_steps > 0: + progress += 1 / shared.state.job_count * shared.state.sampling_step / shared.state.sampling_steps + + time_since_start = time.time() - shared.state.time_start + eta = (time_since_start/progress) + eta_relative = eta-time_since_start + + progress = min(progress, 1) + + shared.state.set_current_image() + + current_image = None + if shared.state.current_image and not req.skip_current_image: + current_image = encode_pil_to_base64(shared.state.current_image) + + return models.ProgressResponse(progress=progress, eta_relative=eta_relative, state=shared.state.dict(), current_image=current_image, textinfo=shared.state.textinfo) + + @app.post("/interrogate") + def interrogateapi(self, interrogatereq: models.InterrogateRequest): + image_b64 = interrogatereq.image + if image_b64 is None: + raise HTTPException(status_code=404, detail="Image not found") + + img = decode_base64_to_image(image_b64) + img = img.convert('RGB') + + # Override object param + with self.queue_lock: + if interrogatereq.model == "clip": + processed = shared.interrogator.interrogate(img) + elif interrogatereq.model == "deepdanbooru": + processed = deepbooru.model.tag(img) + else: + raise HTTPException(status_code=404, detail="Model not found") + + return models.InterrogateResponse(caption=processed) + + @app.post("/interrupt") + def interruptapi(self): + shared.state.interrupt() + + return {} + + @app.post("/unload-checkpoint") + def unloadapi(self): + unload_model_weights() + + return {} + + @app.post("/reload-checkpoint") + def reloadapi(self): + reload_model_weights() + + return {} + + @app.post("/skip") + def skip(self): + shared.state.skip() + + @app.get("/options",response_model=models.OptionsModel) + def get_config(self): + options = {} + for key in shared.opts.data.keys(): + metadata = shared.opts.data_labels.get(key) + if(metadata is not None): + options.update({key: shared.opts.data.get(key, shared.opts.data_labels.get(key).default)}) + else: + options.update({key: shared.opts.data.get(key, None)}) + + return options + + @app.post("/options") + def set_config(self, req: Dict[str, Any]): + checkpoint_name = req.get("sd_model_checkpoint", None) + if checkpoint_name is not None and checkpoint_name not in checkpoint_aliases: + raise RuntimeError(f"model {checkpoint_name!r} not found") + + for k, v in req.items(): + shared.opts.set(k, v, is_api=True) + + shared.opts.save(shared.config_filename) + return + + @app.get("/cmd-flags", response_model=models.FlagsModel) + def get_cmd_flags(self): + return vars(shared.cmd_opts) + + @app.get("/samplers", response_model=List[models.SamplerItem]) + def get_samplers(self): + return [{"name": sampler[0], "aliases":sampler[2], "options":sampler[3]} for sampler in sd_samplers.all_samplers] + + @app.get("/upscalers",response_model=List[models.UpscalerItem]) + def get_upscalers(self): + return [ + { + "name": upscaler.name, + "model_name": upscaler.scaler.model_name, + "model_path": upscaler.data_path, + "model_url": None, + "scale": upscaler.scale, + } + for upscaler in shared.sd_upscalers + ] + + @app.get("/latent-upscale-modes",response_model=List[models.LatentUpscalerModeItem]) + def get_latent_upscale_modes(self): + return [ + { + "name": upscale_mode, + } + for upscale_mode in [*(shared.latent_upscale_modes or {})] + ] + + @app.get("/sd-models", response_model=List[models.SDModelItem]) + def get_sd_models(self): + import modules.sd_models as sd_models + return [{"title": x.title, "model_name": x.model_name, "hash": x.shorthash, "sha256": x.sha256, "filename": x.filename, "config": find_checkpoint_config_near_filename(x)} for x in sd_models.checkpoints_list.values()] + + @app.get("/sd-vae", response_model=List[models.SDVaeItem]) + def get_sd_vaes(self): + import modules.sd_vae as sd_vae + return [{"model_name": x, "filename": sd_vae.vae_dict[x]} for x in sd_vae.vae_dict.keys()] + + @app.get("/hypernetworks", response_model=List[models.HypernetworkItem]) + def get_hypernetworks(self): + return [{"name": name, "path": shared.hypernetworks[name]} for name in shared.hypernetworks] + + @app.get("/face-restorers", response_model=List[models.FaceRestorerItem]) + def get_face_restorers(self): + return [{"name":x.name(), "cmd_dir": getattr(x, "cmd_dir", None)} for x in shared.face_restorers] + + @app.get("/realesrgan-models",response_model=List[models.RealesrganItem]) + def get_realesrgan_models(self): + return [{"name":x.name,"path":x.data_path, "scale":x.scale} for x in get_realesrgan_models(None)] + + @app.get("/prompt-styles", response_model=List[models.PromptStyleItem]) + def get_prompt_styles(self): + styleList = [] + for k in shared.prompt_styles.styles: + style = shared.prompt_styles.styles[k] + styleList.append({"name":style[0], "prompt": style[1], "negative_prompt": style[2]}) + + return styleList + + @app.get("/embeddings", response_model=models.EmbeddingsResponse) + def get_embeddings(self): + db = sd_hijack.model_hijack.embedding_db + + def convert_embedding(embedding): + return { + "step": embedding.step, + "sd_checkpoint": embedding.sd_checkpoint, + "sd_checkpoint_name": embedding.sd_checkpoint_name, + "shape": embedding.shape, + "vectors": embedding.vectors, + } + + + def convert_embeddings(embeddings): + return {embedding.name: convert_embedding(embedding) for embedding in embeddings.values()} + + return { + "loaded": convert_embeddings(db.word_embeddings), + "skipped": convert_embeddings(db.skipped_embeddings), + } + + @app.post("/refresh-checkpoints") + def refresh_checkpoints(self): + with self.queue_lock: + shared.refresh_checkpoints() + + @app.post("/refresh-vae") + def refresh_vae(self): + with self.queue_lock: + shared_items.refresh_vae_list() + + @app.post("/create/embedding", response_model=models.CreateResponse) + def create_embedding(self, args: dict): + try: + shared.state.begin(job="create_embedding") + filename = create_embedding(**args) # create empty embedding + sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings() # reload embeddings so new one can be immediately used + return models.CreateResponse(info=f"create embedding filename: {filename}") + except AssertionError as e: + return models.TrainResponse(info=f"create embedding error: {e}") + finally: + shared.state.end() + + @app.post("/create/hypernetwork", response_model=models.CreateResponse) + def create_hypernetwork(self, args: dict): + try: + shared.state.begin(job="create_hypernetwork") + filename = create_hypernetwork(**args) # create empty embedding + return models.CreateResponse(info=f"create hypernetwork filename: {filename}") + except AssertionError as e: + return models.TrainResponse(info=f"create hypernetwork error: {e}") + finally: + shared.state.end() + + @app.post("/preprocess", response_model=models.PreprocessResponse) + def preprocess(self, args: dict): + try: + shared.state.begin(job="preprocess") + preprocess(**args) # quick operation unless blip/booru interrogation is enabled + shared.state.end() + return models.PreprocessResponse(info='preprocess complete') + except KeyError as e: + return models.PreprocessResponse(info=f"preprocess error: invalid token: {e}") + except Exception as e: + return models.PreprocessResponse(info=f"preprocess error: {e}") + finally: + shared.state.end() + + @app.post("/train/embedding", response_model=models.TrainResponse) + def train_embedding(self, args: dict): + try: + shared.state.begin(job="train_embedding") + apply_optimizations = shared.opts.training_xattention_optimizations + error = None + filename = '' + if not apply_optimizations: + sd_hijack.undo_optimizations() + try: + embedding, filename = train_embedding(**args) # can take a long time to complete + except Exception as e: + error = e + finally: + if not apply_optimizations: + sd_hijack.apply_optimizations() + return models.TrainResponse(info=f"train embedding complete: filename: {filename} error: {error}") + except Exception as msg: + return models.TrainResponse(info=f"train embedding error: {msg}") + finally: + shared.state.end() + + @app.post("/train/hypernetwork", response_model=models.TrainResponse) + def train_hypernetwork(self, args: dict): + try: + shared.state.begin(job="train_hypernetwork") + shared.loaded_hypernetworks = [] + apply_optimizations = shared.opts.training_xattention_optimizations + error = None + filename = '' + if not apply_optimizations: + sd_hijack.undo_optimizations() + try: + hypernetwork, filename = train_hypernetwork(**args) + except Exception as e: + error = e + finally: + shared.sd_model.cond_stage_model.to(devices.device) + shared.sd_model.first_stage_model.to(devices.device) + if not apply_optimizations: + sd_hijack.apply_optimizations() + shared.state.end() + return models.TrainResponse(info=f"train embedding complete: filename: {filename} error: {error}") + except Exception as exc: + return models.TrainResponse(info=f"train embedding error: {exc}") + finally: + shared.state.end() + + @app.get("/memory", response_model=models.MemoryResponse) + def get_memory(self): + try: + import os + import psutil + process = psutil.Process(os.getpid()) + res = process.memory_info() # only rss is cross-platform guaranteed so we dont rely on other values + ram_total = 100 * res.rss / process.memory_percent() # and total memory is calculated as actual value is not cross-platform safe + ram = { 'free': ram_total - res.rss, 'used': res.rss, 'total': ram_total } + except Exception as err: + ram = { 'error': f'{err}' } + try: + import torch + if torch.cuda.is_available(): + s = torch.cuda.mem_get_info() + system = { 'free': s[0], 'used': s[1] - s[0], 'total': s[1] } + s = dict(torch.cuda.memory_stats(shared.device)) + allocated = { 'current': s['allocated_bytes.all.current'], 'peak': s['allocated_bytes.all.peak'] } + reserved = { 'current': s['reserved_bytes.all.current'], 'peak': s['reserved_bytes.all.peak'] } + active = { 'current': s['active_bytes.all.current'], 'peak': s['active_bytes.all.peak'] } + inactive = { 'current': s['inactive_split_bytes.all.current'], 'peak': s['inactive_split_bytes.all.peak'] } + warnings = { 'retries': s['num_alloc_retries'], 'oom': s['num_ooms'] } + cuda = { + 'system': system, + 'active': active, + 'allocated': allocated, + 'reserved': reserved, + 'inactive': inactive, + 'events': warnings, + } + else: + cuda = {'error': 'unavailable'} + except Exception as err: + cuda = {'error': f'{err}'} + return models.MemoryResponse(ram=ram, cuda=cuda) + + def launch(self, server_name, port, root_path): + self.app.include_router(self.router) + uvicorn.run(self.app, host=server_name, port=port, timeout_keep_alive=shared.cmd_opts.timeout_keep_alive, root_path=root_path) + + @app.post("/server-kill") + def kill_webui(self): + restart.stop_program() + + @app.post("/server-restart") + def restart_webui(self): + if restart.is_restartable(): + restart.restart_program() + return Response(status_code=501) + + @app.post("/server-stop") + def stop_webui(request): + shared.state.server_command = "stop" + return Response("Stopping.") + From 9e682a3e5f8f04a04a3df49f5f22da03933cf34b Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Wed, 4 Oct 2023 15:44:28 -0400 Subject: [PATCH 14/61] Update raypi.py --- modules/api/raypi.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/modules/api/raypi.py b/modules/api/raypi.py index 2a7e51dbbcb..baafbc40d52 100644 --- a/modules/api/raypi.py +++ b/modules/api/raypi.py @@ -206,7 +206,7 @@ async def http_exception_handler(request: Request, e: HTTPException): @serve.deployment( ray_actor_options={"num_gpus": 1}, autoscaling_config={"min_replicas": 0, "max_replicas": 2}, - route_prefix="/sdapi/v1", + #route_prefix="/sdapi/v1", ) @serve.ingress(app) class Api: @@ -220,6 +220,7 @@ def __init__(self, app: FastAPI): self.app = app #self.queue_lock = queue_lock api_middleware(self.app) + self.launch_ray() print("API initialized") self.default_script_arg_txt2img = [] @@ -793,3 +794,43 @@ def stop_webui(request): shared.state.server_command = "stop" return Response("Stopping.") + def launch_ray(self): + self.app.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=models.TextToImageResponse) + self.app.add_api_route("/sdapi/v1/img2img", self.img2imgapi, methods=["POST"], response_model=models.ImageToImageResponse) + self.app.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ExtrasSingleImageResponse) + self.app.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ExtrasBatchImagesResponse) + self.app.add_api_route("/sdapi/v1/png-info", self.pnginfoapi, methods=["POST"], response_model=models.PNGInfoResponse) + self.app.add_api_route("/sdapi/v1/progress", self.progressapi, methods=["GET"], response_model=models.ProgressResponse) + self.app.add_api_route("/sdapi/v1/interrogate", self.interrogateapi, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/interrupt", self.interruptapi, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/skip", self.skip, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/options", self.get_config, methods=["GET"], response_model=models.OptionsModel) + self.app.add_api_route("/sdapi/v1/options", self.set_config, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/cmd-flags", self.get_cmd_flags, methods=["GET"], response_model=models.FlagsModel) + self.app.add_api_route("/sdapi/v1/samplers", self.get_samplers, methods=["GET"], response_model=List[models.SamplerItem]) + self.app.add_api_route("/sdapi/v1/upscalers", self.get_upscalers, methods=["GET"], response_model=List[models.UpscalerItem]) + self.app.add_api_route("/sdapi/v1/latent-upscale-modes", self.get_latent_upscale_modes, methods=["GET"], response_model=List[models.LatentUpscalerModeItem]) + self.app.add_api_route("/sdapi/v1/sd-models", self.get_sd_models, methods=["GET"], response_model=List[models.SDModelItem]) + self.app.add_api_route("/sdapi/v1/sd-vae", self.get_sd_vaes, methods=["GET"], response_model=List[models.SDVaeItem]) + self.app.add_api_route("/sdapi/v1/hypernetworks", self.get_hypernetworks, methods=["GET"], response_model=List[models.HypernetworkItem]) + self.app.add_api_route("/sdapi/v1/face-restorers", self.get_face_restorers, methods=["GET"], response_model=List[models.FaceRestorerItem]) + self.app.add_api_route("/sdapi/v1/realesrgan-models", self.get_realesrgan_models, methods=["GET"], response_model=List[models.RealesrganItem]) + self.app.add_api_route("/sdapi/v1/prompt-styles", self.get_prompt_styles, methods=["GET"], response_model=List[models.PromptStyleItem]) + self.app.add_api_route("/sdapi/v1/embeddings", self.get_embeddings, methods=["GET"], response_model=models.EmbeddingsResponse) + self.app.add_api_route("/sdapi/v1/refresh-checkpoints", self.refresh_checkpoints, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/refresh-vae", self.refresh_vae, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/create/embedding", self.create_embedding, methods=["POST"], response_model=models.CreateResponse) + self.app.add_api_route("/sdapi/v1/create/hypernetwork", self.create_hypernetwork, methods=["POST"], response_model=models.CreateResponse) + self.app.add_api_route("/sdapi/v1/preprocess", self.preprocess, methods=["POST"], response_model=models.PreprocessResponse) + self.app.add_api_route("/sdapi/v1/train/embedding", self.train_embedding, methods=["POST"], response_model=models.TrainResponse) + self.app.add_api_route("/sdapi/v1/train/hypernetwork", self.train_hypernetwork, methods=["POST"], response_model=models.TrainResponse) + self.app.add_api_route("/sdapi/v1/memory", self.get_memory, methods=["GET"], response_model=models.MemoryResponse) + self.app.add_api_route("/sdapi/v1/unload-checkpoint", self.unloadapi, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/reload-checkpoint", self.reloadapi, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/scripts", self.get_scripts_list, methods=["GET"], response_model=models.ScriptsList) + self.app.add_api_route("/sdapi/v1/script-info", self.get_script_info, methods=["GET"], response_model=List[models.ScriptInfo]) + + if shared.cmd_opts.api_server_stop: + self.app.add_api_route("/sdapi/v1/server-kill", self.kill_webui, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/server-restart", self.restart_webui, methods=["POST"]) + self.app.add_api_route("/sdapi/v1/server-stop", self.stop_webui, methods=["POST"]) From 06a003a764a8d62d4fc3a54b827e77f58104b0ca Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 5 Oct 2023 12:49:38 -0400 Subject: [PATCH 15/61] changes --- .gitignore | 3 +- modules/api/api.py | 4 +- modules/api/ray.py | 41 ++++------ modules/api/raypi.py | 175 ++++++++++++++++--------------------------- 4 files changed, 83 insertions(+), 140 deletions(-) diff --git a/.gitignore b/.gitignore index 841e25d966d..53af02d29df 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,5 @@ notification.mp3 /package-lock.json /.coverage* -sd-data \ No newline at end of file +sd-data +modules/api/raypi2.py diff --git a/modules/api/api.py b/modules/api/api.py index fea1dc4d73d..a08094c4a7d 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -198,7 +198,7 @@ async def http_exception_handler(request: Request, e: HTTPException): class Api: - def __init__(self, app: FastAPI): + def __init__(self, app: FastAPI, queue_lock: Lock): if shared.cmd_opts.api_auth: self.credentials = {} for auth in shared.cmd_opts.api_auth.split(","): @@ -207,7 +207,7 @@ def __init__(self, app: FastAPI): self.router = APIRouter() self.app = app - #self.queue_lock = queue_lock + self.queue_lock = queue_lock api_middleware(self.app) print("API initialized") self.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=models.TextToImageResponse) diff --git a/modules/api/ray.py b/modules/api/ray.py index 3ac07847ecf..02fc685f1dd 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -1,14 +1,14 @@ from ray import serve import ray from fastapi import FastAPI -from modules.api.raypi import Api +from modules.api.raypi import Raypi from modules import initialize_util from modules import script_callbacks from modules import initialize import time from ray.serve.handle import DeploymentHandle - +from modules.call_queue import queue_lock from modules.shared_cmd_options import cmd_opts ray.init() @@ -24,38 +24,25 @@ "per worker. Ignore this if your cluster auto-scales." ) -#initialize.initialize() -#app = FastAPI() -#initialize_util.setup_middleware(app) -#api = Api(app) -#app.include_router(api.router) -#script_callbacks.before_ui_callback() -#script_callbacks.app_started_callback(None, app) +initialize.initialize() +app = FastAPI() +#app.include_router(Raypi(app).router) +initialize_util.setup_middleware(app) +script_callbacks.before_ui_callback() +script_callbacks.app_started_callback(None, app) + + -def ray_only(): - from fastapi import FastAPI - from modules.shared_cmd_options import cmd_opts - from modules import script_callbacks - # Shutdown any existing Serve replicas, if they're still around. - serve.shutdown() - serve.start() - initialize.initialize() - app = FastAPI() - initialize_util.setup_middleware(app) - script_callbacks.before_ui_callback() - script_callbacks.app_started_callback(None, app) - #Api.deploy() - #api = Api(app) # Create an instance of the Api class - serve.run(Api.bind() , port=8000) #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app +def ray_only(): + serve.shutdown() + serve.start() + serve.run(Raypi.bind(), port=8000) #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app print("Done setting up replicas! Now accepting requests...") while True: time.sleep(1000) - - - diff --git a/modules/api/raypi.py b/modules/api/raypi.py index baafbc40d52..deedd096fc2 100644 --- a/modules/api/raypi.py +++ b/modules/api/raypi.py @@ -34,6 +34,10 @@ import piexif.helper from contextlib import closing +from modules import initialize_util +from modules import script_callbacks +from modules import initialize + from ray import serve app = FastAPI() @@ -199,35 +203,26 @@ async def fastapi_exception_handler(request: Request, e: Exception): async def http_exception_handler(request: Request, e: HTTPException): return handle_exception(request, e) - from ray import serve - import ray - - -@serve.deployment( - ray_actor_options={"num_gpus": 1}, - autoscaling_config={"min_replicas": 0, "max_replicas": 2}, - #route_prefix="/sdapi/v1", - ) +@serve.deployment( + ray_actor_options={"num_gpus": 1}, + autoscaling_config={"min_replicas": 0, "max_replicas": 2}, + ) @serve.ingress(app) -class Api: - def __init__(self, app: FastAPI): +class Raypi: + def __init__(self): if shared.cmd_opts.api_auth: self.credentials = {} for auth in shared.cmd_opts.api_auth.split(","): user, password = auth.split(":") self.credentials[user] = password - - self.app = app - #self.queue_lock = queue_lock - api_middleware(self.app) - self.launch_ray() + + print("API initialized") self.default_script_arg_txt2img = [] self.default_script_arg_img2img = [] - def auth(self, credentials: HTTPBasicCredentials = Depends(HTTPBasic())): if credentials.username in self.credentials: if compare_digest(credentials.password, self.credentials[credentials.username]): @@ -337,24 +332,23 @@ def text2imgapi(self, txt2imgreq: models.StableDiffusionTxt2ImgProcessingAPI): send_images = args.pop('send_images', True) args.pop('save_images', None) - with self.queue_lock: - with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: - p.is_api = True - p.scripts = script_runner - p.outpath_grids = opts.outdir_txt2img_grids - p.outpath_samples = opts.outdir_txt2img_samples - - try: - shared.state.begin(job="scripts_txt2img") - if selectable_scripts is not None: - p.script_args = script_args - processed = scripts.scripts_txt2img.run(p, *p.script_args) # Need to pass args as list here - else: - p.script_args = tuple(script_args) # Need to pass args as tuple here - processed = process_images(p) - finally: - shared.state.end() - shared.total_tqdm.clear() + + with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: + p.is_api = True + p.scripts = script_runner + p.outpath_grids = opts.outdir_txt2img_grids + p.outpath_samples = opts.outdir_txt2img_samples + try: + shared.state.begin(job="scripts_txt2img") + if selectable_scripts is not None: + p.script_args = script_args + processed = scripts.scripts_txt2img.run(p, *p.script_args) # Need to pass args as list here + else: + p.script_args = tuple(script_args) # Need to pass args as tuple here + processed = process_images(p) + finally: + shared.state.end() + shared.total_tqdm.clear() b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else [] @@ -398,25 +392,24 @@ def img2imgapi(self, img2imgreq: models.StableDiffusionImg2ImgProcessingAPI): send_images = args.pop('send_images', True) args.pop('save_images', None) - with self.queue_lock: - with closing(StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)) as p: - p.init_images = [decode_base64_to_image(x) for x in init_images] - p.is_api = True - p.scripts = script_runner - p.outpath_grids = opts.outdir_img2img_grids - p.outpath_samples = opts.outdir_img2img_samples - - try: - shared.state.begin(job="scripts_img2img") - if selectable_scripts is not None: - p.script_args = script_args - processed = scripts.scripts_img2img.run(p, *p.script_args) # Need to pass args as list here - else: - p.script_args = tuple(script_args) # Need to pass args as tuple here - processed = process_images(p) - finally: - shared.state.end() - shared.total_tqdm.clear() + + with closing(StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)) as p: + p.init_images = [decode_base64_to_image(x) for x in init_images] + p.is_api = True + p.scripts = script_runner + p.outpath_grids = opts.outdir_img2img_grids + p.outpath_samples = opts.outdir_img2img_samples + try: + shared.state.begin(job="scripts_img2img") + if selectable_scripts is not None: + p.script_args = script_args + processed = scripts.scripts_img2img.run(p, *p.script_args) # Need to pass args as list here + else: + p.script_args = tuple(script_args) # Need to pass args as tuple here + processed = process_images(p) + finally: + shared.state.end() + shared.total_tqdm.clear() b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else [] @@ -432,8 +425,8 @@ def extras_single_image_api(self, req: models.ExtrasSingleImageRequest): reqDict['image'] = decode_base64_to_image(reqDict['image']) - with self.queue_lock: - result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict) + + result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict) return models.ExtrasSingleImageResponse(image=encode_pil_to_base64(result[0][0]), html_info=result[1]) @@ -444,8 +437,8 @@ def extras_batch_images_api(self, req: models.ExtrasBatchImagesRequest): image_list = reqDict.pop('imageList', []) image_folder = [decode_base64_to_image(x.data) for x in image_list] - with self.queue_lock: - result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict) + + result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict) return models.ExtrasBatchImagesResponse(images=list(map(encode_pil_to_base64, result[0])), html_info=result[1]) @@ -505,14 +498,13 @@ def interrogateapi(self, interrogatereq: models.InterrogateRequest): img = img.convert('RGB') # Override object param - with self.queue_lock: - if interrogatereq.model == "clip": - processed = shared.interrogator.interrogate(img) - elif interrogatereq.model == "deepdanbooru": - processed = deepbooru.model.tag(img) - else: - raise HTTPException(status_code=404, detail="Model not found") - + if interrogatereq.model == "clip": + processed = shared.interrogator.interrogate(img) + elif interrogatereq.model == "deepdanbooru": + processed = deepbooru.model.tag(img) + else: + raise HTTPException(status_code=404, detail="Model not found") + return models.InterrogateResponse(caption=processed) @app.post("/interrupt") @@ -646,13 +638,13 @@ def convert_embeddings(embeddings): @app.post("/refresh-checkpoints") def refresh_checkpoints(self): - with self.queue_lock: - shared.refresh_checkpoints() + + shared.refresh_checkpoints() @app.post("/refresh-vae") def refresh_vae(self): - with self.queue_lock: - shared_items.refresh_vae_list() + + shared_items.refresh_vae_list() @app.post("/create/embedding", response_model=models.CreateResponse) def create_embedding(self, args: dict): @@ -794,43 +786,6 @@ def stop_webui(request): shared.state.server_command = "stop" return Response("Stopping.") - def launch_ray(self): - self.app.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=models.TextToImageResponse) - self.app.add_api_route("/sdapi/v1/img2img", self.img2imgapi, methods=["POST"], response_model=models.ImageToImageResponse) - self.app.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ExtrasSingleImageResponse) - self.app.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ExtrasBatchImagesResponse) - self.app.add_api_route("/sdapi/v1/png-info", self.pnginfoapi, methods=["POST"], response_model=models.PNGInfoResponse) - self.app.add_api_route("/sdapi/v1/progress", self.progressapi, methods=["GET"], response_model=models.ProgressResponse) - self.app.add_api_route("/sdapi/v1/interrogate", self.interrogateapi, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/interrupt", self.interruptapi, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/skip", self.skip, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/options", self.get_config, methods=["GET"], response_model=models.OptionsModel) - self.app.add_api_route("/sdapi/v1/options", self.set_config, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/cmd-flags", self.get_cmd_flags, methods=["GET"], response_model=models.FlagsModel) - self.app.add_api_route("/sdapi/v1/samplers", self.get_samplers, methods=["GET"], response_model=List[models.SamplerItem]) - self.app.add_api_route("/sdapi/v1/upscalers", self.get_upscalers, methods=["GET"], response_model=List[models.UpscalerItem]) - self.app.add_api_route("/sdapi/v1/latent-upscale-modes", self.get_latent_upscale_modes, methods=["GET"], response_model=List[models.LatentUpscalerModeItem]) - self.app.add_api_route("/sdapi/v1/sd-models", self.get_sd_models, methods=["GET"], response_model=List[models.SDModelItem]) - self.app.add_api_route("/sdapi/v1/sd-vae", self.get_sd_vaes, methods=["GET"], response_model=List[models.SDVaeItem]) - self.app.add_api_route("/sdapi/v1/hypernetworks", self.get_hypernetworks, methods=["GET"], response_model=List[models.HypernetworkItem]) - self.app.add_api_route("/sdapi/v1/face-restorers", self.get_face_restorers, methods=["GET"], response_model=List[models.FaceRestorerItem]) - self.app.add_api_route("/sdapi/v1/realesrgan-models", self.get_realesrgan_models, methods=["GET"], response_model=List[models.RealesrganItem]) - self.app.add_api_route("/sdapi/v1/prompt-styles", self.get_prompt_styles, methods=["GET"], response_model=List[models.PromptStyleItem]) - self.app.add_api_route("/sdapi/v1/embeddings", self.get_embeddings, methods=["GET"], response_model=models.EmbeddingsResponse) - self.app.add_api_route("/sdapi/v1/refresh-checkpoints", self.refresh_checkpoints, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/refresh-vae", self.refresh_vae, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/create/embedding", self.create_embedding, methods=["POST"], response_model=models.CreateResponse) - self.app.add_api_route("/sdapi/v1/create/hypernetwork", self.create_hypernetwork, methods=["POST"], response_model=models.CreateResponse) - self.app.add_api_route("/sdapi/v1/preprocess", self.preprocess, methods=["POST"], response_model=models.PreprocessResponse) - self.app.add_api_route("/sdapi/v1/train/embedding", self.train_embedding, methods=["POST"], response_model=models.TrainResponse) - self.app.add_api_route("/sdapi/v1/train/hypernetwork", self.train_hypernetwork, methods=["POST"], response_model=models.TrainResponse) - self.app.add_api_route("/sdapi/v1/memory", self.get_memory, methods=["GET"], response_model=models.MemoryResponse) - self.app.add_api_route("/sdapi/v1/unload-checkpoint", self.unloadapi, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/reload-checkpoint", self.reloadapi, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/scripts", self.get_scripts_list, methods=["GET"], response_model=models.ScriptsList) - self.app.add_api_route("/sdapi/v1/script-info", self.get_script_info, methods=["GET"], response_model=List[models.ScriptInfo]) - - if shared.cmd_opts.api_server_stop: - self.app.add_api_route("/sdapi/v1/server-kill", self.kill_webui, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/server-restart", self.restart_webui, methods=["POST"]) - self.app.add_api_route("/sdapi/v1/server-stop", self.stop_webui, methods=["POST"]) + def launch(self, server_name, port, root_path): + self.app.include_router(self.router) + uvicorn.run(self.app, host=server_name, port=port, timeout_keep_alive=shared.cmd_opts.timeout_keep_alive, root_path=root_path) From e09a6d044f8a62c23a3a07c6fea61e809c28b160 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 5 Oct 2023 15:14:53 -0400 Subject: [PATCH 16/61] raypi working --- .gitignore | 1 + modules/api/api.py | 23 +++++++++++++++++++---- modules/api/ray.py | 28 ++++++++++++++++------------ modules/api/raypi.py | 33 ++++++++++++++++++++++----------- modules/initialize.py | 2 +- webui.py | 23 +++++++++++++++++++++-- 6 files changed, 80 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 53af02d29df..2ed62f7e797 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ notification.mp3 sd-data modules/api/raypi2.py +modules/api/raypi3.py diff --git a/modules/api/api.py b/modules/api/api.py index a08094c4a7d..117da300317 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -16,6 +16,9 @@ from fastapi.encoders import jsonable_encoder from secrets import compare_digest +from modules import initialize +initialize.imports() + import modules.shared as shared from modules import sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing, errors, restart, shared_items from modules.api import models @@ -33,7 +36,9 @@ import piexif import piexif.helper from contextlib import closing +from ray import serve +app = FastAPI() def script_name_to_index(name, scripts): try: @@ -197,8 +202,18 @@ async def http_exception_handler(request: Request, e: HTTPException): return handle_exception(request, e) +api_middleware(app) + + +@serve.deployment( + ray_actor_options={"num_gpus": 1}, + autoscaling_config={"min_replicas": 0, "max_replicas": 2}, + #route_prefix="/sdapi/v1", + ) +@serve.ingress(app) class Api: - def __init__(self, app: FastAPI, queue_lock: Lock): + def __init__(self): + initialize.initialize() if shared.cmd_opts.api_auth: self.credentials = {} for auth in shared.cmd_opts.api_auth.split(","): @@ -206,9 +221,9 @@ def __init__(self, app: FastAPI, queue_lock: Lock): self.credentials[user] = password self.router = APIRouter() - self.app = app - self.queue_lock = queue_lock - api_middleware(self.app) + + + print("API initialized") self.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=models.TextToImageResponse) self.add_api_route("/sdapi/v1/img2img", self.img2imgapi, methods=["POST"], response_model=models.ImageToImageResponse) diff --git a/modules/api/ray.py b/modules/api/ray.py index 02fc685f1dd..79d2d2d06cb 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -2,14 +2,10 @@ import ray from fastapi import FastAPI from modules.api.raypi import Raypi -from modules import initialize_util -from modules import script_callbacks -from modules import initialize +from modules.api.api import Raypi import time -from ray.serve.handle import DeploymentHandle -from modules.call_queue import queue_lock -from modules.shared_cmd_options import cmd_opts + ray.init() #ray.init("ray://localhost:10001") @@ -24,15 +20,20 @@ "per worker. Ignore this if your cluster auto-scales." ) -initialize.initialize() app = FastAPI() -#app.include_router(Raypi(app).router) -initialize_util.setup_middleware(app) -script_callbacks.before_ui_callback() -script_callbacks.app_started_callback(None, app) +@serve.deployment( + ray_actor_options={"num_gpus": 1}, + autoscaling_config={"min_replicas": 0, "max_replicas": 2}, + #route_prefix="/sdapi/v1", + ) +@serve.ingress(app) +class RayDeployment: + def __init__(self): + pass +# 2: Deploy the deployment. @@ -40,7 +41,10 @@ def ray_only(): serve.shutdown() serve.start() - serve.run(Raypi.bind(), port=8000) #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app + #Raypi.deploy() + + + serve.run(Raypi.bind(), port=8000, route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app print("Done setting up replicas! Now accepting requests...") diff --git a/modules/api/raypi.py b/modules/api/raypi.py index deedd096fc2..3042a29c4b1 100644 --- a/modules/api/raypi.py +++ b/modules/api/raypi.py @@ -16,6 +16,9 @@ from fastapi.encoders import jsonable_encoder from secrets import compare_digest +from modules import initialize +initialize.imports() + import modules.shared as shared from modules import sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing, errors, restart, shared_items from modules.api import models @@ -36,8 +39,9 @@ from modules import initialize_util from modules import script_callbacks -from modules import initialize + +import launch from ray import serve app = FastAPI() @@ -203,20 +207,31 @@ async def fastapi_exception_handler(request: Request, e: Exception): async def http_exception_handler(request: Request, e: HTTPException): return handle_exception(request, e) -@serve.deployment( - ray_actor_options={"num_gpus": 1}, - autoscaling_config={"min_replicas": 0, "max_replicas": 2}, - ) + +api_middleware(app) + +@serve.deployment( + ray_actor_options={"num_gpus": 1}, + autoscaling_config={"min_replicas": 0, "max_replicas": 2}, + #route_prefix="/sdapi/v1", + ) @serve.ingress(app) class Raypi: def __init__(self): + print("Initializing API") + initialize.initialize() + print("preparing env") + launch.prepare_environment() + #app.include_router(Raypi(app).router) + if shared.cmd_opts.api_auth: self.credentials = {} for auth in shared.cmd_opts.api_auth.split(","): user, password = auth.split(":") self.credentials[user] = password - - + print("preparing env") + launch.prepare_environment() + print("API initialized") self.default_script_arg_txt2img = [] @@ -785,7 +800,3 @@ def restart_webui(self): def stop_webui(request): shared.state.server_command = "stop" return Response("Stopping.") - - def launch(self, server_name, port, root_path): - self.app.include_router(self.router) - uvicorn.run(self.app, host=server_name, port=port, timeout_keep_alive=shared.cmd_opts.timeout_keep_alive, root_path=root_path) diff --git a/modules/initialize.py b/modules/initialize.py index f24f76375db..bb2573c6684 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -51,7 +51,7 @@ def initialize(): initialize_util.fix_torch_version() initialize_util.fix_asyncio_event_loop_policy() initialize_util.validate_tls_options() - initialize_util.configure_sigint_handler() + #initialize_util.configure_sigint_handler() initialize_util.configure_opts_onchange() from modules import modelloader diff --git a/webui.py b/webui.py index 224b4e8d00a..36a2b6ab71a 100644 --- a/webui.py +++ b/webui.py @@ -24,6 +24,25 @@ def create_api(app): return api +def ray_api(): + from modules.api. ray import ray_only + + from modules.shared_cmd_options import cmd_opts + + launch_api = cmd_opts.api + initialize.initialize() + + from modules import shared, ui_tempdir, script_callbacks, ui, progress, ui_extra_networks + + script_callbacks.before_ui_callback() + startup_timer.record("scripts before_ui_callback") + shared.demo = ui.create_ui() + startup_timer.record("create ui") + if not cmd_opts.no_gradio_queue: + shared.demo.queue(64) + ray_only() + + def api_only(): from fastapi import FastAPI from modules.shared_cmd_options import cmd_opts @@ -156,11 +175,11 @@ def webui(): if __name__ == "__main__": from modules.shared_cmd_options import cmd_opts - from modules.api.ray import ray_only + if cmd_opts.nowebui: api_only() elif cmd_opts.ray: - ray_only() + ray_api() else: webui() From 4c2fe081430d9ed28ef97a7e1510b271f80bcc2c Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 5 Oct 2023 15:43:51 -0400 Subject: [PATCH 17/61] cleanup --- modules/api/ray.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 79d2d2d06cb..62f860b7c0b 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -1,8 +1,8 @@ from ray import serve import ray -from fastapi import FastAPI + from modules.api.raypi import Raypi -from modules.api.api import Raypi + import time @@ -20,22 +20,6 @@ "per worker. Ignore this if your cluster auto-scales." ) -app = FastAPI() - -@serve.deployment( - ray_actor_options={"num_gpus": 1}, - autoscaling_config={"min_replicas": 0, "max_replicas": 2}, - #route_prefix="/sdapi/v1", - ) -@serve.ingress(app) -class RayDeployment: - def __init__(self): - pass - - -# 2: Deploy the deployment. - - def ray_only(): From 3dfda817c4f7b902e867b061cb40cb3d36f9a09c Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 5 Oct 2023 15:52:25 -0400 Subject: [PATCH 18/61] Update ray.py --- modules/api/ray.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 62f860b7c0b..cda664dbaa5 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -11,17 +11,6 @@ #ray.init("ray://localhost:10001") - -NUM_REPLICAS: int = 1 -if NUM_REPLICAS > ray.available_resources()["GPU"]: - print( - "Your cluster does not currently have enough resources to run with these settings. " - "Consider decreasing the number of workers, or decreasing the resources needed " - "per worker. Ignore this if your cluster auto-scales." - ) - - - def ray_only(): serve.shutdown() serve.start() From f1c22af1983dc1bbff21d558a081ab61caf397bb Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Sat, 7 Oct 2023 12:50:49 -0400 Subject: [PATCH 19/61] added ray env vars --- modules/api/raypi.py | 10 ++++++---- webui-user.sh | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/api/raypi.py b/modules/api/raypi.py index 3042a29c4b1..2a4d3f9a877 100644 --- a/modules/api/raypi.py +++ b/modules/api/raypi.py @@ -39,7 +39,7 @@ from modules import initialize_util from modules import script_callbacks - +import os import launch from ray import serve @@ -211,9 +211,11 @@ async def http_exception_handler(request: Request, e: HTTPException): api_middleware(app) @serve.deployment( - ray_actor_options={"num_gpus": 1}, - autoscaling_config={"min_replicas": 0, "max_replicas": 2}, - #route_prefix="/sdapi/v1", + ray_actor_options={"num_gpus": int(os.environ.get("RAY_NUM_GPUS", 0))}, + autoscaling_config={ + "min_replicas": int(os.environ.get("RAY_MIN_REPLICAS", 0)), + "max_replicas": int(os.environ.get("RAY_MAX_REPLICAS", 0)) + }, ) @serve.ingress(app) class Raypi: diff --git a/webui-user.sh b/webui-user.sh index b7400b74a0e..884cac76281 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -47,3 +47,6 @@ export ACCELERATE="True" #export NO_TCMALLOC="True" ########################################### +export RAY_NUM_GPUS=1 +export RAY_MIN_REPLICAS=0 +export RAY_MAX_REPLICAS=2 \ No newline at end of file From ad566e018ac4793ff648503fa0bb2f9167642a06 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:36:42 -0400 Subject: [PATCH 20/61] add env vars etc --- modules/api/ray.py | 5 ----- webui-user.sh | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index cda664dbaa5..2c6adc31013 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -14,12 +14,7 @@ def ray_only(): serve.shutdown() serve.start() - #Raypi.deploy() - - serve.run(Raypi.bind(), port=8000, route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app - - print("Done setting up replicas! Now accepting requests...") while True: time.sleep(1000) diff --git a/webui-user.sh b/webui-user.sh index 884cac76281..ea1c680930f 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -46,7 +46,7 @@ export ACCELERATE="True" # Uncomment to disable TCMalloc #export NO_TCMALLOC="True" -########################################### +###############ray############################ export RAY_NUM_GPUS=1 export RAY_MIN_REPLICAS=0 export RAY_MAX_REPLICAS=2 \ No newline at end of file From 41311ea2a975b59cd62f68b866c5bb0d21dba956 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:53:45 -0400 Subject: [PATCH 21/61] adding port env var for ray --- modules/api/ray.py | 4 ++-- webui-user.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 2c6adc31013..86d416e3d44 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -4,7 +4,7 @@ from modules.api.raypi import Raypi import time - +import os ray.init() @@ -14,7 +14,7 @@ def ray_only(): serve.shutdown() serve.start() - serve.run(Raypi.bind(), port=8000, route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app + serve.run(Raypi.bind(), port=int(os.environ.get("RAY_NUM_GPUS", 8000)), route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app print("Done setting up replicas! Now accepting requests...") while True: time.sleep(1000) diff --git a/webui-user.sh b/webui-user.sh index ea1c680930f..e37a2585b26 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -49,4 +49,5 @@ export ACCELERATE="True" ###############ray############################ export RAY_NUM_GPUS=1 export RAY_MIN_REPLICAS=0 -export RAY_MAX_REPLICAS=2 \ No newline at end of file +export RAY_MAX_REPLICAS=2 +export RAY_PORT=8000 From 42aacab2fc4616b17ca8680ce18c507d527609ba Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 13 Oct 2023 15:13:03 -0400 Subject: [PATCH 22/61] fixing env var --- modules/api/ray.py | 2 +- webui-user.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 86d416e3d44..a9f73616292 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -14,7 +14,7 @@ def ray_only(): serve.shutdown() serve.start() - serve.run(Raypi.bind(), port=int(os.environ.get("RAY_NUM_GPUS", 8000)), route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app + serve.run(Raypi.bind(), port=int(os.environ.get("RAY_PORT", 8000)), route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app print("Done setting up replicas! Now accepting requests...") while True: time.sleep(1000) diff --git a/webui-user.sh b/webui-user.sh index e37a2585b26..7868eee5de6 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -50,4 +50,4 @@ export ACCELERATE="True" export RAY_NUM_GPUS=1 export RAY_MIN_REPLICAS=0 export RAY_MAX_REPLICAS=2 -export RAY_PORT=8000 +export RAY_PORT=7575 From 350c53d340bf2a6fc32dbf8913c12228bbd5894a Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:20:11 -0400 Subject: [PATCH 23/61] serve start deteached --- modules/api/ray.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index a9f73616292..917075d461a 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -1,7 +1,7 @@ from ray import serve import ray -from modules.api.raypi import Raypi +from modules.api.raypi import FastAPIWrapper import time import os @@ -13,7 +13,9 @@ def ray_only(): serve.shutdown() - serve.start() + #serve.start(port=int(os.environ.get("RAY_PORT", 8000))) + serve.start(detached=True, http_options={"host": "0.0.0.0"}) + print(f"Starting Raypi on port {os.environ.get('RAY_PORT', 8000)}") serve.run(Raypi.bind(), port=int(os.environ.get("RAY_PORT", 8000)), route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app print("Done setting up replicas! Now accepting requests...") while True: From 39d9112b14898add5fdc7720bef7c626bd2e9acc Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:52:20 -0400 Subject: [PATCH 24/61] Update ray.py --- modules/api/ray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 917075d461a..93f6fe079b9 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -1,7 +1,7 @@ from ray import serve import ray -from modules.api.raypi import FastAPIWrapper +from modules.api.raypi import Raypi import time import os From a955bf20e26f4fca310dca771f254695d1b05f0f Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 13 Oct 2023 17:04:43 -0400 Subject: [PATCH 25/61] Update ray.py --- modules/api/ray.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 93f6fe079b9..34ce3929340 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -14,7 +14,11 @@ def ray_only(): serve.shutdown() #serve.start(port=int(os.environ.get("RAY_PORT", 8000))) - serve.start(detached=True, http_options={"host": "0.0.0.0"}) + serve.start(detached=True, http_options={ + "host": "0.0.0.0", + "port": int(os.environ.get("RAY_PORT", 8000)) + } + ) print(f"Starting Raypi on port {os.environ.get('RAY_PORT', 8000)}") serve.run(Raypi.bind(), port=int(os.environ.get("RAY_PORT", 8000)), route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app print("Done setting up replicas! Now accepting requests...") From 0758191caa9ee07aee4792e1fc435d1f0ccc9300 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 13 Oct 2023 22:39:30 -0400 Subject: [PATCH 26/61] adding ip --- modules/api/ray.py | 2 +- webui-user.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 34ce3929340..48aa8742f06 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -15,7 +15,7 @@ def ray_only(): serve.shutdown() #serve.start(port=int(os.environ.get("RAY_PORT", 8000))) serve.start(detached=True, http_options={ - "host": "0.0.0.0", + "host": os.environ.get("RAY_IP", "0.0.0.0"), "port": int(os.environ.get("RAY_PORT", 8000)) } ) diff --git a/webui-user.sh b/webui-user.sh index 7868eee5de6..a9e0854ed91 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -51,3 +51,4 @@ export RAY_NUM_GPUS=1 export RAY_MIN_REPLICAS=0 export RAY_MAX_REPLICAS=2 export RAY_PORT=7575 +export RAY_IP="0.0.0.0" From 71124881ba08d13a400ba403e9e6febc59a1ce16 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Fri, 13 Oct 2023 22:54:43 -0400 Subject: [PATCH 27/61] adding ray head addy --- modules/api/ray.py | 3 +-- webui-user.sh | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 48aa8742f06..fe5fa4d0dd5 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -7,13 +7,12 @@ import os -ray.init() +ray.init(os.environ.get("RAY_HEAD_ADDRESS", "")) #ray.init("ray://localhost:10001") def ray_only(): serve.shutdown() - #serve.start(port=int(os.environ.get("RAY_PORT", 8000))) serve.start(detached=True, http_options={ "host": os.environ.get("RAY_IP", "0.0.0.0"), "port": int(os.environ.get("RAY_PORT", 8000)) diff --git a/webui-user.sh b/webui-user.sh index a9e0854ed91..d0302f940b9 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -52,3 +52,4 @@ export RAY_MIN_REPLICAS=0 export RAY_MAX_REPLICAS=2 export RAY_PORT=7575 export RAY_IP="0.0.0.0" +#export RAY_HEAD_ADDRESS='ray://ray-head:10001' From e5aa50d2fb40e85b9cc7f3fe638c33031f16ec49 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Mon, 16 Oct 2023 11:11:03 -0400 Subject: [PATCH 28/61] Update ray.py --- modules/api/ray.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index fe5fa4d0dd5..09b3f7b0b8c 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -7,9 +7,12 @@ import os -ray.init(os.environ.get("RAY_HEAD_ADDRESS", "")) +#ray.init(os.environ.get("RAY_HEAD_ADDRESS", "")) #ray.init("ray://localhost:10001") - +if "RAY_HEAD_ADDRESS" in os.environ: + ray.init(os.environ.get("RAY_HEAD_ADDRESS")) +else: + ray.init() def ray_only(): serve.shutdown() From 5b66013e6d4067c884212846f52c3286f1a242ef Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:16:39 -0400 Subject: [PATCH 29/61] fix spacing --- webui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui.py b/webui.py index 36a2b6ab71a..6791669ecb8 100644 --- a/webui.py +++ b/webui.py @@ -25,7 +25,7 @@ def create_api(app): def ray_api(): - from modules.api. ray import ray_only + from modules.api.ray import ray_only from modules.shared_cmd_options import cmd_opts From 1e52016307d886e534cd5674abcf21e150160b72 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:33:07 -0400 Subject: [PATCH 30/61] fix launch-utils ray api startup , was bypassing webui.py --- modules/launch_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/launch_utils.py b/modules/launch_utils.py index 88123977249..bf9b7e42cee 100644 --- a/modules/launch_utils.py +++ b/modules/launch_utils.py @@ -434,8 +434,7 @@ def start(): if '--nowebui' in sys.argv: webui.api_only() elif '--ray' in sys.argv: - from modules.api.ray import ray_only - ray_only() + webui.ray_api() else: webui.webui() From 9d390bf4e172688d73aecd253b8a0a6f1b34e446 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:40:40 -0400 Subject: [PATCH 31/61] added ray_docker env var --- modules/api/ray.py | 14 +++++++++----- webui-user.sh | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/api/ray.py b/modules/api/ray.py index 09b3f7b0b8c..254280f5731 100644 --- a/modules/api/ray.py +++ b/modules/api/ray.py @@ -16,11 +16,15 @@ def ray_only(): serve.shutdown() - serve.start(detached=True, http_options={ - "host": os.environ.get("RAY_IP", "0.0.0.0"), - "port": int(os.environ.get("RAY_PORT", 8000)) - } - ) + if "RAY_DOCKER" in os.environ: + print("starting ray in docker") + serve.start(detached=True, http_options={ + "host": os.environ.get("RAY_IP", "0.0.0.0"), + "port": int(os.environ.get("RAY_PORT", 8000)) + } + ) + else: + serve.start() print(f"Starting Raypi on port {os.environ.get('RAY_PORT', 8000)}") serve.run(Raypi.bind(), port=int(os.environ.get("RAY_PORT", 8000)), route_prefix="/sdapi/v1") #route_prefix="/sdapi/v1" # Call the launch_ray method to get the FastAPI app print("Done setting up replicas! Now accepting requests...") diff --git a/webui-user.sh b/webui-user.sh index d0302f940b9..a1fbc813285 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -53,3 +53,4 @@ export RAY_MAX_REPLICAS=2 export RAY_PORT=7575 export RAY_IP="0.0.0.0" #export RAY_HEAD_ADDRESS='ray://ray-head:10001' +#export RAY_DOCKER="True" From e2960ffa04bf21be745b19a77c1db7361856cf84 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:41:12 -0400 Subject: [PATCH 32/61] changed min replicas --- webui-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui-user.sh b/webui-user.sh index a1fbc813285..089013c4502 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -48,7 +48,7 @@ export ACCELERATE="True" ###############ray############################ export RAY_NUM_GPUS=1 -export RAY_MIN_REPLICAS=0 +export RAY_MIN_REPLICAS=1 export RAY_MAX_REPLICAS=2 export RAY_PORT=7575 export RAY_IP="0.0.0.0" From e22fa2af56431b08f1600cf9e8a816f1fbacf3f5 Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Tue, 17 Oct 2023 09:01:17 -0400 Subject: [PATCH 33/61] Update webui-user.sh --- webui-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui-user.sh b/webui-user.sh index 70306c60d5b..15e9a46e7b0 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -10,7 +10,7 @@ #clone_dir="stable-diffusion-webui" # Commandline arguments for webui.py, for example: export COMMANDLINE_ARGS="--medvram --opt-split-attention" -#export COMMANDLINE_ARGS="" +export COMMANDLINE_ARGS="--allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen" # python3 executable #python_cmd="python3" From d1c7df44d5b3f1964c87e37bd067ecb2fe8685de Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Tue, 17 Oct 2023 09:18:23 -0400 Subject: [PATCH 34/61] restoring some stuff back to normal --- modules/api/api.py | 26 ++++---------------------- modules/shared_items.py | 2 +- webui-user.sh | 2 +- webui.py | 4 ++-- 4 files changed, 8 insertions(+), 26 deletions(-) diff --git a/modules/api/api.py b/modules/api/api.py index 117da300317..c3dcc85c176 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -16,9 +16,6 @@ from fastapi.encoders import jsonable_encoder from secrets import compare_digest -from modules import initialize -initialize.imports() - import modules.shared as shared from modules import sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing, errors, restart, shared_items from modules.api import models @@ -36,9 +33,7 @@ import piexif import piexif.helper from contextlib import closing -from ray import serve -app = FastAPI() def script_name_to_index(name, scripts): try: @@ -202,18 +197,8 @@ async def http_exception_handler(request: Request, e: HTTPException): return handle_exception(request, e) -api_middleware(app) - - -@serve.deployment( - ray_actor_options={"num_gpus": 1}, - autoscaling_config={"min_replicas": 0, "max_replicas": 2}, - #route_prefix="/sdapi/v1", - ) -@serve.ingress(app) class Api: - def __init__(self): - initialize.initialize() + def __init__(self, app: FastAPI, queue_lock: Lock): if shared.cmd_opts.api_auth: self.credentials = {} for auth in shared.cmd_opts.api_auth.split(","): @@ -221,10 +206,9 @@ def __init__(self): self.credentials[user] = password self.router = APIRouter() - - - - print("API initialized") + self.app = app + self.queue_lock = queue_lock + api_middleware(self.app) self.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=models.TextToImageResponse) self.add_api_route("/sdapi/v1/img2img", self.img2imgapi, methods=["POST"], response_model=models.ImageToImageResponse) self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ExtrasSingleImageResponse) @@ -268,7 +252,6 @@ def __init__(self): self.default_script_arg_txt2img = [] self.default_script_arg_img2img = [] - def add_api_route(self, path: str, endpoint, **kwargs): if shared.cmd_opts.api_auth: return self.app.add_api_route(path, endpoint, dependencies=[Depends(self.auth)], **kwargs) @@ -802,4 +785,3 @@ def restart_webui(self): def stop_webui(request): shared.state.server_command = "stop" return Response("Stopping.") - diff --git a/modules/shared_items.py b/modules/shared_items.py index 301b32873d2..67dc4a1db57 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -120,4 +120,4 @@ def __reduce__(self): # The arguments are the current values of the sd_model property return (self.__class__, (self.sd_model,)) -#sys.modules['modules.shared'].__class__ = Shared +sys.modules['modules.shared'].__class__ = Shared diff --git a/webui-user.sh b/webui-user.sh index 089013c4502..ed82d025f60 100644 --- a/webui-user.sh +++ b/webui-user.sh @@ -11,7 +11,7 @@ # Commandline arguments for webui.py, for example: export COMMANDLINE_ARGS="--medvram --opt-split-attention" #export COMMANDLINE_ARGS="" -export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen --ray" +export COMMANDLINE_ARGS="--data-dir sd-data --allow-code --administrator --precision full --no-half --xformers --api --enable-insecure-extension-access --cors-allow-origins * --listen" # python3 executable #python_cmd="python3" diff --git a/webui.py b/webui.py index 6791669ecb8..ae439bab4cd 100644 --- a/webui.py +++ b/webui.py @@ -19,8 +19,8 @@ def create_api(app): from modules.api.api import Api from modules.call_queue import queue_lock - api = Api(app) - #api = Api(app, queue_lock) + + api = Api(app, queue_lock) return api From 72b1119c9757bda2f4f6949f9eb25d5addae9eae Mon Sep 17 00:00:00 2001 From: webcoderz <19884161+webcoderz@users.noreply.github.com> Date: Tue, 17 Oct 2023 12:47:20 -0400 Subject: [PATCH 35/61] fully working sd webui api on ray serve fully working sd webui api on ray serve --- extensions-builtin/Lora/networks.py | 38 ++++++++---- .../Lora/ui_extra_networks_lora.py | 21 +++++-- modules/api/api.py | 14 +++-- modules/api/raypi.py | 16 ++--- modules/hypernetworks/hypernetwork.py | 47 ++++++++++----- modules/hypernetworks/ui.py | 8 ++- modules/images.py | 8 ++- modules/img2img.py | 5 +- modules/initialize.py | 6 +- modules/initialize_util.py | 4 +- modules/processing.py | 28 +++++---- modules/sd_samplers_cfg_denoiser.py | 11 ++-- modules/sd_samplers_common.py | 17 ++++-- modules/sd_samplers_kdiffusion.py | 9 +-- modules/sd_samplers_timesteps.py | 14 +++-- modules/sd_unet.py | 13 +++-- modules/sd_vae.py | 4 +- modules/sd_vae_approx.py | 8 ++- modules/sd_vae_taesd.py | 7 ++- modules/shared.py | 6 +- modules/shared_items.py | 8 ++- modules/textual_inversion/dataset.py | 5 +- .../textual_inversion/textual_inversion.py | 58 +++++++++++++------ modules/txt2img.py | 5 +- modules/ui.py | 6 +- ...xtra_networks_checkpoints_user_metadata.py | 5 +- scripts/img2imgalt.py | 29 +++++++--- scripts/xyz_grid.py | 5 +- webui-user.sh | 2 +- 29 files changed, 267 insertions(+), 140 deletions(-) diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 96f935b236f..64c4512386e 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -15,6 +15,7 @@ from typing import Union from modules import shared, devices, sd_models, errors, scripts, sd_hijack +from modules.shared import shared_instance module_types = [ network_lora.ModuleTypeLora(), @@ -112,8 +113,10 @@ def match(match_list, regex_text): def assign_network_names_to_compvis_modules(sd_model): network_layer_mapping = {} - if shared.sd_model.is_sdxl: - for i, embedder in enumerate(shared.sd_model.conditioner.embedders): + #if shared.sd_model.is_sdxl: + if shared_instance.sd_model.is_sdxl: + #for i, embedder in enumerate(shared.sd_model.conditioner.embedders): + for i, embedder in enumerate(shared_instance.sd_model.conditioner.embedders): if not hasattr(embedder, 'wrapped'): continue @@ -122,12 +125,14 @@ def assign_network_names_to_compvis_modules(sd_model): network_layer_mapping[network_name] = module module.network_layer_name = network_name else: - for name, module in shared.sd_model.cond_stage_model.wrapped.named_modules(): + #for name, module in shared.sd_model.cond_stage_model.wrapped.named_modules(): + for name, module in shared_instance.sd_model.cond_stage_model.wrapped.named_modules(): network_name = name.replace(".", "_") network_layer_mapping[network_name] = module module.network_layer_name = network_name - for name, module in shared.sd_model.model.named_modules(): + #for name, module in shared.sd_model.model.named_modules(): + for name, module in shared_instance.sd_model.model.named_modules(): network_name = name.replace(".", "_") network_layer_mapping[network_name] = module module.network_layer_name = network_name @@ -142,11 +147,14 @@ def load_network(name, network_on_disk): sd = sd_models.read_state_dict(network_on_disk.filename) # this should not be needed but is here as an emergency fix for an unknown error people are experiencing in 1.2.0 - if not hasattr(shared.sd_model, 'network_layer_mapping'): - assign_network_names_to_compvis_modules(shared.sd_model) + #if not hasattr(shared.sd_model, 'network_layer_mapping'): + #assign_network_names_to_compvis_modules(shared.sd_model) + if not hasattr(shared_instance.sd_model, 'network_layer_mapping'): + assign_network_names_to_compvis_modules(shared_instance.sd_model) keys_failed_to_match = {} - is_sd2 = 'model_transformer_resblocks' in shared.sd_model.network_layer_mapping + #is_sd2 = 'model_transformer_resblocks' in shared.sd_model.network_layer_mapping + is_sd2 = 'model_transformer_resblocks' in shared_instance.sd_model.network_layer_mapping matched_networks = {} @@ -154,25 +162,31 @@ def load_network(name, network_on_disk): key_network_without_network_parts, network_part = key_network.split(".", 1) key = convert_diffusers_name_to_compvis(key_network_without_network_parts, is_sd2) - sd_module = shared.sd_model.network_layer_mapping.get(key, None) + #sd_module = shared.sd_model.network_layer_mapping.get(key, None) + sd_module = shared_instance.sd_model.network_layer_mapping.get(key, None) if sd_module is None: m = re_x_proj.match(key) if m: - sd_module = shared.sd_model.network_layer_mapping.get(m.group(1), None) + #sd_module = shared.sd_model.network_layer_mapping.get(m.group(1), None) + sd_module = shared_instance.sd_model.network_layer_mapping.get(m.group(1), None) + # SDXL loras seem to already have correct compvis keys, so only need to replace "lora_unet" with "diffusion_model" if sd_module is None and "lora_unet" in key_network_without_network_parts: key = key_network_without_network_parts.replace("lora_unet", "diffusion_model") - sd_module = shared.sd_model.network_layer_mapping.get(key, None) + #sd_module = shared.sd_model.network_layer_mapping.get(key, None) + sd_module = shared_instance.sd_model.network_layer_mapping.get(key, None) elif sd_module is None and "lora_te1_text_model" in key_network_without_network_parts: key = key_network_without_network_parts.replace("lora_te1_text_model", "0_transformer_text_model") - sd_module = shared.sd_model.network_layer_mapping.get(key, None) + #sd_module = shared.sd_model.network_layer_mapping.get(key, None) + sd_module = shared_instance.sd_model.network_layer_mapping.get(key, None) # some SD1 Loras also have correct compvis keys if sd_module is None: key = key_network_without_network_parts.replace("lora_te1_text_model", "transformer_text_model") - sd_module = shared.sd_model.network_layer_mapping.get(key, None) + #sd_module = shared.sd_model.network_layer_mapping.get(key, None) + sd_module = shared_instance.sd_model.network_layer_mapping.get(key, None) if sd_module is None: keys_failed_to_match[key_network] = key diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index 55409a7829d..8bb11a400dd 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -6,7 +6,7 @@ from modules import shared, ui_extra_networks from modules.ui_extra_networks import quote_js from ui_edit_user_metadata import LoraUserMetadataEditor - +from modules.shared import shared_instance class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): def __init__(self): @@ -52,17 +52,26 @@ def create_item(self, name, index=None, enable_filter=True): if shared.opts.lora_show_all or not enable_filter: pass +# elif sd_version == network.SdVersion.Unknown: +# model_version = network.SdVersion.SDXL if shared.sd_model.is_sdxl else network.SdVersion.SD2 if shared.sd_model.is_sd2 else network.SdVersion.SD1 +# if model_version.name in shared.opts.lora_hide_unknown_for_versions: +# return None +# elif shared.sd_model.is_sdxl and sd_version != network.SdVersion.SDXL: +# return None +# elif shared.sd_model.is_sd2 and sd_version != network.SdVersion.SD2: +# return None +# elif shared.sd_model.is_sd1 and sd_version != network.SdVersion.SD1: +# return None elif sd_version == network.SdVersion.Unknown: - model_version = network.SdVersion.SDXL if shared.sd_model.is_sdxl else network.SdVersion.SD2 if shared.sd_model.is_sd2 else network.SdVersion.SD1 + model_version = network.SdVersion.SDXL if shared_instance.sd_model.is_sdxl else network.SdVersion.SD2 if shared_instance.sd_model.is_sd2 else network.SdVersion.SD1 if model_version.name in shared.opts.lora_hide_unknown_for_versions: return None - elif shared.sd_model.is_sdxl and sd_version != network.SdVersion.SDXL: + elif shared_instance.sd_model.is_sdxl and sd_version != network.SdVersion.SDXL: return None - elif shared.sd_model.is_sd2 and sd_version != network.SdVersion.SD2: + elif shared_instance.sd_model.is_sd2 and sd_version != network.SdVersion.SD2: return None - elif shared.sd_model.is_sd1 and sd_version != network.SdVersion.SD1: + elif shared_instance.sd_model.is_sd1 and sd_version != network.SdVersion.SD1: return None - return item def list_items(self): diff --git a/modules/api/api.py b/modules/api/api.py index c3dcc85c176..aeee9d5968c 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -33,7 +33,7 @@ import piexif import piexif.helper from contextlib import closing - +from modules.shared import shared_instance def script_name_to_index(name, scripts): try: @@ -364,7 +364,8 @@ def text2imgapi(self, txt2imgreq: models.StableDiffusionTxt2ImgProcessingAPI): args.pop('save_images', None) with self.queue_lock: - with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: + with closing(StableDiffusionProcessingTxt2Img(sd_model=shared_instance.sd_model, **args)) as p: + #with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: p.is_api = True p.scripts = script_runner p.outpath_grids = opts.outdir_txt2img_grids @@ -424,7 +425,8 @@ def img2imgapi(self, img2imgreq: models.StableDiffusionImg2ImgProcessingAPI): args.pop('save_images', None) with self.queue_lock: - with closing(StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)) as p: + #with closing(StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)) as p: + with closing(StableDiffusionProcessingImg2Img(sd_model=shared_instance.sd_model, **args)) as p: p.init_images = [decode_base64_to_image(x) for x in init_images] p.is_api = True p.scripts = script_runner @@ -724,8 +726,10 @@ def train_hypernetwork(self, args: dict): except Exception as e: error = e finally: - shared.sd_model.cond_stage_model.to(devices.device) - shared.sd_model.first_stage_model.to(devices.device) + #shared.sd_model.cond_stage_model.to(devices.device) + #shared.sd_model.first_stage_model.to(devices.device) + shared_instance.sd_model.cond_stage_model.to(devices.device) + shared_instance.sd_model.first_stage_model.to(devices.device) if not apply_optimizations: sd_hijack.apply_optimizations() shared.state.end() diff --git a/modules/api/raypi.py b/modules/api/raypi.py index 2a4d3f9a877..5b68b90d235 100644 --- a/modules/api/raypi.py +++ b/modules/api/raypi.py @@ -40,7 +40,7 @@ from modules import initialize_util from modules import script_callbacks import os - +from modules.shared import shared_instance import launch from ray import serve @@ -349,8 +349,8 @@ def text2imgapi(self, txt2imgreq: models.StableDiffusionTxt2ImgProcessingAPI): send_images = args.pop('send_images', True) args.pop('save_images', None) - - with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: + with closing(StableDiffusionProcessingTxt2Img(sd_model=shared_instance.sd_model, **args)) as p: + #with closing(StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)) as p: p.is_api = True p.scripts = script_runner p.outpath_grids = opts.outdir_txt2img_grids @@ -409,8 +409,8 @@ def img2imgapi(self, img2imgreq: models.StableDiffusionImg2ImgProcessingAPI): send_images = args.pop('send_images', True) args.pop('save_images', None) - - with closing(StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)) as p: + with closing(StableDiffusionProcessingImg2Img(sd_model=shared_instance.sd_model, **args)) as p: + #with closing(StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)) as p: p.init_images = [decode_base64_to_image(x) for x in init_images] p.is_api = True p.scripts = script_runner @@ -737,8 +737,10 @@ def train_hypernetwork(self, args: dict): except Exception as e: error = e finally: - shared.sd_model.cond_stage_model.to(devices.device) - shared.sd_model.first_stage_model.to(devices.device) + #shared.sd_model.cond_stage_model.to(devices.device) + #shared.sd_model.first_stage_model.to(devices.device) + shared_instance.sd_model.cond_stage_model.to(devices.device) + shared_instance.sd_model.first_stage_model.to(devices.device) if not apply_optimizations: sd_hijack.apply_optimizations() shared.state.end() diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 70f1cbd26b6..f4876ab1e61 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -18,7 +18,7 @@ from collections import deque from statistics import stdev, mean - +from modules.shared import shared_instance optimizer_dict = {optim_name : cls_obj for optim_name, cls_obj in inspect.getmembers(torch.optim, inspect.isclass) if optim_name != "Optimizer"} @@ -525,7 +525,8 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi pin_memory = shared.opts.pin_memory - ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize, use_weight=use_weight) + #ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize, use_weight=use_weight) + ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared_instance.sd_model, cond_model=shared_instance.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize, use_weight=use_weight) if shared.opts.save_training_settings_to_txt: saved_params = dict( @@ -542,8 +543,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi if unload: shared.parallel_processing_allowed = False - shared.sd_model.cond_stage_model.to(devices.cpu) - shared.sd_model.first_stage_model.to(devices.cpu) + #shared.sd_model.cond_stage_model.to(devices.cpu) + #shared.sd_model.first_stage_model.to(devices.cpu) + shared_instance.sd_model.cond_stage_model.to(devices.cpu) + shared_instance.sd_model.first_stage_model.to(devices.cpu) weights = hypernetwork.weights() hypernetwork.train() @@ -614,16 +617,21 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi if use_weight: w = batch.weight.to(devices.device, non_blocking=pin_memory) if tag_drop_out != 0 or shuffle_tags: - shared.sd_model.cond_stage_model.to(devices.device) - c = shared.sd_model.cond_stage_model(batch.cond_text).to(devices.device, non_blocking=pin_memory) - shared.sd_model.cond_stage_model.to(devices.cpu) + #shared.sd_model.cond_stage_model.to(devices.device) + shared_instance.sd_model.cond_stage_model.to(devices.device) + c = shared_instance.sd_model.cond_stage_model(batch.cond_text).to(devices.device, non_blocking=pin_memory) + shared_instance.sd_model.cond_stage_model.to(devices.cpu) + #c = shared.sd_model.cond_stage_model(batch.cond_text).to(devices.device, non_blocking=pin_memory) + #shared.sd_model.cond_stage_model.to(devices.cpu) else: c = stack_conds(batch.cond).to(devices.device, non_blocking=pin_memory) if use_weight: - loss = shared.sd_model.weighted_forward(x, c, w)[0] / gradient_step + loss = shared_instance.sd_model.weighted_forward(x, c, w)[0] / gradient_step + #loss = shared.sd_model.weighted_forward(x, c, w)[0] / gradient_step del w else: - loss = shared.sd_model.forward(x, c)[0] / gradient_step + #loss = shared.sd_model.forward(x, c)[0] / gradient_step + loss = shared_instance.sd_model.forward(x, c)[0] / gradient_step del x del c @@ -683,11 +691,14 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi cuda_rng_state = None if torch.cuda.is_available(): cuda_rng_state = torch.cuda.get_rng_state_all() - shared.sd_model.cond_stage_model.to(devices.device) - shared.sd_model.first_stage_model.to(devices.device) + #shared.sd_model.cond_stage_model.to(devices.device) + #shared.sd_model.first_stage_model.to(devices.device) + shared_instance.sd_model.cond_stage_model.to(devices.device) + shared_instance.sd_model.first_stage_model.to(devices.device) p = processing.StableDiffusionProcessingTxt2Img( - sd_model=shared.sd_model, + #sd_model=shared.sd_model, + sd_model=shared_instance.sd_model, do_not_save_grid=True, do_not_save_samples=True, ) @@ -716,8 +727,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi image = processed.images[0] if len(processed.images) > 0 else None if unload: - shared.sd_model.cond_stage_model.to(devices.cpu) - shared.sd_model.first_stage_model.to(devices.cpu) + #shared.sd_model.cond_stage_model.to(devices.cpu) + #shared.sd_model.first_stage_model.to(devices.cpu) + shared_instance.sd_model.cond_stage_model.to(devices.cpu) + shared_instance.sd_model.first_stage_model.to(devices.cpu) torch.set_rng_state(rng_state) if torch.cuda.is_available(): torch.cuda.set_rng_state_all(cuda_rng_state) @@ -760,8 +773,10 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi del optimizer hypernetwork.optimizer_state_dict = None # dereference it after saving, to save memory. - shared.sd_model.cond_stage_model.to(devices.device) - shared.sd_model.first_stage_model.to(devices.device) + #shared.sd_model.cond_stage_model.to(devices.device) + #shared.sd_model.first_stage_model.to(devices.device) + shared_instance.sd_model.cond_stage_model.to(devices.device) + shared_instance.sd_model.first_stage_model.to(devices.device) shared.parallel_processing_allowed = old_parallel_processing_allowed return hypernetwork, filename diff --git a/modules/hypernetworks/ui.py b/modules/hypernetworks/ui.py index 8b6255e2b67..cce2ef28419 100644 --- a/modules/hypernetworks/ui.py +++ b/modules/hypernetworks/ui.py @@ -3,7 +3,7 @@ import gradio as gr import modules.hypernetworks.hypernetwork from modules import devices, sd_hijack, shared - +from modules.shared import shared_instance not_available = ["hardswish", "multiheadattention"] keys = [x for x in modules.hypernetworks.hypernetwork.HypernetworkModule.activation_dict if x not in not_available] @@ -32,7 +32,9 @@ def train_hypernetwork(*args): except Exception: raise finally: - shared.sd_model.cond_stage_model.to(devices.device) - shared.sd_model.first_stage_model.to(devices.device) + #shared.sd_model.cond_stage_model.to(devices.device) + #shared.sd_model.first_stage_model.to(devices.device) + shared_instance.sd_model.cond_stage_model.to(devices.device) + shared_instance.sd_model.first_stage_model.to(devices.device) sd_hijack.apply_optimizations() diff --git a/modules/images.py b/modules/images.py index eb644733898..cdf160b7fee 100644 --- a/modules/images.py +++ b/modules/images.py @@ -20,6 +20,8 @@ from modules import sd_samplers, shared, script_callbacks, errors from modules.paths_internal import roboto_ttf_file from modules.shared import opts +from modules.shared import shared_instance + LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS) @@ -350,8 +352,10 @@ class FilenameGenerator: 'height': lambda self: self.image.height, 'styles': lambda self: self.p and sanitize_filename_part(", ".join([style for style in self.p.styles if not style == "None"]) or "None", replace_spaces=False), 'sampler': lambda self: self.p and sanitize_filename_part(self.p.sampler_name, replace_spaces=False), - 'model_hash': lambda self: getattr(self.p, "sd_model_hash", shared.sd_model.sd_model_hash), - 'model_name': lambda self: sanitize_filename_part(shared.sd_model.sd_checkpoint_info.name_for_extra, replace_spaces=False), + 'model_hash': lambda self: getattr(self.p, "sd_model_hash", shared_instance.sd_model.sd_model_hash), + #'model_hash': lambda self: getattr(self.p, "sd_model_hash", shared.sd_model.sd_model_hash), + #'model_name': lambda self: sanitize_filename_part(shared.sd_model.sd_checkpoint_info.name_for_extra, replace_spaces=False), + 'model_name': lambda self: sanitize_filename_part(shared_instance.sd_model.sd_checkpoint_info.name_for_extra, replace_spaces=False), 'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'), 'datetime': lambda self, *args: self.datetime(*args), # accepts formats: [datetime], [datetime], [datetime