✨ LCM realtime rendering tab (#38)

* 🚧 wip * 🚧 Add seed control * 🔧 Allow shallow mutation * 🚧 Add prompt input * 🚧 Add send to canvas button * 🔧 Add lcm config * 🚧 Add denosing strength slider * 🚧 make negative prompt as drawer * nit * nit * 🐛 Fix controlnet * 🚧 basic functionality implemented * 🐛 Fix seed * fix height/width * 🐛 Prevent rendering if payload has not changed * adaptive render scheduling * nit * handle get input image exception * nit * multiple configs * Add controlnet scribble support * Fix active bound issue * Add link document button * Add send to canvas feature * 📝 documentation
huchenlei · Nov 26, 2023 · 30ce8ed · 30ce8ed
1 parent a73315c
commit 30ce8ed
Show file tree

Hide file tree

Showing 13 changed files with 425 additions and 51 deletions.
diff --git a/.eslintrc.cjs b/.eslintrc.cjs
@@ -11,5 +11,10 @@ module.exports = {
   ],
   parserOptions: {
     ecmaVersion: 'latest'
+  },
+  rules: {
+    "vue/no-mutating-props": ["error", {
+      "shallowOnly": true
+    }]
   }
 }
diff --git a/README.md b/README.md
@@ -33,6 +33,29 @@ set COMMANDLINE_ARGS=--cors-origins https://huchenlei.github.io [Rest of ARGS...
 **Step3: Search for stable-diffusion-ps-pea**
 ![Install](https://github.com/huchenlei/stable-diffusion-ps-pea/assets/20929282/35c2b802-4f31-45c2-8a24-e55f621adfae)
 ## Features
+
+##### :fire:[New Feature][2023-11-26] Realtime rendering powered by [LCM](https://github.com/luosiallen/latent-consistency-model)
+Recent advancement in LCM(Latent Consistency Model) has significantly increased the speed of
+inference of stable diffusion. The inference time now can be so fast that we can do real-time
+rendering of the canvas.
+
+Some preparations before you start exploring the real-time rendering tab:
+- Make sure to download the latest version of [`config_sharing/huchenlei_configs.json5`](https://github.com/huchenlei/stable-diffusion-ps-pea/blob/main/public/config/huchenlei_configs.json5) and upload it in the config tab. The new config file provides `lcm_base`, `lcm_lora_sd15`, `lcm_sd15_scribble` configs that are necessary.
+- Make sure you have LCM LoRA named `lcm_lora_sd15.safetensor` in A1111. Or you can change the name of LoRA in config `lcm_lora_sd15`. You can download LCM LoRAs [here](https://huggingface.co/collections/latent-consistency/latent-consistency-models-loras-654cdd24e111e16f0865fba6).
+
+After these preparations, you can now navigate to the real-time render tab (📹).
+- Select `lcm_base`, `lcm_lora_sd15` in RealtimeConfig.
+- Start drawing on canvas and enjoy!
+
+Other features:
+- If you have any selections on canvas, LCM will only render the selected area.
+- You can add `lcm_sd15_scribble` to RealtimeConfig, which will invoke ControlNet scribble model on canvas content. Make sure you have solid black brush color active when scribbling.
+- You can click `Send to canvas` to send the rendered view to canvas.
+
+![Screen Capture 034 - Photopea - Online Photo Editor - www photopea com](https://github.com/huchenlei/stable-diffusion-ps-pea/assets/20929282/0d53c264-6f74-42e2-9581-ba98a6b021ba)
+
+...More documentation work in progress...
+
 **Reference range selection**
 In A1111 img2img inpaint, one painpoint is that the inpaint area selection is either `WholeImage` or `OnlyMasked`. This
 might not be an issue when the image is within reasonable size (512x512). Once the image becomes big (1024x1024+), the

diff --git a/public/config/huchenlei_configs.json5 b/public/config/huchenlei_configs.json5
@@ -7,7 +7,7 @@
                 "prompt",
             ],
             "lhs": "",
-            "rhs": "(masterpiece: 1.3), (highres: 1.3), best quality",
+            "rhs": "(masterpiece: 1.3), (highres: 1.3), best quality,",
         },
         {
             "kind": "E",
@@ -46,6 +46,109 @@
             "rhs": 4,
         }
     ],
+    // General configs for LCM usage.
+    "lcm_base": [
+        {
+            "kind": "E",
+            "path": [
+                "generationMode",
+            ],
+            "lhs": 0,
+            "rhs": 1, // Set to img2img
+        },
+        {
+            "kind": "E",
+            "path": [
+                "commonPayload",
+                "sampler_name",
+            ],
+            "lhs": "",
+            "rhs": "Euler a", // 'Euler a' if LCM sampler is not available
+        },
+        {
+            "kind": "E",
+            "path": [
+                "commonPayload",
+                "steps",
+            ],
+            "lhs": 20,
+            "rhs": 8,
+        },
+        {
+            "kind": "E",
+            "path": [
+                "commonPayload",
+                "cfg_scale",
+            ],
+            "lhs": 7,
+            "rhs": 1.5,
+        },
+        {
+            "kind": "E",
+            "path": [
+                "commonPayload",
+                "n_iter",
+            ],
+            "lhs": 1,
+            "rhs": 1,
+        },
+        {
+            "kind": "E",
+            "path": [
+                "commonPayload",
+                "batch_size",
+            ],
+            "lhs": 1,
+            "rhs": 1,
+        },
+    ],
+    "lcm_sd15_lora": [
+        {
+            "kind": "E",
+            "path": [
+                "commonPayload",
+                "prompt",
+            ],
+            "lhs": "",
+            "rhs": "<lora:lcm_lora_sd15:1>",
+        },
+    ],
+    "lcm_sd15_scribble": [
+        {
+            "kind": "E",
+            "path": [
+                "generationMode",
+            ],
+            "lhs": 0,
+            "rhs": 0, // Set to txt2img
+        },
+        {
+            "kind": "A",
+            "path": [
+                "controlnetUnits"
+            ],
+            "index": 0,
+            "item": {
+                "kind": "N",
+                "rhs": {
+                    "control_mode": 0,
+                    "enabled": true,
+                    "guidance_end": 1,
+                    "guidance_start": 0,
+                    "low_vram": false,
+                    "model": "control_v11p_sd15_scribble [d4ba51ff]",
+                    "module": "none",
+                    "pixel_perfect": false,
+                    "processor_res": 512,
+                    "resize_mode": 1,
+                    "threshold_a": 64,
+                    "threshold_b": 64,
+                    "weight": 1,
+                    "linkedLayerName": ""
+                }
+            }
+        }
+    ],
     "LamaGenFill": [
         {
             "kind": "E",
@@ -83,16 +186,13 @@
             "item": {
                 "kind": "N",
                 "rhs": {
-                    "batch_images": "",
                     "control_mode": 2,
                     "enabled": true,
                     "guidance_end": 1,
                     "guidance_start": 0,
-                    "input_mode": 0,
                     "low_vram": false,
                     "model": "control_v11p_sd15_inpaint [ebff9138]",
                     "module": "inpaint_only+lama",
-                    "output_dir": "",
                     "pixel_perfect": false,
                     "processor_res": 512,
                     "resize_mode": 1,
@@ -114,16 +214,13 @@
             "item": {
                 "kind": "N",
                 "rhs": {
-                    "batch_images": "",
                     "control_mode": 0,
                     "enabled": true,
                     "guidance_end": 1,
                     "guidance_start": 0,
-                    "input_mode": 0,
                     "low_vram": false,
                     "model": "control_v11p_sd15_inpaint [ebff9138]",
                     "module": "inpaint_only",
-                    "output_dir": "",
                     "pixel_perfect": false,
                     "processor_res": 512,
                     "resize_mode": 1,
@@ -151,16 +248,13 @@
             "item": {
                 "kind": "N",
                 "rhs": {
-                    "batch_images": "",
                     "control_mode": 0,
                     "enabled": true,
                     "guidance_end": 1,
                     "guidance_start": 0,
-                    "input_mode": 0,
                     "low_vram": false,
                     "model": "control_v11f1e_sd15_tile [a371b31b]",
                     "module": "tile_colorfix",
-                    "output_dir": "",
                     "pixel_perfect": false,
                     "processor_res": 512,
                     "resize_mode": 1,
@@ -213,16 +307,13 @@
             "item": {
                 "kind": "N",
                 "rhs": {
-                    "batch_images": "",
                     "control_mode": 2,
                     "enabled": true,
                     "guidance_end": 1,
                     "guidance_start": 0,
-                    "input_mode": 0,
                     "low_vram": false,
                     "model": "control_v11f1e_sd15_tile [a371b31b]",
                     "module": "tile_colorfix",
-                    "output_dir": "",
                     "pixel_perfect": false,
                     "processor_res": 512,
                     "resize_mode": 1,

diff --git a/public/js/photopea.js b/public/js/photopea.js
@@ -252,17 +252,16 @@ function exportControlNetInputImage(format) {
     }
 }
 
-function getControlNetSelectionBound() {
+function getActiveBound() {
     if (hasSelection()) {
         app.echoToOE(boundsToString(
             app.activeDocument.selection.bounds
         ));
-    } else if (hasActiveLayer()) {
-        app.echoToOE(boundsToString(
-            app.activeDocument.activeLayer.bounds
-        ));
     } else {
-        app.echoToOE("error");
+        // Get whole document if no selection is available.
+        app.echoToOE(JSON.stringify(
+            [0, 0, app.activeDocument.width, app.activeDocument.height]
+        ));
     }
 }
 
@@ -409,6 +408,11 @@ function pickSegColor(rgb) {
         app.activeDocument.selection.fill(color);
         app.activeDocument.selection.deselect();
     }
-    
+
     app.echoToOE('success');
 }
+
+function getActiveDocName() {
+    const doc = app.activeDocument;
+    app.echoToOE(doc.name);
+}
diff --git a/src/App.vue b/src/App.vue
@@ -1,7 +1,14 @@
 <script setup lang="ts">
 import { RouterLink, RouterView, useRoute } from 'vue-router';
 import { useA1111ContextStore } from '@/stores/a1111ContextStore';
-import { GithubOutlined, LoginOutlined, PlayCircleOutlined, HistoryOutlined, SettingOutlined } from '@ant-design/icons-vue';
+import {
+  GithubOutlined,
+  LoginOutlined,
+  PlayCircleOutlined,
+  HistoryOutlined,
+  SettingOutlined,
+  VideoCameraOutlined,
+} from '@ant-design/icons-vue';
 import { reactive } from 'vue';
 
 const context = useA1111ContextStore().a1111Context;
@@ -10,6 +17,7 @@ const route = useRoute();
 const menuItems = reactive([
   { key: '/', path: '/', icon: LoginOutlined, textKey: 'nav.connection', requiresInit: false },
   { key: '/generation', path: '/generation', icon: PlayCircleOutlined, textKey: 'nav.generation', requiresInit: true },
+  { key: '/realtime', path: '/realtime', icon: VideoCameraOutlined, textKey: 'nav.realtime', requiresInit: true },
   { key: '/history', path: '/history', icon: HistoryOutlined, textKey: 'nav.history', requiresInit: true },
   { key: '/config', path: '/config', icon: SettingOutlined, textKey: 'nav.config', requiresInit: false },
 ]);

diff --git a/src/ControlNet.ts b/src/ControlNet.ts
@@ -7,28 +7,20 @@ enum ControlMode {
     ControlNet,
 };
 
-enum InputMode {
-    Simple,
-    Batch,
-};
-
 interface ControlNetImage {
     image: string;
     mask: string | null;
 };
 
 interface IControlNetUnit {
-    batch_images: string;
     control_mode: ControlMode;
     enabled: boolean;
     guidance_end: number;
     guidance_start: number;
     image: ControlNetImage | undefined;
-    input_mode: InputMode;
     low_vram: boolean;
     model: string;
     module: string;
-    output_dir: string;
     pixel_perfect: boolean;
     processor_res: number;
     resize_mode: ResizeMode;
@@ -39,17 +31,14 @@ interface IControlNetUnit {
 }
 
 class ControlNetUnit implements IControlNetUnit {
-    batch_images: string = '';
     control_mode: ControlMode = ControlMode.Balanced;
     enabled: boolean = false;
     guidance_end: number = 1.0;
     guidance_start: number = 0.0;
     image: ControlNetImage | undefined = undefined;
-    input_mode: InputMode = InputMode.Simple;
     low_vram: boolean = false;
     model: string = '';
     module: string = '';
-    output_dir: string = '';
     pixel_perfect: boolean = false;
     processor_res: number = 512;
     resize_mode: ResizeMode = ResizeMode.InnerFit;
@@ -217,7 +206,6 @@ export {
     type IControlNetUnit,
     type ModuleDetail,
     type ControlType,
-    type InputMode,
     ControlNetUnit,
     ControlMode,
     ControlNetContext,

diff --git a/src/components/ControlNetUnit.vue b/src/components/ControlNetUnit.vue
@@ -182,7 +182,7 @@ export default {
                 const context = useA1111ContextStore().controlnetContext;
                 const [imageBuffer, bounds] = await photopeaContext.executeTask(async () => {
                     const imageBuffer = await photopeaContext.invoke('exportControlNetInputImage', 'PNG') as ArrayBuffer;
-                    const bounds = JSON.parse(await photopeaContext.invoke('getControlNetSelectionBound')) as PhotopeaBound;
+                    const bounds = JSON.parse(await photopeaContext.invoke('getActiveBound')) as PhotopeaBound;
                     return [imageBuffer, bounds];
                 });
                 const image = await cropImage(imageBuffer, bounds);

diff --git a/src/components/ExtraNetworks.vue b/src/components/ExtraNetworks.vue
@@ -66,10 +66,6 @@ export default {
             emit('add:prompt', `(${item.name}:1.0)`);
         }
 
-        const afterVisibleChange = (bool: boolean) => {
-            console.log('visible', bool);
-        };
-
         const showDrawer = () => {
             visible.value = true;
         };
@@ -79,7 +75,6 @@ export default {
             NetworkType,
             currentNetworkType,
             searchKeyword,
-            afterVisibleChange,
             showDrawer,
             loraImages,
             embeddingImages,
@@ -95,11 +90,12 @@ export default {
         <DeploymentUnitOutlined />
     </a-button>
 
-    <a-drawer v-model:visible="visible" title="Extra Networks" placement="right" @after-visible-change="afterVisibleChange">
+    <a-drawer v-model:visible="visible" title="Extra Networks" placement="right">
         <a-space direction="vertical">
             <a-radio-group v-model:value="currentNetworkType" button-style="solid">
-                <a-radio-button v-for="network in [NetworkType.LoRA, NetworkType.Embedding]" :value="network">{{ network
-                }}</a-radio-button>
+                <a-radio-button v-for="network in [NetworkType.LoRA, NetworkType.Embedding]" :value="network"
+                    :key="network">{{ network
+                    }}</a-radio-button>
             </a-radio-group>
             <a-input v-model:value="searchKeyword" style="width: 100%" :placeholder="$t('gen.search') + '...'" />