train-flux.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9c71076f-3569-47fd-9502-d856ddea1d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_FOLDER = \"./images\"\n",
    "OUTPUT_FOLDER = \"./output\"\n",
    "TRIGGER_WORD = \"teleski\"\n",
    "LORA_RANK = 16\n",
    "BATCHSIZE = 1\n",
    "LEARNING_RATE = 0.0001\n",
    "STEPS_TRAIN = 3000\n",
    "STEPS_SAVE = 250\n",
    "STEPS_SAMPLE = 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b8eb0634-382b-4f46-abc3-6113c947362f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "\n",
    "job_to_run = OrderedDict([\n",
    "    ('job', 'extension'),\n",
    "    ('config', OrderedDict([\n",
    "        # this name will be the folder and filename name\n",
    "        ('name', 'my_first_flux_lora_v1'),\n",
    "        ('process', [\n",
    "            OrderedDict([\n",
    "                ('type', 'sd_trainer'),\n",
    "                ('training_folder', OUTPUT_FOLDER),\n",
    "                ('performance_log_every', 100),\n",
    "                ('device', 'cuda:0'),\n",
    "                ('trigger_word', TRIGGER_WORD),\n",
    "                ('network', OrderedDict([\n",
    "                    ('type', 'lora'),\n",
    "                    ('linear', LORA_RANK),\n",
    "                    ('linear_alpha', LORA_RANK)\n",
    "                ])),\n",
    "                ('save', OrderedDict([\n",
    "                    ('dtype', 'float16'),  # precision to save\n",
    "                    ('save_every', STEPS_SAVE),  # save every this many steps\n",
    "                    ('max_step_saves_to_keep', 10)  # how many intermittent saves to keep\n",
    "                ])),\n",
    "                ('datasets', [\n",
    "                    # datasets are a folder of images. captions need to be txt files with the same name as the image\n",
    "                    # for instance image2.jpg and image2.txt. Only jpg, jpeg, and png are supported currently\n",
    "                    # images will automatically be resized and bucketed into the resolution specified\n",
    "                    OrderedDict([\n",
    "                        ('folder_path', INPUT_FOLDER),\n",
    "                        ('caption_ext', 'txt'),\n",
    "                        ('caption_dropout_rate', 0.05),  # will drop out the caption 5% of time\n",
    "                        ('shuffle_tokens', False),  # shuffle caption order, split by commas\n",
    "                        ('cache_latents_to_disk', True),  # leave this true unless you know what you're doing\n",
    "                        ('resolution', [512, 768, 1024])  # flux enjoys multiple resolutions\n",
    "                    ])\n",
    "                ]),\n",
    "                ('train', OrderedDict([\n",
    "                    ('batch_size', BATCHSIZE),\n",
    "                    ('steps', STEPS_TRAIN),  # total number of steps to train 500 - 4000 is a good range\n",
    "                    ('gradient_accumulation_steps', 1),\n",
    "                    ('train_unet', True),\n",
    "                    ('train_text_encoder', False),  # probably won't work with flux\n",
    "                    ('content_or_style', 'balanced'),  # content, style, balanced\n",
    "                    ('gradient_checkpointing', True),  # need the on unless you have a ton of vram\n",
    "                    ('noise_scheduler', 'flowmatch'),  # for training only\n",
    "                    ('optimizer', 'adamw8bit'),\n",
    "                    ('lr', LEARNING_RATE),\n",
    "                    # ema will smooth out learning, but could slow it down. Recommended to leave on.\n",
    "                    ('ema_config', OrderedDict([\n",
    "                        ('use_ema', True),\n",
    "                        ('ema_decay', 0.99)\n",
    "                    ])),\n",
    "                    # will probably need this if gpu supports it for flux, other dtypes may not work correctly\n",
    "                    ('dtype', 'bf16')\n",
    "                ])),\n",
    "                ('model', OrderedDict([\n",
    "                    # huggingface model name or path\n",
    "                    ('name_or_path', 'black-forest-labs/FLUX.1-dev'),\n",
    "                    ('is_flux', True),\n",
    "                    ('quantize', True),  # run 8bit mixed precision\n",
    "                    ('low_vram', True),  # uncomment this if the GPU is connected to your monitors. It will use less vram to quantize, but is slower.\n",
    "                ])),\n",
    "                ('sample', OrderedDict([\n",
    "                    ('sampler', 'flowmatch'),  # must match train.noise_scheduler\n",
    "                    ('sample_every', STEPS_SAMPLE),  # sample every this many steps\n",
    "                    ('width', 1024),\n",
    "                    ('height', 1024),\n",
    "                    ('prompts', [\n",
    "                        # you can add [trigger] to the prompts here and it will be replaced with the trigger word\n",
    "                        '[trigger] with a water bottle by the lake',\n",
    "                        '[trigger] playing tennis on a rainy day',\n",
    "                        '[trigger] smiling wearing a tuxedo, in a crowded room',\n",
    "                    ]),\n",
    "                    ('neg', ''),  # not used on flux\n",
    "                    ('seed', 42),\n",
    "                    ('walk_seed', True),\n",
    "                    ('guidance_scale', 4),\n",
    "                    ('sample_steps', 20)\n",
    "                ]))\n",
    "            ])\n",
    "        ])\n",
    "    ])),\n",
    "    # you can add any additional meta info here. [name] is replaced with config name at top\n",
    "    ('meta', OrderedDict([\n",
    "        ('name', '[name]'),\n",
    "        ('version', '1.0')\n",
    "    ]))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "68612d5f-7bae-4394-be4d-f94afba7a5ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
      "Token is valid (permission: write).\n",
      "Your token has been saved to /home/sagemaker-user/.cache/huggingface/token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "!huggingface-cli login --token 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"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "178a77b8-03bc-4b71-a27e-cd48dc62f181",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/opt/conda/lib/python3.10/site-packages/torchvision/image.so: undefined symbol: _ZN3c1017RegisterOperatorsD1Ev'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?\n",
      "  warn(\n",
      "2024-09-15 15:42:23.615689: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-09-15 15:42:23.615734: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-09-15 15:42:23.615745: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-09-15 15:42:23.620703: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: SSE4.1 SSE4.2 AVX AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"type\": \"sd_trainer\",\n",
      "    \"training_folder\": \"./output\",\n",
      "    \"performance_log_every\": 100,\n",
      "    \"device\": \"cuda:0\",\n",
      "    \"trigger_word\": \"teleski\",\n",
      "    \"network\": {\n",
      "        \"type\": \"lora\",\n",
      "        \"linear\": 16,\n",
      "        \"linear_alpha\": 16\n",
      "    },\n",
      "    \"save\": {\n",
      "        \"dtype\": \"float16\",\n",
      "        \"save_every\": 250,\n",
      "        \"max_step_saves_to_keep\": 10\n",
      "    },\n",
      "    \"datasets\": [\n",
      "        {\n",
      "            \"folder_path\": \"./images\",\n",
      "            \"caption_ext\": \"txt\",\n",
      "            \"caption_dropout_rate\": 0.05,\n",
      "            \"shuffle_tokens\": false,\n",
      "            \"cache_latents_to_disk\": true,\n",
      "            \"resolution\": [\n",
      "                512,\n",
      "                768,\n",
      "                1024\n",
      "            ]\n",
      "        }\n",
      "    ],\n",
      "    \"train\": {\n",
      "        \"batch_size\": 1,\n",
      "        \"steps\": 3000,\n",
      "        \"gradient_accumulation_steps\": 1,\n",
      "        \"train_unet\": true,\n",
      "        \"train_text_encoder\": false,\n",
      "        \"content_or_style\": \"balanced\",\n",
      "        \"gradient_checkpointing\": true,\n",
      "        \"noise_scheduler\": \"flowmatch\",\n",
      "        \"optimizer\": \"adamw8bit\",\n",
      "        \"lr\": 0.0001,\n",
      "        \"ema_config\": {\n",
      "            \"use_ema\": true,\n",
      "            \"ema_decay\": 0.99\n",
      "        },\n",
      "        \"dtype\": \"bf16\"\n",
      "    },\n",
      "    \"model\": {\n",
      "        \"name_or_path\": \"black-forest-labs/FLUX.1-dev\",\n",
      "        \"is_flux\": true,\n",
      "        \"quantize\": true,\n",
      "        \"low_vram\": true\n",
      "    },\n",
      "    \"sample\": {\n",
      "        \"sampler\": \"flowmatch\",\n",
      "        \"sample_every\": 250,\n",
      "        \"width\": 1024,\n",
      "        \"height\": 1024,\n",
      "        \"prompts\": [\n",
      "            \"[trigger] with a water bottle by the lake\",\n",
      "            \"[trigger] playing tennis on a rainy day\",\n",
      "            \"[trigger] smiling wearing a tuxedo, in a crowded room\"\n",
      "        ],\n",
      "        \"neg\": \"\",\n",
      "        \"seed\": 42,\n",
      "        \"walk_seed\": true,\n",
      "        \"guidance_scale\": 4,\n",
      "        \"sample_steps\": 20\n",
      "    }\n",
      "}\n",
      "Using EMA\n",
      "\n",
      "#############################################\n",
      "# Running job: my_first_flux_lora_v1\n",
      "#############################################\n",
      "\n",
      "\n",
      "Running  1 process\n",
      "Loading Flux model\n",
      "Loading transformer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/controlnet_aux/mediapipe_face/mediapipe_face_common.py:7: UserWarning: The module 'mediapipe' is not installed. The package will have limited functionality. Please install it using the command: pip install 'mediapipe'\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_5m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_5m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.\n",
      "  return register_model(fn_wrapper)\n",
      "/opt/conda/lib/python3.10/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_11m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_11m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.\n",
      "  return register_model(fn_wrapper)\n",
      "/opt/conda/lib/python3.10/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.\n",
      "  return register_model(fn_wrapper)\n",
      "/opt/conda/lib/python3.10/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_384 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_384. This is because the name being registered conflicts with an existing name. Please check if this is not expected.\n",
      "  return register_model(fn_wrapper)\n",
      "/opt/conda/lib/python3.10/site-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_512 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_512. This is because the name being registered conflicts with an existing name. Please check if this is not expected.\n",
      "  return register_model(fn_wrapper)\n",
      "/home/sagemaker-user/./ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py:61: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler()\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81fcdeeda774484eb753cff31609fdd9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 3 files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quantizing transformer\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a6de01a19fd486f9442df44e6ca60a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "scheduler/scheduler_config.json:   0%|          | 0.00/274 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading vae\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e0f5ecbada549389750115d57cbfe85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vae/config.json:   0%|          | 0.00/774 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7fbac946eac4fa380b21026e44b03ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "diffusion_pytorch_model.safetensors:   0%|          | 0.00/168M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading t5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e42642cb5cf04b60a9f7cddbb23a7d9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_2/tokenizer_config.json:   0%|          | 0.00/20.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0e7502d64e249b7ab9cd8051d071909",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21df221df9234407b4bf0672bc4f1520",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_2/tokenizer.json:   0%|          | 0.00/2.42M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "224449934d9e46a08455becd96c87134",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_2/special_tokens_map.json:   0%|          | 0.00/2.54k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05a3b540aec945ef853d04f6d0f169c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "text_encoder_2/config.json:   0%|          | 0.00/782 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc9de535c6354ce3bbaba36b7aabc3eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "(…)t_encoder_2/model.safetensors.index.json:   0%|          | 0.00/19.9k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b6f6ec093534927ab8a3443e2b8da6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "334e54d9c6c748a492b626b1d4a5dde3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00001-of-00002.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f461df7ec5ee415da9945ba85f945833",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00002-of-00002.safetensors:   0%|          | 0.00/4.53G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e44db5cfce21485c92b96c376b410b88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quantizing T5\n",
      "Loading clip\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8fcd9a7f41941bbaaaeec39dd5219e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "text_encoder/config.json:   0%|          | 0.00/613 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a64320b14504cf0bf055e148b0aa0c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/246M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e31dd199514f4ba6b424590a15522bda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer/tokenizer_config.json:   0%|          | 0.00/705 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9894a34fa015412186b33653015909b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer/vocab.json:   0%|          | 0.00/1.06M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0186b27e4cff4779be6d75920db3823e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer/merges.txt:   0%|          | 0.00/525k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "656abf22982449e6a6b8c60543e09928",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer/special_tokens_map.json:   0%|          | 0.00/588 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "making pipe\n",
      "preparing\n",
      "create LoRA network. base dim (rank): 16, alpha: 16\n",
      "neuron dropout: p=None, rank dropout: p=None, module dropout: p=None\n",
      "create LoRA for Text Encoder: 0 modules.\n",
      "create LoRA for U-Net: 494 modules.\n",
      "enable LoRA for U-Net\n",
      "Dataset: ./images\n",
      "  -  Preprocessing image dimensions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:00<00:00, 115.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  -  Found 16 images\n",
      "Bucket sizes for ./images:\n",
      "384x640: 7 files\n",
      "448x576: 6 files\n",
      "512x512: 3 files\n",
      "3 buckets made\n",
      "Caching latents for ./images\n",
      " - Saving latents to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Caching latents to disk: 100%|██████████| 16/16 [00:01<00:00, 11.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: ./images\n",
      "  -  Preprocessing image dimensions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:00<00:00, 37076.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  -  Found 16 images\n",
      "Bucket sizes for ./images:\n",
      "576x960: 7 files\n",
      "640x832: 6 files\n",
      "768x768: 3 files\n",
      "3 buckets made\n",
      "Caching latents for ./images\n",
      " - Saving latents to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Caching latents to disk: 100%|██████████| 16/16 [00:02<00:00,  7.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: ./images\n",
      "  -  Preprocessing image dimensions\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 16/16 [00:00<00:00, 36751.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  -  Found 16 images\n",
      "Bucket sizes for ./images:\n",
      "704x1216: 7 files\n",
      "832x1152: 6 files\n",
      "960x960: 1 files\n",
      "1024x1024: 1 files\n",
      "896x896: 1 files\n",
      "5 buckets made\n",
      "Caching latents for ./images\n",
      " - Saving latents to disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Caching latents to disk: 100%|██████████| 16/16 [00:03<00:00,  4.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating baseline samples before training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:   0%|          | 0/3000 [00:00<?, ?it/s] /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.\n",
      "  with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context:  # type: ignore[attr-defined]\n",
      "my_first_flux_lora_v1:   3%|▎         | 99/3000 [12:08<6:06:29,  7.58s/it, lr: 1.0e-04 loss: 2.894e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.7572s avg - train_loop, num = 10\n",
      " - 4.8264s avg - backward, num = 10\n",
      " - 2.2447s avg - predict_unet, num = 10\n",
      " - 0.3037s avg - calculate_loss, num = 10\n",
      " - 0.2274s avg - optimizer_step, num = 10\n",
      " - 0.0955s avg - encode_prompt, num = 10\n",
      " - 0.1664s avg - reset_batch, num = 2\n",
      " - 0.0047s avg - get_batch, num = 10\n",
      " - 0.0021s avg - preprocess_batch, num = 10\n",
      " - 0.0008s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:   7%|▋         | 199/3000 [24:00<3:51:49,  4.97s/it, lr: 1.0e-04 loss: 5.279e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 4.9760s avg - train_loop, num = 10\n",
      " - 2.9288s avg - backward, num = 10\n",
      " - 1.5236s avg - predict_unet, num = 10\n",
      " - 0.2001s avg - calculate_loss, num = 10\n",
      " - 0.1693s avg - optimizer_step, num = 10\n",
      " - 0.0954s avg - encode_prompt, num = 10\n",
      " - 0.1654s avg - reset_batch, num = 2\n",
      " - 0.0063s avg - get_batch, num = 10\n",
      " - 0.0021s avg - preprocess_batch, num = 10\n",
      " - 0.0008s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:   8%|▊         | 249/3000 [30:19<4:35:09,  6.00s/it, lr: 1.0e-04 loss: 4.640e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:13<02:27, 73.91s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:27<01:13, 73.99s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:42<00:00, 74.02s/it]\u001b[A\n",
      "my_first_flux_lora_v1:   8%|▊         | 249/3000 [30:19<4:35:09,  6.00s/it, lr: 1.0e-04 loss: 4.640e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:   8%|▊         | 249/3000 [30:22<4:35:09,  6.00s/it, lr: 1.0e-04 loss: 4.640e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  10%|▉         | 299/3000 [36:20<5:08:04,  6.84s/it, lr: 1.0e-04 loss: 5.680e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 6.8134s avg - train_loop, num = 10\n",
      " - 4.2239s avg - backward, num = 10\n",
      " - 1.9756s avg - predict_unet, num = 10\n",
      " - 0.2862s avg - calculate_loss, num = 10\n",
      " - 0.2094s avg - optimizer_step, num = 10\n",
      " - 0.0950s avg - encode_prompt, num = 10\n",
      " - 0.1638s avg - reset_batch, num = 2\n",
      " - 0.0020s avg - preprocess_batch, num = 10\n",
      " - 0.0014s avg - get_batch, num = 10\n",
      " - 0.0008s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  13%|█▎        | 399/3000 [48:19<4:43:13,  6.53s/it, lr: 1.0e-04 loss: 4.287e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 5.7149s avg - train_loop, num = 10\n",
      " - 3.4888s avg - backward, num = 10\n",
      " - 1.6872s avg - predict_unet, num = 10\n",
      " - 0.2382s avg - calculate_loss, num = 10\n",
      " - 0.1892s avg - optimizer_step, num = 10\n",
      " - 0.0947s avg - encode_prompt, num = 10\n",
      " - 0.1635s avg - reset_batch, num = 2\n",
      " - 0.0018s avg - preprocess_batch, num = 10\n",
      " - 0.0015s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  17%|█▋        | 499/3000 [1:00:46<4:37:02,  6.65s/it, lr: 1.0e-04 loss: 3.615e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:14<02:29, 74.54s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:28<01:14, 74.21s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:42<00:00, 74.14s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  17%|█▋        | 499/3000 [1:00:46<4:37:02,  6.65s/it, lr: 1.0e-04 loss: 3.615e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  17%|█▋        | 499/3000 [1:00:49<4:37:02,  6.65s/it, lr: 1.0e-04 loss: 3.615e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n",
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 6.9916s avg - train_loop, num = 10\n",
      " - 4.3499s avg - backward, num = 10\n",
      " - 2.0167s avg - predict_unet, num = 10\n",
      " - 0.2957s avg - calculate_loss, num = 10\n",
      " - 0.2143s avg - optimizer_step, num = 10\n",
      " - 0.0949s avg - encode_prompt, num = 10\n",
      " - 0.1691s avg - reset_batch, num = 2\n",
      " - 0.0018s avg - preprocess_batch, num = 10\n",
      " - 0.0016s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  20%|█▉        | 599/3000 [1:12:42<3:48:01,  5.70s/it, lr: 1.0e-04 loss: 3.623e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.1491s avg - train_loop, num = 10\n",
      " - 4.4561s avg - backward, num = 10\n",
      " - 2.0573s avg - predict_unet, num = 10\n",
      " - 0.3031s avg - calculate_loss, num = 10\n",
      " - 0.2163s avg - optimizer_step, num = 10\n",
      " - 0.0947s avg - encode_prompt, num = 10\n",
      " - 0.1687s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0014s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0002s avg - prepare_latents, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  23%|██▎       | 699/3000 [1:25:02<4:43:50,  7.40s/it, lr: 1.0e-04 loss: 3.630e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 8.2356s avg - train_loop, num = 10\n",
      " - 5.1850s avg - backward, num = 10\n",
      " - 2.3401s avg - predict_unet, num = 10\n",
      " - 0.3513s avg - calculate_loss, num = 10\n",
      " - 0.2419s avg - optimizer_step, num = 10\n",
      " - 0.0948s avg - encode_prompt, num = 10\n",
      " - 0.1683s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0015s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  25%|██▍       | 749/3000 [1:31:04<5:27:35,  8.73s/it, lr: 1.0e-04 loss: 3.539e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:14<02:28, 74.12s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:28<01:14, 74.02s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:42<00:00, 74.01s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  25%|██▍       | 749/3000 [1:31:04<5:27:35,  8.73s/it, lr: 1.0e-04 loss: 3.539e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  25%|██▍       | 749/3000 [1:31:07<5:27:35,  8.73s/it, lr: 1.0e-04 loss: 3.539e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  27%|██▋       | 799/3000 [1:36:42<2:58:48,  4.87s/it, lr: 1.0e-04 loss: 3.821e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 5.8543s avg - train_loop, num = 10\n",
      " - 3.5839s avg - backward, num = 10\n",
      " - 1.7246s avg - predict_unet, num = 10\n",
      " - 0.2430s avg - calculate_loss, num = 10\n",
      " - 0.1912s avg - optimizer_step, num = 10\n",
      " - 0.0946s avg - encode_prompt, num = 10\n",
      " - 0.1708s avg - reset_batch, num = 2\n",
      " - 0.0018s avg - preprocess_batch, num = 10\n",
      " - 0.0014s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  30%|██▉       | 899/3000 [1:49:10<4:58:06,  8.51s/it, lr: 1.0e-04 loss: 3.283e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.9918s avg - train_loop, num = 10\n",
      " - 5.0277s avg - backward, num = 10\n",
      " - 2.2701s avg - predict_unet, num = 10\n",
      " - 0.3420s avg - calculate_loss, num = 10\n",
      " - 0.2355s avg - optimizer_step, num = 10\n",
      " - 0.0949s avg - encode_prompt, num = 10\n",
      " - 0.1811s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0015s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  33%|███▎      | 999/3000 [2:01:17<2:55:50,  5.27s/it, lr: 1.0e-04 loss: 6.055e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:14<02:28, 74.49s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:28<01:14, 74.06s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:41<00:00, 73.76s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  33%|███▎      | 999/3000 [2:01:17<2:55:50,  5.27s/it, lr: 1.0e-04 loss: 6.055e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  33%|███▎      | 999/3000 [2:01:20<2:55:50,  5.27s/it, lr: 1.0e-04 loss: 6.055e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n",
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 6.2725s avg - train_loop, num = 10\n",
      " - 3.8664s avg - backward, num = 10\n",
      " - 1.8288s avg - predict_unet, num = 10\n",
      " - 0.2646s avg - calculate_loss, num = 10\n",
      " - 0.1998s avg - optimizer_step, num = 10\n",
      " - 0.0949s avg - encode_prompt, num = 10\n",
      " - 0.1745s avg - reset_batch, num = 2\n",
      " - 0.0018s avg - preprocess_batch, num = 10\n",
      " - 0.0015s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  37%|███▋      | 1099/3000 [2:13:16<3:51:45,  7.31s/it, lr: 1.0e-04 loss: 6.542e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.5408s avg - train_loop, num = 10\n",
      " - 4.7229s avg - backward, num = 10\n",
      " - 2.1547s avg - predict_unet, num = 10\n",
      " - 0.3204s avg - calculate_loss, num = 10\n",
      " - 0.2275s avg - optimizer_step, num = 10\n",
      " - 0.0946s avg - encode_prompt, num = 10\n",
      " - 0.1710s avg - reset_batch, num = 2\n",
      " - 0.0018s avg - preprocess_batch, num = 10\n",
      " - 0.0014s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  40%|███▉      | 1199/3000 [2:25:31<2:58:59,  5.96s/it, lr: 1.0e-04 loss: 2.886e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.6640s avg - train_loop, num = 10\n",
      " - 4.7771s avg - backward, num = 10\n",
      " - 2.1743s avg - predict_unet, num = 10\n",
      " - 0.3231s avg - calculate_loss, num = 10\n",
      " - 0.2277s avg - optimizer_step, num = 10\n",
      " - 0.0957s avg - encode_prompt, num = 10\n",
      " - 0.1661s avg - reset_batch, num = 3\n",
      " - 0.0074s avg - get_batch, num = 10\n",
      " - 0.0021s avg - preprocess_batch, num = 10\n",
      " - 0.0008s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  42%|████▏     | 1249/3000 [2:31:25<2:38:06,  5.42s/it, lr: 1.0e-04 loss: 3.341e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:14<02:29, 74.57s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:28<01:13, 73.99s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:42<00:00, 73.94s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  42%|████▏     | 1249/3000 [2:31:25<2:38:06,  5.42s/it, lr: 1.0e-04 loss: 3.341e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 1250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  42%|████▏     | 1249/3000 [2:31:28<2:38:06,  5.42s/it, lr: 1.0e-04 loss: 3.341e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  43%|████▎     | 1299/3000 [2:37:35<2:45:52,  5.85s/it, lr: 1.0e-04 loss: 7.026e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 6.4589s avg - train_loop, num = 10\n",
      " - 3.9510s avg - backward, num = 10\n",
      " - 1.8749s avg - predict_unet, num = 10\n",
      " - 0.2675s avg - calculate_loss, num = 10\n",
      " - 0.2022s avg - optimizer_step, num = 10\n",
      " - 0.0958s avg - encode_prompt, num = 10\n",
      " - 0.1719s avg - reset_batch, num = 2\n",
      " - 0.0070s avg - get_batch, num = 10\n",
      " - 0.0021s avg - preprocess_batch, num = 10\n",
      " - 0.0008s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  47%|████▋     | 1399/3000 [2:49:47<3:55:29,  8.83s/it, lr: 1.0e-04 loss: 4.658e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.9872s avg - train_loop, num = 10\n",
      " - 5.0068s avg - backward, num = 10\n",
      " - 2.2455s avg - predict_unet, num = 10\n",
      " - 0.3390s avg - calculate_loss, num = 10\n",
      " - 0.2338s avg - optimizer_step, num = 10\n",
      " - 0.0958s avg - encode_prompt, num = 10\n",
      " - 0.1729s avg - reset_batch, num = 2\n",
      " - 0.0079s avg - get_batch, num = 10\n",
      " - 0.0022s avg - preprocess_batch, num = 10\n",
      " - 0.0008s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  50%|████▉     | 1499/3000 [3:01:33<3:00:41,  7.22s/it, lr: 1.0e-04 loss: 3.328e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:14<02:29, 74.56s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:28<01:14, 74.24s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:42<00:00, 74.06s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  50%|████▉     | 1499/3000 [3:01:33<3:00:41,  7.22s/it, lr: 1.0e-04 loss: 3.328e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 1500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  50%|████▉     | 1499/3000 [3:01:36<3:00:41,  7.22s/it, lr: 1.0e-04 loss: 3.328e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n",
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 6.3021s avg - train_loop, num = 10\n",
      " - 3.8817s avg - backward, num = 10\n",
      " - 1.8404s avg - predict_unet, num = 10\n",
      " - 0.2634s avg - calculate_loss, num = 10\n",
      " - 0.2014s avg - optimizer_step, num = 10\n",
      " - 0.0951s avg - encode_prompt, num = 10\n",
      " - 0.1747s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0014s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0002s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  53%|█████▎    | 1599/3000 [3:13:57<3:07:54,  8.05s/it, lr: 1.0e-04 loss: 5.310e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.4270s avg - train_loop, num = 10\n",
      " - 4.6490s avg - backward, num = 10\n",
      " - 2.1239s avg - predict_unet, num = 10\n",
      " - 0.3148s avg - calculate_loss, num = 10\n",
      " - 0.2226s avg - optimizer_step, num = 10\n",
      " - 0.0949s avg - encode_prompt, num = 10\n",
      " - 0.1618s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0016s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  57%|█████▋    | 1699/3000 [3:26:07<2:52:10,  7.94s/it, lr: 1.0e-04 loss: 3.899e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.3515s avg - train_loop, num = 10\n",
      " - 4.5840s avg - backward, num = 10\n",
      " - 2.1204s avg - predict_unet, num = 10\n",
      " - 0.3114s avg - calculate_loss, num = 10\n",
      " - 0.2214s avg - optimizer_step, num = 10\n",
      " - 0.0946s avg - encode_prompt, num = 10\n",
      " - 0.1714s avg - reset_batch, num = 2\n",
      " - 0.0018s avg - preprocess_batch, num = 10\n",
      " - 0.0015s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0002s avg - prepare_latents, num = 10\n",
      " - 0.0002s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  58%|█████▊    | 1749/3000 [3:32:09<3:07:01,  8.97s/it, lr: 1.0e-04 loss: 3.377e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:13<02:27, 73.58s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:27<01:13, 73.85s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:40<00:00, 73.55s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  58%|█████▊    | 1749/3000 [3:32:09<3:07:01,  8.97s/it, lr: 1.0e-04 loss: 3.377e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 1750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  58%|█████▊    | 1749/3000 [3:32:12<3:07:01,  8.97s/it, lr: 1.0e-04 loss: 3.377e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  60%|█████▉    | 1799/3000 [3:38:31<2:43:48,  8.18s/it, lr: 1.0e-04 loss: 3.663e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.4845s avg - train_loop, num = 10\n",
      " - 4.6887s avg - backward, num = 10\n",
      " - 2.1348s avg - predict_unet, num = 10\n",
      " - 0.3197s avg - calculate_loss, num = 10\n",
      " - 0.2261s avg - optimizer_step, num = 10\n",
      " - 0.0945s avg - encode_prompt, num = 10\n",
      " - 0.1720s avg - reset_batch, num = 2\n",
      " - 0.0018s avg - preprocess_batch, num = 10\n",
      " - 0.0016s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  63%|██████▎   | 1899/3000 [3:50:22<2:47:44,  9.14s/it, lr: 1.0e-04 loss: 3.884e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.6474s avg - train_loop, num = 10\n",
      " - 4.7903s avg - backward, num = 10\n",
      " - 2.1897s avg - predict_unet, num = 10\n",
      " - 0.3231s avg - calculate_loss, num = 10\n",
      " - 0.2292s avg - optimizer_step, num = 10\n",
      " - 0.0947s avg - encode_prompt, num = 10\n",
      " - 0.1737s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0015s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  67%|██████▋   | 1999/3000 [4:02:31<1:59:44,  7.18s/it, lr: 1.0e-04 loss: 3.621e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:13<02:27, 73.95s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:27<01:13, 73.66s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:40<00:00, 73.25s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  67%|██████▋   | 1999/3000 [4:02:31<1:59:44,  7.18s/it, lr: 1.0e-04 loss: 3.621e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 2000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  67%|██████▋   | 1999/3000 [4:02:34<1:59:44,  7.18s/it, lr: 1.0e-04 loss: 3.621e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n",
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 8.8612s avg - train_loop, num = 10\n",
      " - 5.6181s avg - backward, num = 10\n",
      " - 2.4908s avg - predict_unet, num = 10\n",
      " - 0.3821s avg - calculate_loss, num = 10\n",
      " - 0.2505s avg - optimizer_step, num = 10\n",
      " - 0.0951s avg - encode_prompt, num = 10\n",
      " - 0.1680s avg - reset_batch, num = 2\n",
      " - 0.0020s avg - preprocess_batch, num = 10\n",
      " - 0.0016s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  70%|██████▉   | 2099/3000 [4:14:26<1:52:16,  7.48s/it, lr: 1.0e-04 loss: 4.731e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.0293s avg - train_loop, num = 10\n",
      " - 4.3711s avg - backward, num = 10\n",
      " - 2.0316s avg - predict_unet, num = 10\n",
      " - 0.2967s avg - calculate_loss, num = 10\n",
      " - 0.2142s avg - optimizer_step, num = 10\n",
      " - 0.0949s avg - encode_prompt, num = 10\n",
      " - 0.1686s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0014s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  73%|███████▎  | 2199/3000 [4:26:18<1:54:37,  8.59s/it, lr: 1.0e-04 loss: 2.566e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Timer 'my_first_flux_lora_v1 Timer':\n",
      " - 7.7016s avg - train_loop, num = 10\n",
      " - 4.8387s avg - backward, num = 10\n",
      " - 2.1894s avg - predict_unet, num = 10\n",
      " - 0.3290s avg - calculate_loss, num = 10\n",
      " - 0.2280s avg - optimizer_step, num = 10\n",
      " - 0.0947s avg - encode_prompt, num = 10\n",
      " - 0.1763s avg - reset_batch, num = 2\n",
      " - 0.0019s avg - preprocess_batch, num = 10\n",
      " - 0.0016s avg - get_batch, num = 10\n",
      " - 0.0007s avg - prepare_noise, num = 10\n",
      " - 0.0003s avg - batch_cleanup, num = 10\n",
      " - 0.0003s avg - prepare_latents, num = 10\n",
      " - 0.0000s avg - prepare_prompt, num = 10\n",
      " - 0.0000s avg - scheduler_step, num = 10\n",
      " - 0.0000s avg - grad_setup, num = 10\n",
      " - 0.0000s avg - log_to_tensorboard, num = 1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  75%|███████▍  | 2249/3000 [4:32:37<1:05:43,  5.25s/it, lr: 1.0e-04 loss: 1.959e-01]\n",
      "Generating Images:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n",
      "Generating Images:  33%|███▎      | 1/3 [01:14<02:28, 74.05s/it]\u001b[A\n",
      "Generating Images:  67%|██████▋   | 2/3 [02:27<01:13, 73.79s/it]\u001b[A\n",
      "Generating Images: 100%|██████████| 3/3 [03:41<00:00, 73.75s/it]\u001b[A\n",
      "my_first_flux_lora_v1:  75%|███████▍  | 2249/3000 [4:32:37<1:05:43,  5.25s/it, lr: 1.0e-04 loss: 1.959e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving at step 2250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  75%|███████▍  | 2249/3000 [4:32:40<1:05:43,  5.25s/it, lr: 1.0e-04 loss: 1.959e-01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to ./output/my_first_flux_lora_v1/optimizer.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "my_first_flux_lora_v1:  75%|███████▌  | 2254/3000 [4:33:03<1:11:07,  5.72s/it, lr: 1.0e-04 loss: 5.535e-01]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 6\u001b[0m\n\u001b[1;32m      3\u001b[0m sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./ai-toolkit\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtoolkit\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mjob\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m run_job\n\u001b[0;32m----> 6\u001b[0m \u001b[43mrun_job\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjob_to_run\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/./ai-toolkit/toolkit/job.py:43\u001b[0m, in \u001b[0;36mrun_job\u001b[0;34m(config, name)\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrun_job\u001b[39m(\n\u001b[1;32m     39\u001b[0m         config: Union[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mdict\u001b[39m, OrderedDict],\n\u001b[1;32m     40\u001b[0m         name\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     41\u001b[0m ):\n\u001b[1;32m     42\u001b[0m     job \u001b[38;5;241m=\u001b[39m get_job(config, name)\n\u001b[0;32m---> 43\u001b[0m     \u001b[43mjob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     44\u001b[0m     job\u001b[38;5;241m.\u001b[39mcleanup()\n",
      "File \u001b[0;32m~/./ai-toolkit/jobs/ExtensionJob.py:22\u001b[0m, in \u001b[0;36mExtensionJob.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRunning  \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m process\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mif\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess)\u001b[38;5;250m \u001b[39m\u001b[38;5;241m==\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01melse\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mes\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m process \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess:\n\u001b[0;32m---> 22\u001b[0m     \u001b[43mprocess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/./ai-toolkit/jobs/process/BaseSDTrainProcess.py:1709\u001b[0m, in \u001b[0;36mBaseSDTrainProcess.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1705\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgrad_accumulation_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m   1707\u001b[0m \u001b[38;5;66;03m# flush()\u001b[39;00m\n\u001b[1;32m   1708\u001b[0m \u001b[38;5;66;03m### HOOK ###\u001b[39;00m\n\u001b[0;32m-> 1709\u001b[0m loss_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhook_train_loop\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_list\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1710\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtimer\u001b[38;5;241m.\u001b[39mstop(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain_loop\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m   1711\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m did_first_flush:\n",
      "File \u001b[0;32m~/./ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py:1558\u001b[0m, in \u001b[0;36mSDTrainer.hook_train_loop\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m   1556\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[1;32m   1557\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m batch_list:\n\u001b[0;32m-> 1558\u001b[0m     loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_single_accumulation\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1559\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m total_loss \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1560\u001b[0m         total_loss \u001b[38;5;241m=\u001b[39m loss\n",
      "File \u001b[0;32m~/./ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py:1545\u001b[0m, in \u001b[0;36mSDTrainer.train_single_accumulation\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m   1543\u001b[0m                 loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m   1544\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1545\u001b[0m                 \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscaler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1547\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loss\u001b[38;5;241m.\u001b[39mdetach()\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/_tensor.py:521\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    511\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    512\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    513\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    514\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    519\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m    520\u001b[0m     )\n\u001b[0;32m--> 521\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    522\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m    523\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/autograd/__init__.py:289\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    284\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    286\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m    287\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    288\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 289\u001b[0m \u001b[43m_engine_run_backward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    290\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    291\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    292\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    293\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m    \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    297\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/autograd/graph.py:769\u001b[0m, in \u001b[0;36m_engine_run_backward\u001b[0;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    767\u001b[0m     unregister_hooks \u001b[38;5;241m=\u001b[39m _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[1;32m    768\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 769\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    770\u001b[0m \u001b[43m        \u001b[49m\u001b[43mt_outputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m    771\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[1;32m    772\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    773\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m attach_logging_hooks:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path.append('./ai-toolkit')\n",
    "from toolkit.job import run_job\n",
    "\n",
    "run_job(job_to_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33160842-8583-4cd4-9aef-08d0bb08723a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}