cv_video.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time to apply to Video! </br>\n",
    "the next step would be to analyze these for the whole video. </br>\n",
    "So keep the background image, load every new frame of the video, run your calculations, and then save it to a video file. This tutorial might be helpful."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error opening the video file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "OpenCV: Couldn't read video stream from file \"Resources/Cars.mp4\"\n"
     ]
    }
   ],
   "source": [
    "# example codes\n",
    "# reference : https://learnopencv.com/reading-and-writing-videos-using-opencv/\n",
    "import cv2 \n",
    " \n",
    "# Create a video capture object, in this case we are reading the video from a file\n",
    "vid_capture = cv2.VideoCapture('Resources/Cars.mp4')\n",
    " \n",
    "if (vid_capture.isOpened() == False):\n",
    "  print(\"Error opening the video file\")\n",
    "# Read fps and frame count\n",
    "else:\n",
    "  # Get frame rate information\n",
    "  # You can replace 5 with CAP_PROP_FPS as well, they are enumerations\n",
    "  fps = vid_capture.get(5)\n",
    "  print('Frames per second : ', fps,'FPS')\n",
    " \n",
    "  # Get frame count\n",
    "  # You can replace 7 with CAP_PROP_FRAME_COUNT as well, they are enumerations\n",
    "  frame_count = vid_capture.get(7)\n",
    "  print('Frame count : ', frame_count)\n",
    " \n",
    "while(vid_capture.isOpened()):\n",
    "  # vid_capture.read() methods returns a tuple, first element is a bool \n",
    "  # and the second is frame\n",
    "  ret, frame = vid_capture.read()\n",
    "  if ret == True:\n",
    "    cv2.imshow('Frame',frame)\n",
    "    # 20 is in milliseconds, try to increase the value, say 50 and observe\n",
    "    key = cv2.waitKey(20)\n",
    "     \n",
    "    if key == ord('q'):\n",
    "      break\n",
    "  else:\n",
    "    break\n",
    " \n",
    "# Release the video capture object\n",
    "vid_capture.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "OpenCV: FFMPEG: tag 0x34363268/'h264' is not supported with codec id 27 and format 'mov / QuickTime / MOV'\n",
      "OpenCV: FFMPEG: fallback to use tag 0x31637661/'avc1'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file not found\n",
      "file not found\n",
      "file not found\n",
      "file not found\n",
      "file not found\n",
      "file not found\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "color_directory = \"2018-04-20-10-04-11/kinect/color\"\n",
    "depth_directory = \"2018-04-20-10-04-11/kinect/depth\"\n",
    "\n",
    "color_filenames = os.listdir(color_directory)\n",
    "depth_filenames = os.listdir(depth_directory)\n",
    "\n",
    "color_filenames.sort()\n",
    "depth_filenames.sort()\n",
    "\n",
    "#background_depth = cv2.imread(os.path.join(depth_directory, depth_filenames[0]))\n",
    "background_depth= np.load(os.path.join(depth_directory, depth_filenames[0]))\n",
    "\n",
    "output_filename = 'output_video.mov'\n",
    "fourcc = cv2.VideoWriter_fourcc(*'h264') # 비디오 코덱을 지정하는 4자 코드\n",
    "fps = 25.0 # 초당 프레임 수\n",
    "frame_size = (960,540) # 프레임 크기 (예: 가로 640, 세로 480)\n",
    "result_vid = cv2.VideoWriter(output_filename, fourcc, fps, frame_size) # VideoWriter 객체 생성\n",
    "# with color_filenames, convert to frames and store as video \n",
    "for filename in color_filenames:\n",
    "    frame = cv2.imread(os.path.join(color_directory, filename))  # 파일을 읽어 프레임으로 변환\n",
    "    result_vid.write(frame)  # 프레임을 결과 비디오에 추가\n",
    "\n",
    "\n",
    "#result_vid.write(color_filenames)\n",
    "result_vid.release() \n",
    "\n",
    "prev_bounding_box = ...\n",
    "\n",
    "for filename in depth_filenames:\n",
    "\n",
    "    # load Image\n",
    "    depth_image = np.load(os.path.join(depth_directory, filename)) # 파일을 읽어옴\n",
    "    color_filename = filename.split(\".\")[0] + \".png\"\n",
    "    color_filename = os.path.join(color_directory,color_filename)\n",
    "    \n",
    "    if not os.path.exists(color_filename):\n",
    "        print(\"file not found\")\n",
    "        continue\n",
    "    \n",
    "    color_image = cv2.imread(color_filename)\n",
    "    \n",
    "    # take depth difference\n",
    "    diff = depth_image-background_depth\n",
    "\n",
    "    ## process bounding box - copied from 'practice1.ipynb'\n",
    "    # Normalize the difference image to range [0, 255]\n",
    "    normalized_diff = cv2.normalize(np.abs(diff), None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n",
    "    # Binarize\n",
    "    _, im_th = cv2.threshold(normalized_diff, 30, 255, cv2.THRESH_BINARY)\n",
    "    # Taking a matrix of size 5 as the kernel ; gernerate 5*5 array filled with 1\n",
    "    kernel = np.ones((5, 5), np.uint8) \n",
    "\n",
    "    img_erosion = cv2.erode(im_th, kernel, iterations=3) \n",
    "    img_dilation = cv2.dilate(im_th, kernel, iterations=3) \n",
    "    # Get all group of pixels using cv.connectedComponentsWithStats()\n",
    "    num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(img_erosion)\n",
    "\n",
    "    #print(np.amin(depth_image),np.amax(depth_image))\n",
    "    #depth_image = cv2.cvtColor(depth_image,cv2.COLOR_GRAY2BGR)\n",
    "\n",
    "    # Step 4: Filter out small areas and draw bounding boxes\n",
    "    threshold = 100\n",
    "    # Iterate through stats to filter out small areas and draw bounding boxes\n",
    "    for i in range(1, num_labels): # num_labels : connected number of labels\n",
    "        area = stats[i, cv2.CC_STAT_AREA] # stats[i, cv2.CC_STAT_AREA]is the number of pixels(area) of ith label(group)\n",
    "        if area > threshold:\n",
    "            x, y, w, h = stats[i, cv2.CC_STAT_LEFT], stats[i, cv2.CC_STAT_TOP], stats[i, cv2.CC_STAT_WIDTH], stats[i, cv2.CC_STAT_HEIGHT]\n",
    "            cv2.rectangle(color_image, (x, y), (x + w, y + h), (0, 255, 0), 2)  # Draw bounding box on color image\n",
    "\n",
    "    # Display the result (comment out)\n",
    "    #plt.imshow(cv2.cvtColor(color_image, cv2.COLOR_BGR2RGB)) # Convert BGR to RGB for proper display with matplotlib\n",
    "    #plt.show()\n",
    "\n",
    "    cv2.imshow(\"Frame\",color_image)\n",
    "\n",
    "    result_vid.write(color_image)\n",
    "\n",
    "result_vid.release()\n",
    "\n",
    "# mp4 on mac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculating the bounding box i/u ; later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /Users/sojeonglee/.cache/torch/hub/ultralytics_yolov5_master\n",
      "YOLOv5 🚀 2024-4-5 Python-3.11.5 torch-2.2.1 CPU\n",
      "\n",
      "Fusing layers... \n",
      "[W NNPACK.cpp:64] Could not initialize NNPACK! Reason: Unsupported hardware.\n",
      "YOLOv5s summary: 213 layers, 7225885 parameters, 0 gradients, 16.4 GFLOPs\n",
      "Adding AutoShape... \n"
     ]
    },
    {
     "ename": "UnidentifiedImageError",
     "evalue": "cannot identify image file 'output_video.mov'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnidentifiedImageError\u001b[0m                    Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 21\u001b[0m\n\u001b[1;32m     18\u001b[0m result_video \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124moutput_video.mov\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;66;03m# Inference\u001b[39;00m\n\u001b[0;32m---> 21\u001b[0m results \u001b[38;5;241m=\u001b[39m model(result_video)\n\u001b[1;32m     23\u001b[0m \u001b[38;5;66;03m# Results\u001b[39;00m\n\u001b[1;32m     24\u001b[0m results\u001b[38;5;241m.\u001b[39mprint()\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m    113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    114\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/.cache/torch/hub/ultralytics_yolov5_master/models/common.py:835\u001b[0m, in \u001b[0;36mAutoShape.forward\u001b[0;34m(self, ims, size, augment, profile)\u001b[0m\n\u001b[1;32m    833\u001b[0m f \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimage\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m  \u001b[38;5;66;03m# filename\u001b[39;00m\n\u001b[1;32m    834\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(im, (\u001b[38;5;28mstr\u001b[39m, Path)):  \u001b[38;5;66;03m# filename or uri\u001b[39;00m\n\u001b[0;32m--> 835\u001b[0m     im, f \u001b[38;5;241m=\u001b[39m Image\u001b[38;5;241m.\u001b[39mopen(requests\u001b[38;5;241m.\u001b[39mget(im, stream\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\u001b[38;5;241m.\u001b[39mraw \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mstr\u001b[39m(im)\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m im), im\n\u001b[1;32m    836\u001b[0m     im \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39masarray(exif_transpose(im))\n\u001b[1;32m    837\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(im, Image\u001b[38;5;241m.\u001b[39mImage):  \u001b[38;5;66;03m# PIL Image\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/lib/python3.11/site-packages/PIL/Image.py:3280\u001b[0m, in \u001b[0;36mopen\u001b[0;34m(fp, mode, formats)\u001b[0m\n\u001b[1;32m   3278\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(message)\n\u001b[1;32m   3279\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcannot identify image file \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (filename \u001b[38;5;28;01mif\u001b[39;00m filename \u001b[38;5;28;01melse\u001b[39;00m fp)\n\u001b[0;32m-> 3280\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m UnidentifiedImageError(msg)\n",
      "\u001b[0;31mUnidentifiedImageError\u001b[0m: cannot identify image file 'output_video.mov'"
     ]
    }
   ],
   "source": [
    "# ai -> color image\n",
    "# camera capture color image only\n",
    "# depth image only\n",
    "\n",
    "# ai -> bounding box on color image\n",
    "# train -> depth image  only for\n",
    "# fine tuning\n",
    "\n",
    "# pytorch model zoo\n",
    "# torchvision models - models and weights ~~ 사이트 들어가기 -> object detection model!! 사용하면 됨\n",
    "# yolo pytorch hub : codes 있음\n",
    "import torch\n",
    "\n",
    "# Model\n",
    "model = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)\n",
    "\n",
    "# video\n",
    "result_video = 'output_video.mov'\n",
    "\n",
    "# Inference\n",
    "results = model(result_video)\n",
    "\n",
    "# Results\n",
    "results.print()\n",
    "results.show()  # or .show()\n",
    "\n",
    "results.xyxy[0]  # img1 predictions (tensor)\n",
    "results.pandas().xyxy[0]  # img1 predictions (pandas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# single frame -> comparison ; back ground frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import torch\n",
    "from PIL import Image\n",
    "\n",
    "# Model\n",
    "model = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)\n",
    "\n",
    "# video\n",
    "result_video = 'output_video.mov'\n",
    "\n",
    "# Open the video file\n",
    "cap = cv2.VideoCapture(result_video)\n",
    "\n",
    "# Frame counter\n",
    "frame_count = 0\n",
    "\n",
    "# Process each frame\n",
    "while cap.isOpened():\n",
    "    # Read a frame from the video\n",
    "    ret, frame = cap.read()\n",
    "\n",
    "    if ret:\n",
    "        # Convert the frame to RGB\n",
    "        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "        \n",
    "        # Convert the frame to PIL Image\n",
    "        pil_image = Image.fromarray(frame_rgb)\n",
    "        \n",
    "        # Inference with YOLOv5\n",
    "        results = model(pil_image)\n",
    "\n",
    "        # Print and show results\n",
    "        results.print()\n",
    "        results.show()\n",
    "        \n",
    "        frame_count += 1\n",
    "    else:\n",
    "        break\n",
    "\n",
    "# Release the video capture object\n",
    "cap.release()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /Users/sojeonglee/.cache/torch/hub/ultralytics_yolov5_master\n",
      "YOLOv5 🚀 2024-4-5 Python-3.11.5 torch-2.2.1 CPU\n",
      "\n",
      "Fusing layers... \n",
      "YOLOv5s summary: 213 layers, 7225885 parameters, 0 gradients, 16.4 GFLOPs\n",
      "Adding AutoShape... \n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import torch\n",
    "from torchvision.models.detection import fasterrcnn_resnet50_fpn\n",
    "\n",
    "# Load YOLOv5 model\n",
    "model = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)\n",
    "\n",
    "# Video settings\n",
    "video_path = 'output_video.MOV'\n",
    "output_path = 'output_video_yolo.mp4'\n",
    "\n",
    "# Open the video file\n",
    "cap = cv2.VideoCapture(video_path)\n",
    "fps = cap.get(cv2.CAP_PROP_FPS)\n",
    "frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "fourcc = cv2.VideoWriter_fourcc(*'mp4v')\n",
    "out = cv2.VideoWriter(output_path, fourcc, fps, (frame_width, frame_height))\n",
    "\n",
    "# Process each frame\n",
    "while cap.isOpened():\n",
    "    # Read a frame from the video\n",
    "    ret, frame = cap.read()\n",
    "\n",
    "    if ret:\n",
    "        # Convert the frame to RGB\n",
    "        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "        \n",
    "        # Inference with YOLOv5\n",
    "        results = model(frame_rgb)\n",
    "        \n",
    "        # Draw bounding boxes on the frame\n",
    "        annotated_frame = results.render()[0]\n",
    "        \n",
    "        # Convert the frame back to BGR\n",
    "        annotated_frame_bgr = cv2.cvtColor(annotated_frame, cv2.COLOR_RGB2BGR)\n",
    "        \n",
    "        # Write the annotated frame to the output video\n",
    "        out.write(annotated_frame_bgr)\n",
    "    else:\n",
    "        break\n",
    "\n",
    "# Release the video capture object and close the output file\n",
    "cap.release()\n",
    "out.release()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "OpenCV-master-py3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}