proj3_faceswap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Original & License: https://github.com/chewbacca89/OpenCV-with-Python/blob/master/Lecture%207.3%20-%20Mini%20Project%20%23%207%20-%20Live%20Face%20Swaps.ipynb\n",
    "\n",
    "Original original: http://matthewearl.github.io/2015/07/28/switching-eds-with-python/\n",
    "\n",
    "DLIB face predictor (too big for github): http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright (c) 2015 Matthew Earl\n",
    "# \n",
    "# Permission is hereby granted, free of charge, to any person obtaining a copy\n",
    "# of this software and associated documentation files (the \"Software\"), to deal\n",
    "# in the Software without restriction, including without limitation the rights\n",
    "# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
    "# copies of the Software, and to permit persons to whom the Software is\n",
    "# furnished to do so, subject to the following conditions:\n",
    "# \n",
    "#     The above copyright notice and this permission notice shall be included\n",
    "#     in all copies or substantial portions of the Software.\n",
    "# \n",
    "#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS\n",
    "#     OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\n",
    "#     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN\n",
    "#     NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,\n",
    "#     DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\n",
    "#     OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE\n",
    "#     USE OR OTHER DEALINGS IN THE SOFTWARE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import dlib\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "#Classifier constants\n",
    "RIGHT_EYE_POINTS = range(36, 42)\n",
    "LEFT_EYE_POINTS = range(42, 48)\n",
    "\n",
    "LEFT_BROW_POINTS = range(22, 27)\n",
    "RIGHT_BROW_POINTS = range(17, 22)\n",
    "NOSE_POINTS = range(27, 35)\n",
    "MOUTH_POINTS = range(48, 61)\n",
    "\n",
    "POINTS = [RIGHT_EYE_POINTS, LEFT_EYE_POINTS, LEFT_BROW_POINTS, RIGHT_BROW_POINTS, NOSE_POINTS, MOUTH_POINTS]\n",
    "FLAT_POINTS = range(68)\n",
    "#alternatively flatten the other list with [item for sublist in POINTS for item in sublist]\n",
    "\n",
    "#paths of helper files\n",
    "PRED_PATH = \"shape_predictor_68_face_landmarks.dat\"\n",
    "HAAR_PATH = \"haarcascade_frontalface_default.xml\"\n",
    "IMG_PATH = \"house.jpg\" #trump.jpg\n",
    "\n",
    "#constants for later use\n",
    "SCALE_FACTOR = 0.75 #smaller = faster but blurrier\n",
    "FEATHER_AMOUNT = int(25*SCALE_FACTOR)//2*2+1 #adapt to scaling-factor. assuming 640x480\n",
    "COLOUR_CORRECT_BLUR_FRAC = 0.6 #ratio of overlay of blur\n",
    "\n",
    "#configure if get_landmarks should use the dlib-detector or the cascade classifier\n",
    "DLIB_ON = True\n",
    "USE_COLOR_CORRECTION = True\n",
    "USE_MASK = True\n",
    "\n",
    "#initializations\n",
    "cascade = cv2.CascadeClassifier(HAAR_PATH)\n",
    "detector = dlib.get_frontal_face_detector()\n",
    "predictor = dlib.shape_predictor(PRED_PATH)\n",
    "\n",
    "\n",
    "def get_landmarks(im):\n",
    "    rects = detector(im, 1) if DLIB_ON else cascade.detectMultiScale(im, 1.3,5)\n",
    "    \n",
    "    if len(rects) != 1:\n",
    "        raise RuntimeError(\"Nr. of faces: \"+str(len(rects)))\n",
    "    \n",
    "    if DLIB_ON:\n",
    "        rect = rects[0]\n",
    "    else:\n",
    "        x,y,w,h = rects[0]\n",
    "        rect = dlib.rectangle(int(x),int(y),int(x+w),int(y+h)) \n",
    "    return np.matrix([[p.x, p.y] for p in predictor(im, rect).parts()])\n",
    "\n",
    "\n",
    "def get_face_mask(im, landmarks):\n",
    "    im = np.zeros(im.shape[:2], dtype=np.float64)\n",
    "\n",
    "    for group in POINTS:\n",
    "        points = cv2.convexHull(landmarks[group])\n",
    "        cv2.fillConvexPoly(im, points, color=1)\n",
    "\n",
    "    im = np.array([im, im, im]).transpose((1, 2, 0))\n",
    "\n",
    "    im = (cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0) * 1.0\n",
    "    im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)\n",
    "\n",
    "    return im\n",
    "    \n",
    "    \n",
    "def transformation_from_points(points1, points2):\n",
    "    \"\"\" Solve the procrustes problem by subtracting centroids, scaling by the\n",
    "    standard deviation, and then using the SVD to calculate the rotation. See\n",
    "    the following for more details:\n",
    "    https://en.wikipedia.org/wiki/Orthogonal_Procrustes_problem \"\"\"\n",
    "    \n",
    "    c1 = np.mean(points1, axis=0)\n",
    "    c2 = np.mean(points2, axis=0)\n",
    "    s1 = np.std(points1)\n",
    "    s2 = np.std(points2)\n",
    "\n",
    "    points1 = (np.array(points1)-c1)/s1\n",
    "    points2 = (np.array(points2)-c2)/s2\n",
    "    \n",
    "    U, S, Vt = np.linalg.svd(points1.T * points2)\n",
    "\n",
    "    # The R we seek is in fact the transpose of the one given by U * Vt. This\n",
    "    # is because the above formulation assumes the matrix goes on the right\n",
    "    # (with row vectors) where as our solution requires the matrix to be on the\n",
    "    # left (with column vectors).\n",
    "    R = (U * Vt).T\n",
    "\n",
    "    return np.vstack([  np.hstack(((s2 / s1) * R,\n",
    "                        c2.T - (s2 / s1) * R * c1.T)),\n",
    "                        np.matrix([0., 0., 1.])])\n",
    "\n",
    "\n",
    "def warp_im(im, M, dshape):\n",
    "    output_im = np.zeros(dshape, dtype=im.dtype)\n",
    "    cv2.warpAffine(im,\n",
    "                   M[:2],\n",
    "                   (dshape[1], dshape[0]),\n",
    "                   dst=output_im,\n",
    "                   borderMode=cv2.BORDER_TRANSPARENT,\n",
    "                   flags=cv2.WARP_INVERSE_MAP)\n",
    "    return output_im\n",
    "\n",
    "\n",
    "def correct_colours(im1, im2, landmarks1):\n",
    "    blur_amount = COLOUR_CORRECT_BLUR_FRAC * np.linalg.norm(\n",
    "                              np.mean(landmarks1[LEFT_EYE_POINTS], axis=0) -\n",
    "                              np.mean(landmarks1[RIGHT_EYE_POINTS], axis=0))\n",
    "    blur_amount = int(blur_amount)//2*2+1\n",
    "    im1_blur = cv2.GaussianBlur(im1, (blur_amount, blur_amount), 0)\n",
    "    im2_blur = cv2.GaussianBlur(im2, (blur_amount, blur_amount), 0)\n",
    "\n",
    "    # Avoid divide-by-zero errors.\n",
    "    im2_blur += (128 * (im2_blur <= 1.0)).astype(im2_blur.dtype)\n",
    "\n",
    "    return (im2.astype(np.float64) * im1_blur.astype(np.float64) /\n",
    "                                                im2_blur.astype(np.float64))\n",
    "\n",
    "\n",
    "def face_swap(im1, im2, landmarks2, mask2, color_correct=True, masking=True):\n",
    "    \"\"\"Takes precomputed values for the image to insert as parameters\"\"\"\n",
    "    #downscale webcam for performance\n",
    "    im1 = cv2.resize(im1, None, fx=SCALE_FACTOR, fy=SCALE_FACTOR, interpolation=cv2.INTER_LINEAR)\n",
    "    \n",
    "    #get landmarks for webcam\n",
    "    landmarks1 = get_landmarks(im1)\n",
    "    \n",
    "    #calculate the transformation matrix that shifts + stretches the overlay-image\n",
    "    M = transformation_from_points(landmarks1[FLAT_POINTS], landmarks2[FLAT_POINTS])\n",
    "    warped_mask = warp_im(mask2, M, im1.shape)\n",
    "    warped_im2 = warp_im(im2, M, im1.shape)\n",
    "    \n",
    "    combined_mask = np.max([get_face_mask(im1, landmarks1), warped_mask],axis=0)\n",
    "    \n",
    "    if color_correct:\n",
    "        warped_im2 = correct_colours(im1, warped_im2, landmarks1)\n",
    "\n",
    "    if masking:\n",
    "        output_im = im1 * (1.0 - combined_mask) + warped_im2 * combined_mask\n",
    "    else:\n",
    "        output_im = warped_im2\n",
    "    \n",
    "    #convert image to displayable format\n",
    "    _, im_arr = cv2.imencode('.jpg', output_im)\n",
    "    image = cv2.imdecode(im_arr, cv2.IMREAD_COLOR)    \n",
    "    frame = cv2.resize(image,None,fx=1.0/SCALE_FACTOR, fy=1.0/SCALE_FACTOR, interpolation = cv2.INTER_LINEAR)\n",
    "    \n",
    "    return frame\n",
    "\n",
    "\n",
    "#actual script\n",
    "if __name__ == \"__main__\":\n",
    "    cap = cv2.VideoCapture(0)\n",
    "\n",
    "    #precompute data for 2nd image once\n",
    "    im2 = cv2.imread(IMG_PATH, cv2.IMREAD_COLOR)\n",
    "    im2landmarks = get_landmarks(im2)\n",
    "    mask2 = get_face_mask(im2, im2landmarks)\n",
    "\n",
    "    while True:\n",
    "        ret, frame = cap.read()\n",
    "        frame = cv2.flip(frame,1)\n",
    "        try:\n",
    "            frame = face_swap(frame, im2, im2landmarks, mask2, USE_COLOR_CORRECTION, USE_MASK)\n",
    "        except RuntimeError as e: #Print error into image\n",
    "            frame = cv2.putText(frame, str(e), (5, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255),1,cv2.LINE_AA)\n",
    "\n",
    "        cv2.imshow(\"Face\", frame)\n",
    "        if cv2.waitKey(1) == 13:\n",
    "            break\n",
    "\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}