Main app.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c60c3f3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "import cv2\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tkinter import *\n",
    "from tkinter.ttk import *\n",
    "import tkinter as tk\n",
    "import json\n",
    "import collections\n",
    "import time\n",
    "CATEGORIES = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H\", \"I\",\"K\", \"L\", \"M\", \"N\", \"O\", \"P\", \"Q\", \"R\", \"S\", \"T\", \"U\", \"V\", \"W\", \"X\", \"Y\", \"?\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bccd5198",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(\"modelfinalpls.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d5461948",
   "metadata": {},
   "outputs": [],
   "source": [
    "def makedir(directory):\n",
    "    # If folder doesn't exist create it\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "        return None\n",
    "    else:\n",
    "         pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4c707be5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cr8data():\n",
    "\n",
    "    cap = cv2.VideoCapture(0)\n",
    "    SIZE_IMG = 200\n",
    "    i=0\n",
    "    image_count = 0\n",
    "    gesture_num = 0\n",
    "    gest_dir = ' '\n",
    "    \n",
    "    # Delete folder and its contents if it already exists.\n",
    "    if os.path.exists('handgestures'):\n",
    "        shutil.rmtree('handgestures')\n",
    "\n",
    "    while True:\n",
    "        while i < 4:\n",
    "\n",
    "            ret, frame = cap.read()\n",
    "            frame = cv2.flip(frame, 1)\n",
    "\n",
    "            #define region of interest\n",
    "            roi = frame[100:400, 320:620]\n",
    "            roi = cv2.resize(roi, (SIZE_IMG, SIZE_IMG), interpolation = cv2.INTER_AREA)\n",
    "            copy = frame.copy()\n",
    "            cv2.rectangle(copy, (320, 100), (620, 400), (255,0,0), 5)\n",
    "\n",
    "            if gesture_num > 23:\n",
    "                break\n",
    "            if i == 0:\n",
    "                image_count = 0\n",
    "                cv2.putText(copy, \"Hit enter to record gesture \" + str(CATEGORIES[gesture_num]), (100 , 40), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 1)\n",
    "                cv2.putText(copy, \"Hit ESC to exit \", (40 , 65), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)\n",
    "                if cv2.waitKey(1) == 27: #ESC\n",
    "                    break\n",
    "\n",
    "            if i == 1:\n",
    "                image_count+=1\n",
    "                if image_count == 3000:\n",
    "                    image_count = 0\n",
    "                    i+=1                \n",
    "                cv2.putText(copy, \"Recording gesture \"+ str(CATEGORIES[gesture_num]) + \" - Train(3000)\", (20 , 40), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 1)\n",
    "                cv2.putText(copy, str(image_count), (450 , 450), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 1)\n",
    "                gest_dir = ('./handgestures/train/{}/'.format(CATEGORIES[gesture_num]))\n",
    "                makedir(gest_dir)\n",
    "                cv2.imwrite(gest_dir + (CATEGORIES[gesture_num]+str(image_count+1)) + \".jpg\", roi)\n",
    "\n",
    "            if i == 2:\n",
    "                image_count+=1\n",
    "                if image_count == 1000:\n",
    "                    image_count = 0\n",
    "                    i+=1\n",
    "                cv2.putText(copy, \"Recording gesture \"+ str(CATEGORIES[gesture_num]) + \" - Test (1000)\", (20 , 40), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 1)\n",
    "                cv2.putText(copy, str(image_count), (450 , 450), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 1)\n",
    "                gest_dir = ('./handgestures/test/{}/'.format(CATEGORIES[gesture_num]))\n",
    "                makedir(gest_dir)\n",
    "                cv2.imwrite(gest_dir + (CATEGORIES[gesture_num]+str(image_count+1)) + \".jpg\", roi)\n",
    "            if i == 3:\n",
    "                gesture_num = gesture_num + 1\n",
    "                i = 0\n",
    "            cv2.imshow('Dataset', copy)\n",
    "            if cv2.waitKey(1) == 13: #Enter\n",
    "                image_count = 0\n",
    "                i+=1\n",
    "        break\n",
    "\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "25c2b5de",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(temp):\n",
    "    IMG_SIZE = 50\n",
    "    cap = cv2.VideoCapture(0)\n",
    "    while True:\n",
    "        \n",
    "        ret, frame1 = cap.read()\n",
    "        \n",
    "        ##############################\n",
    "        frame=cv2.flip(frame1, 1)\n",
    "        \n",
    "        #define region of interest\n",
    "        roi = frame[100:400, 320:620]\n",
    "        \n",
    "        roi = cv2.resize(roi, (IMG_SIZE, IMG_SIZE))\n",
    "\n",
    "        copy = frame.copy()\n",
    "        cv2.rectangle(copy, (320, 100), (620, 400), (255,0,0), 5)\n",
    "        \n",
    "        roi = roi.reshape(-1,IMG_SIZE,IMG_SIZE,3) \n",
    "        \n",
    "        prediction1 = model.predict(roi)\n",
    "        x1=int(prediction1.argmax(axis=1))\n",
    "        if x1 > 24 or x1 < 0:\n",
    "            x1 = 24        \n",
    "        \n",
    "        pls = (CATEGORIES[x1])\n",
    "        \n",
    "        cv2.putText(copy, pls, (440 , 90), cv2.FONT_HERSHEY_COMPLEX, 2, (255, 0, 0), 2)\n",
    "        \n",
    "        if temp == 0:\n",
    "            start_time = time.time()\n",
    "            Text = ' '\n",
    "            db = collections.deque('pls')\n",
    "            db.append(pls)\n",
    "            db2 = []\n",
    "            temp = 1\n",
    "        \n",
    "        elapsed = int((time.time()-start_time))\n",
    "        \n",
    "        if db[0] != db[1]:\n",
    "            start_time = time.time()\n",
    "\n",
    "        if elapsed != 3:\n",
    "            db.append(pls)\n",
    "        if len(db)>3:\n",
    "            db.popleft()\n",
    "        if elapsed == 3:\n",
    "            db2.append(db[1])\n",
    "            db2_index = ((len(db2))-1)\n",
    "            Text = str(Text + db2[db2_index])\n",
    "            start_time = time.time()\n",
    "        cv2.putText(copy, str(elapsed), (600 , 50), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 255), 2)\n",
    "        cv2.putText(copy, Text, (-30 , 50), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 255), 2) #Text\n",
    "        \n",
    "        if cv2.waitKey(1) == 9: #TAB\n",
    "            Text = ' '\n",
    "        if cv2.waitKey(1) == 27: #ESC\n",
    "            break\n",
    "            \n",
    "        cv2.imshow('Prediction', copy)    \n",
    "    \n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9a8d963d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    # creates a Tk() object\n",
    "    master = Tk()\n",
    "    # sets the geometry of main\n",
    "    # root window\n",
    "    master.geometry(\"350x260\")\n",
    "    master.title('Sign language gesture recognition')\n",
    "\n",
    "    def predict2():\n",
    "        predict(0)\n",
    "    \n",
    "    # a button widget which will open a new window on button click\n",
    "    btn1 = Button(master, text =\"Open prediction window\", command = (predict2))\n",
    "    btn1.pack(pady = 10)\n",
    "\n",
    "    \n",
    "    btn2 = Button(master, text =\"Make a new dataset\", command = cr8data)\n",
    "    btn2.pack(pady = 10)\n",
    "        \n",
    "    label1 = Label(master, text =\"Graduation Project Dec. 2021\")\n",
    "    label1.pack(pady = 20)\n",
    "    label2 = Label(master, text =\"By: Shady Mahmoud Ahmed Elkholy\")\n",
    "    label2.pack(pady = 0)\n",
    "    label3 = Label(master, text =\"ID: 1751710378\")\n",
    "    label3.pack(pady = 0)\n",
    "    label4 = Label(master, text =\"Supervised by: Dr. Sara Nabil\")\n",
    "    label4.pack(pady = 20)\n",
    "    \n",
    "    # mainloop, runs infinitely\n",
    "    mainloop()\n",
    "\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7b27bd5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}