Add files via upload

cnn_keras.py

@@ -0,0 +1,105 @@
+import numpy as np
+import pickle
+import cv2, os
+from glob import glob
+from keras import optimizers
+from keras.models import Sequential
+from keras.layers import Dense
+from keras.layers import Dropout
+from keras.layers import Flatten
+from keras.layers import BatchNormalization
+from keras.layers.convolutional import Conv2D
+from keras.layers.convolutional import MaxPooling2D
+from keras.utils import np_utils
+from keras.callbacks import ModelCheckpoint
+from keras import backend as K
+import matplotlib.pyplot as plt
+K.common.set_image_dim_ordering('tf')
+
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+
+def get_image_size():
+	img = cv2.imread('gestures/1/100.jpg', 0)
+	return img.shape
+
+
+def get_num_of_classes():
+	return len(glob('gestures/*'))
+
+
+image_x, image_y = get_image_size()
+
+
+# hyperparameters
+INIT_LR = 3e-3
+EPOCH = 50
+BS = 250
+
+
+def cnn_model():
+	num_of_classes = get_num_of_classes()
+	model = Sequential()
+	model.add(Conv2D(16, (2, 2), input_shape=(image_x, image_y, 1), activation='relu'))
+	model.add(BatchNormalization())
+	model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
+	model.add(Conv2D(32, (3, 3), activation='relu'))
+	model.add(MaxPooling2D(pool_size=(3, 3), strides=(3, 3), padding='same'))
+	model.add(Conv2D(64, (5, 5), activation='relu'))
+	model.add(MaxPooling2D(pool_size=(5, 5), strides=(5, 5), padding='same'))
+	model.add(Flatten())
+	model.add(Dense(128, activation='relu'))
+	model.add(Dropout(0.5))
+	model.add(Dense(num_of_classes, activation='softmax'))
+	sgd = optimizers.SGD(lr=INIT_LR)
+	model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
+	filepath = "cnn_model_keras2.h5"
+	checkpoint1 = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
+	callbacks_list = [checkpoint1]
+	from keras.utils import plot_model
+	plot_model(model, to_file='model.png', show_shapes=True)
+	return model, callbacks_list
+
+
+def train():
+	with open("train_images", "rb") as f:
+		train_images = np.array(pickle.load(f))
+	with open("train_labels", "rb") as f:
+		train_labels = np.array(pickle.load(f), dtype=np.int32)
+
+	with open("val_images", "rb") as f:
+		val_images = np.array(pickle.load(f))
+	with open("val_labels", "rb") as f:
+		val_labels = np.array(pickle.load(f), dtype=np.int32)
+
+	train_images = np.reshape(train_images, (train_images.shape[0], image_x, image_y, 1))
+	val_images = np.reshape(val_images, (val_images.shape[0], image_x, image_y, 1))
+	train_labels = np_utils.to_categorical(train_labels)
+	val_labels = np_utils.to_categorical(val_labels)
+
+	print(val_labels.shape)
+
+	model, callbacks_list = cnn_model()
+	model.summary()
+	m = model.fit(train_images, train_labels, validation_data=(val_images, val_labels), epochs=EPOCH, batch_size=BS, callbacks=callbacks_list)
+	scores = model.evaluate(val_images, val_labels, verbose=0)
+	print("CNN Error: %.2f%%" % (100-scores[1]*100))
+	model.save('cnn_model_keras2.h5')
+
+	# plot the training loss and accuracy
+	n = EPOCH
+	plt.style.use("ggplot")
+	plt.figure()
+	plt.plot(np.arange(0, n), m.history["loss"], label="train_loss")
+	plt.plot(np.arange(0, n), m.history["val_loss"], label="val_loss")
+	plt.plot(np.arange(0, n), m.history["accuracy"], label="train_accuracy")
+	plt.plot(np.arange(0, n), m.history["val_accuracy"], label="val_accuracy")
+	plt.title("Training Loss and Accuracy")
+	plt.xlabel("Epoch #")
+	plt.ylabel("Loss/Accuracy")
+	plt.legend(loc="upper right")
+	plt.savefig("plot.png")
+
+
+train()
+K.clear_session()

create_gestures.py

@@ -0,0 +1,116 @@
+import cv2
+import numpy as np
+import pickle, os, sqlite3, random
+from PIL import Image
+
+image_x, image_y = 50, 50
+
+
+def get_hand_hist():
+    with open("hist", "rb") as f:
+        hist = pickle.load(f)
+    return hist
+
+
+def init_create_folder_database():
+    # create the folder and database if not exist
+    if not os.path.exists("gestures"):
+        os.mkdir("gestures")
+    if not os.path.exists("gesture_db.db"):
+        conn = sqlite3.connect("gesture_db.db")
+        create_table_cmd = "CREATE TABLE gesture ( g_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, g_name TEXT NOT NULL )"
+        conn.execute(create_table_cmd)
+        conn.commit()
+
+
+def create_folder(folder_name):
+    if not os.path.exists(folder_name):
+        os.mkdir(folder_name)
+
+
+def store_in_db(g_id, g_name):
+    conn = sqlite3.connect("gesture_db.db")
+    cmd = "INSERT INTO gesture (g_id, g_name) VALUES (%s, \'%s\')" % (g_id, g_name)
+    try:
+        conn.execute(cmd)
+    except sqlite3.IntegrityError:
+        choice = input("g_id already exists. Want to change the record? (y/n): ")
+        if choice.lower() == 'y':
+            cmd = "UPDATE gesture SET g_name = \'%s\' WHERE g_id = %s" % (g_name, g_id)
+            conn.execute(cmd)
+        else:
+            print("Doing nothing...")
+            return
+    conn.commit()
+
+
+def store_images(g_id):
+    total_pics = 1200
+    hist = get_hand_hist()
+    cam = cv2.VideoCapture(0)
+    if not cam.read()[0]:
+        cam = cv2.VideoCapture(0)
+    x, y, w, h = 300, 100, 300, 300
+
+    create_folder("gestures/" + str(g_id))
+    pic_no = 0
+    flag_start_capturing = False
+    frames = 0
+
+    while True:
+        img = cam.read()[1]
+        img = cv2.flip(img, 1)
+        imgHSV = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+        dst = cv2.calcBackProject([imgHSV], [0, 1], hist, [0, 180, 0, 256], 1)
+        disc = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (10, 10))
+        cv2.filter2D(dst, -1, disc, dst)
+        blur = cv2.GaussianBlur(dst, (11, 11), 0)
+        blur = cv2.medianBlur(blur, 15)
+        # thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[1]
+        thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 11, 4)
+        thresh = cv2.merge((thresh, thresh, thresh))
+        thresh = cv2.cvtColor(thresh, cv2.COLOR_BGR2GRAY)
+        thresh = thresh[y:y + h, x:x + w]
+        contours = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)[1]
+
+        if len(contours) > 0:
+            contour = max(contours, key=cv2.contourArea)
+            if cv2.contourArea(contour) > 1000 and frames > 50:
+                save_img = thresh
+                pic_no += 1
+                rand = random.randint(0, 10)
+                if rand % 2 == 0:
+                    save_img = cv2.flip(save_img, 1)
+                cv2.putText(img, "Capturing...", (30, 60), cv2.FONT_HERSHEY_TRIPLEX, 2, (127, 255, 255))
+
+                f = "gestures/temp.jpg"
+                cv2.imwrite(f, save_img)
+                im = Image.open(f)
+                im = im.resize((50, 50), Image.ANTIALIAS)
+                im.save(f, 'JPEG', quality=90)
+                sim = cv2.imread(f)
+                cv2.imwrite("gestures/" + str(g_id) + "/" + str(pic_no) + ".jpg", sim)
+                os.remove(f)
+
+        cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2)
+        cv2.putText(img, str(pic_no), (30, 400), cv2.FONT_HERSHEY_TRIPLEX, 1.5, (127, 127, 255))
+        cv2.imshow("Capturing gesture", img)
+        cv2.imshow("thresh", thresh)
+        keypress = cv2.waitKey(1)
+        if keypress == ord('c'):
+            if not flag_start_capturing:
+                flag_start_capturing = True
+            else:
+                flag_start_capturing = False
+                frames = 0
+        if flag_start_capturing:
+            frames += 1
+        if pic_no == total_pics:
+            break
+
+
+init_create_folder_database()
+g_id = input("Enter gesture no.: ")
+g_name = input("Enter gesture name/text: ")
+store_in_db(g_id, g_name)
+store_images(g_id)

display_all_gestures.py

@@ -0,0 +1,44 @@
+import cv2, os, random
+import numpy as np
+
+
+def get_image_size():
+	img = cv2.imread('gestures/0/100.jpg', 0)
+	return img.shape
+
+
+gestures = os.listdir('gestures/')
+gestures.sort(key = int)
+begin_index = 0
+end_index = 5
+image_x, image_y = get_image_size()
+
+if len(gestures)%5 != 0:
+	rows = int(len(gestures)/5)+1
+else:
+	rows = int(len(gestures)/5)
+
+full_img = None
+for i in range(rows):
+	col_img = None
+	for j in range(begin_index, end_index):
+		img_path = "gestures/%s/%d.jpg" % (j, random.randint(1, 1200))
+		img = cv2.imread(img_path, 0)
+		if np.any(img is None):
+			img = np.zeros((image_y, image_x), dtype = np.uint8)
+		if np.any(col_img is None):
+			col_img = img
+		else:
+			col_img = np.hstack((col_img, img))
+
+	begin_index += 5
+	end_index += 5
+	if np.any(full_img is None):
+		full_img = col_img
+	else:
+		full_img = np.vstack((full_img, col_img))
+
+
+cv2.imshow("gestures", full_img)
+cv2.imwrite('full_img.jpg', full_img)
+cv2.waitKey(0)

flip_images.py

@@ -0,0 +1,17 @@
+import cv2, os
+
+def flip_images():
+	gest_folder = "gestures"
+	images_labels = []
+	images = []
+	labels = []
+	for g_id in os.listdir(gest_folder):
+		for i in range(1200):
+			path = gest_folder+"/"+g_id+"/"+str(i+1)+".jpg"
+			new_path = gest_folder+"/"+g_id+"/"+str(i+1+1200)+".jpg"
+			print(path)
+			img = cv2.imread(path, 0)
+			img = cv2.flip(img, 1)
+			cv2.imwrite(new_path, img)
+
+flip_images()