diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ae2c865 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM debian:stretch + +# install debian packages +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -qq \ + && apt-get install --no-install-recommends -y \ + # install essentials + build-essential \ + # install python 3 + python3.5 \ + python3-dev \ + python3-pip \ + python3-wheel \ + # Boost for dlib + cmake \ + libboost-all-dev \ + # requirements for keras + python3-h5py \ + python3-yaml \ + python3-pydot \ + python3-setuptools \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY ./requirements.txt . +RUN pip3 --no-cache-dir install -r ./requirements.txt + +WORKDIR /srv/ diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..2651ff4 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,140 @@ +# Prerequisites +Machine learning essentially involves a ton of trial and error. You're letting a program try millions of different settings to land on an algorithm that sort of does what you want it to do. This process is really really slow unless you have the hardware required to speed this up. + +The type of computations that the process does are well suited for graphics cards, rather than regular processors. **It is pretty much required that you run the training process on a desktop or server capable GPU.** Running this on your CPU means it can take weeks to train your model, compared to several hours on a GPU. + +## Hardware Requirements + +**TL;DR: you need at least one of the following:** + +- **A powerful CPU** + - Laptop CPUs can often run the software, but will not be fast enough to train at reasonable speeds +- **A powerful GPU** + - Currently only Nvidia GPUs are supported. AMD graphics cards are not supported. + This is not something that we have control over. It is a requirement of the Tensorflow library. + - The GPU needs to support at least CUDA Compute Capability 3.0 or higher. + To see which version your GPU supports, consult this list: https://developer.nvidia.com/cuda-gpus + Desktop cards later than the 7xx series are most likely supported. +- **A lot of patience** + +## Supported operating systems: + +- **Windows 10** + Windows 7 and 8 might work. Your milage may vary +- **Linux** + Most Ubuntu/Debian or CentOS based Linux distributions will work. +- **macOS** + GPU support on macOS is limited due to lack of drivers/libraries from Nvidia. + +Alternatively there is a docker image that is based on Debian. + + +# Important before you proceed + +**In its current iteration, the project relies heavily on the use of the command line. If you are unfamiliar with command line tools, you should not attempt any of the steps described in this guide.** Wait instead for this tool to become usable, or start learning more about working with the command line. This guide assumes you have intermediate knowledge of the command line. + +The developers are also not responsible for any damage you might cause to your own computer. + +# Installation Instructions + +## Installing dependencies + +### Python 3.6 + +Note that you will need the 64bit version of Python, especially to setup the GPU version! + +#### Windows + +Download the latest version of Python 3 from Python.org: https://www.python.org/downloads/release/python-364/ + +#### macOS + +By default, macOS comes with Python 2.7. For best usage, need Python 3.6. The easiest way to do so is to install it through `Homebrew`. If you are not familiar with `homebrew`, read more about it here: https://brew.sh/ + +To install Python 3.6: + +``` +brew install python3 +``` + +#### Linux + +You know how this works, don't you? + +### Virtualenv + +Install virtualenv next. Virtualenv helps us make a containing environment for our project. This means that any python packages we install for this project will be compartmentalized to this specific environment. We'll install virtualenv with `pip` which is Python's package/dependency manager. + +```pip install virtualenv``` + +or + +```pip3 install virtualenv``` + +Alternative, if your Linux distribution provides its own virtualenv through apt or yum, you can use that as well. + +#### Windows specific: + +`virtualenvwrapper-win` is a package that makes virtualenvs easier to manage on Windows. + +```pip install virtualenvwrapper-win``` + + +## Getting the faceswap code + +Simply download the code from http://github.com/deepfakes/faceswap/ - For development it is recommended to use git instead of downloading the code and extracting it. + +For now, extract the code to a directory where you're comfortable working with it. Navigate to it with the command line. For our example we will use `~/faceswap/` as our project directory. + +## Setting up our virtualenv + +### First steps + +We will now initialize our virtualenv: + +``` +virtualenv faceswap_env/ +``` + +On Windows you can use: + +``` +mkvirtualenv faceswap +setprojectdir . +``` + +This will create a folder with python, pip, and setuptools all ready to go in its own little environment. It will also activate the Virtual Environment which is indicated with the (faceswap) on the left side of the prompt. Anything we install now will be specific to this project. And available to the projects we connect to this environment. + +Let's say you’re content with the work you’ve contributed to this project and you want to move onto something else in the command line. Simply type `deactivate` to deactivate your environment. + +To reactive your environment on Windows, you can use `workon faceswap`. On Mac and Linux, you can use `source ./faceswap_env/bin/activate`. Note that the Mac/Linux command is relative to the project and virtualenv directory. + +### Setting up for our project + +With your virtualenv activated, install the dependencies from the requirements files. Like so: + +```bash +pip install -r requirements.txt +``` + +If you want to use your GPU instead of your CPU, substitute `requirements.txt` with `requirements-gpu.txt`: + +```bash +pip install -r requirements-gpu.txt +``` + +Should you choose the GPU version, Tensorflow might ask you to install the CUDA Toolkit and the cuDNN libraries. Instructions on installing those can be found on Nvidia's website. + +Once all these requirements are installed, you can attempt to run the faceswap tools. Use the `-h` or `--help` options for a list of options. + +```bash +python faceswap.py -h +``` + +Proceed to [../blob/master/USAGE.md](USAGE.md) + +## Notes + +This guide is far from complete. Functionality may change over time, and new dependencies are added and removed as time goes on. + +If you are experiencing issues, please raise them in the [faceswap-playground](https://github.com/deepfakes/faceswap-playground) repository instead of the main repo. diff --git a/README.md b/README.md index 1f6e211..a611d72 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,113 @@ -# deepfakes -This is the code for "DeepFakes" by Siraj Raval on Youtube + +**Notice:** This repository is not operated or maintained by [/u/deepfakes](https://www.reddit.com/user/deepfakes/). Please read the explanation below for details. + +--- + +# deepfakes_faceswap + +Faceswap is a tool that utilizes deep learning to recognize and swap faces in pictures and videos. + +## Overview +The project has multiple entry points. You will have to: + - Gather photos (or use the one provided in the training data provided below) + - **Extract** faces from your raw photos + - **Train** a model on your photos (or use the one provided in the training data provided below) + - **Convert** your sources with the model + +### Extract +From your setup folder, run `python faceswap.py extract`. This will take photos from `src` folder and extract faces into `extract` folder. + +### Train +From your setup folder, run `python faceswap.py train`. This will take photos from two folders containing pictures of both faces and train a model that will be saved inside the `models` folder. + +### Convert +From your setup folder, run `python faceswap.py convert`. This will take photos from `original` folder and apply new faces into `modified` folder. + +#### General notes: +- All of the scripts mentioned have `-h`/`--help` options with a arguments that they will accept. You're smart, you can figure out how this works, right?! + +Note: there is no conversion for video yet. You can use MJPG to convert video into photos, process images, and convert images back to video + +## Training Data +**Whole project with training images and trained model (~300MB):** +https://anonfile.com/p7w3m0d5be/face-swap.zip or [click here to download](https://anonfile.com/p7w3m0d5be/face-swap.zip) + +## How To setup and run the project + +### Setup + +Clone the repo and setup you environment. There is a Dockerfile that should kickstart you. Otherwise you can setup things manually, see in the Dockerfiles for dependencies. + +Check out [../blob/master/INSTALL.md](INSTALL.md) and [../blob/master/USAGE.md](USAGE.md) for basic information on how to configure virtualenv and use the program. + +You also need a modern GPU with CUDA support for best performance + +**Some tips:** + +Reusing existing models will train much faster than starting from nothing. +If there is not enough training data, start with someone who looks similar, then switch the data. + +#### Docker +If you prefer using Docker, You can start the project with: + - Build: `docker build -t deepfakes .` + - Run: `docker run --rm --name deepfakes -v [src_folder]:/srv -it deepfakes bash` . `bash` can be replaced by your command line +Note that the Dockerfile does not have all good requirments, so it will fail on some python 3 commands. +Also note that it does not have a GUI output, so the train.py will fail on showing image. You can comment this, or save it as a file. + +## How to contribute + +### For people interested in the generative models + - Go to the 'faceswap-model' to discuss/suggest/commit alternatives to the current algorithm. + +### For devs + - Read this README entirely + - Fork the repo + - Download the data with the link provided below + - Play with it + - Check issues with the 'dev' tag + - For devs more interested in computer vision and openCV, look at issues with the 'opencv' tag. Also feel free to add your own alternatives/improvments + +### For non-dev advanced users + - Read this README entirely + - Clone the repo + - Download the data with the link provided below + - Play with it + - Check issues with the 'advuser' tag + - Also go to the 'faceswap-playground' repo and help others. + +### For end-users + - Get the code here and play with it if you can + - You can also go to the 'faceswap-playground' repo and help or get help from others. + - Be patient. This is relatively new technology for developers as well. Much effort is already being put into making this program easy to use for the average user. It just takes time! + - **Notice** Any issue related to running the code has to be open in the 'faceswap-playground' project! + +### For haters +Sorry no time for that + +# About github.com/deepfakes + +## What is this repo? +It is a community repository for active users. + +## Why this repo? +The joshua-wu repo seems not active. Simple bugs like missing _http://_ in front of url has not been solved since days. + +## Why is it named 'deepfakes' if it is not /u/deepfakes? + 1. Because a typosquat would have happened sooner or later as project grows + 2. Because all glory go to /u/deepfakes + 3. Because it will better federate contributors and users + +## What if /u/deepfakes feels bad about that? +This is a friendly typosquat, and it is fully dedicated to the project. If /u/deepfakes wants to take over this repo/user and drive the project, he is welcomed to do so (Raise an issue, and he will be contacted on Reddit). Please do not send /u/deepfakes messages for help with the code you find here. + +# About machine learning + +## How does a computer know how to recognise/shape a faces? How does machine learning work? What is a neural network? + +It's complicated. Here's a good video that makes the process understandable: +[![How Machines Learn](https://img.youtube.com/vi/R9OHn5ZF4Uo/0.jpg)](https://www.youtube.com/watch?v=R9OHn5ZF4Uo) + +Here's a slightly more in depth video that tries to explain the basic functioning of a neural network: +[![How Machines Learn](https://img.youtube.com/vi/aircAruvnKk/0.jpg)](https://www.youtube.com/watch?v=aircAruvnKk) + +tl;dr: training data + trial and error diff --git a/USAGE.md b/USAGE.md new file mode 100644 index 0000000..994ca9d --- /dev/null +++ b/USAGE.md @@ -0,0 +1,78 @@ +**Before attempting any of this, please make sure you have read, understood and completed [the installation instructions](../master/INSTALL.md). If you are experiencing issues, please raise them in the [faceswap-playground](https://github.com/deepfakes/faceswap-playground) repository instead of the main repo.** + +# Workflow + +So, you want to swap faces in pictures and videos? Well hold up, because first you gotta understand what this collection of scripts will do, how it does it and what it can't currently do. + +The basic operation of this script is simple. It trains a machine learning model to recognize and transform two faces based on pictures. The machine learning model is our little "bot" that we're teaching to do the actual swapping and the pictures are the "training data" that we use to train it. Note that the bot is primarily processing faces. Other objects might not work. + +So here's our plan. We want to create a reality where Donald Trump lost the presidency to Nic Cage; we have his inauguration video; let's replace Trump with Cage. + +## Gather training data + +In order to accomplish this, the bot needs to learn to recognize both face A (Trump) and face B (Nic Cage). By default, the bot doesn't know what a Trump or a Nic Cage looks like. So we need to show it some pictures and let it guess which is which. So we need pictures of both of these faces first. + +A possible source is Google, DuckDuckGo or Bing image search. There are scripts to download large amounts of images. Alternatively, if you have a lot of videos of the person you're looking for (like interviews, public speeches, movies), you can convert a video to still images/frames and use those. + +Feel free to list your image sets in the [faceswap-playground](https://github.com/deepfakes/faceswap-playground), or add more methods to this file. + +So now we have a folder full of pictures of Trump and a separate folder of Nic Cage. Let's save them in our directory where we put the faceswap project. Example: `~/faceswap/photo/trump` and `~/faceswap/photo/cage` + +## Extracting our training data + +So here's a problem. We have a ton of pictures of both our subjects, but they're just pictures of them doing stuff or in an environment with other people. Their bodies are on there, they're on there with other people... It's a mess. We can only train our bot if the data we have is consistent and focusses on the subject we want to swap. This is where faceswap first comes in. + +```bash +# To convert trump: +python faceswap.py extract -i ~/faceswap/photo/trump -o ~/faceswap/data/trump +# To convert cage: +python faceswap.py extract -i ~/faceswap/photo/cage -o ~/faceswap/data/cage +``` + +We specify our photo input directory and the output folder where our training data will be saved. The script will then try its best to recognize face landmarks, crop the image to that size, and save it to the output folder. Note: this script will make grabbing test data much easier, but it is not perfect. It will (incorrectly) detect multiple faces in some photos and does not recognize if the face is the person who we want to swap. Therefore: **Always check your training data before you start training.** The training data will influence how good your model will be at swapping. + +## Training + +The training process will take the longest, especially on CPU. We specify the folders where the two faces are, and where we will save our training model. It will start hammering the training data once you run the command. I personally really like to go by the preview and quit the processing once I'm happy with the results. + +```bash +python faceswap.py train -A ~/faceswap/data/trump -B ~/faceswap/data/cage -m ~/faceswap/models/ +# or -p to show a preview +python faceswap.py train -A ~/faceswap/data/trump -B ~/faceswap/data/cage -m ~/faceswap/models/ -p +```` + +If you use the preview feature, select the preview window and press Q to save your processed data and quit gracefully. Without the preview enabled, you might have to forcefully quit by hitting Ctrl+C to cancel the command. Note that it will save the model once it's gone through about 100 iterations, which can take quite a while. So make sure you save before stopping the process. + +## SWAPPING + +Now that we're happy with our trained model, we can convert our video. How does it work? Similarly to the extraction script, actually! The conversion script basically detects a face in a picture using the same algorithm, quickly crops the image to the right size, runs our bot on this cropped image of the face it has found, and then (crudely) pastes the processed face back into the picture. + +### Testing out our bot + +Remember those initial pictures we had of Trump? Let's try swapping a face there. We will use that directory as our input directory, create a new folder where the output will be saved, and tell them which model to use. + +```bash +python faceswap.py convert -i ~/faceswap/photo/trump/ -o ~/faceswap/output/ -m ~/faceswap/models/ +``` + +It should now start swapping faces of all these pictures. + +### Preparing a video + +A video is just a series of pictures (frames). You can export a video to still frames using `ffmpeg` for example. Below is an example command to process a video to frames. + +```bash +ffmpeg -i /path/to/my/video.mp4 /path/to/output/video-frame-%d.png +``` + +If you then use the resulting directory with frames to faceswap, it will automatically go through all of those. And here's a command to stitch png frames to a single video again: + +```bash +ffmpeg -i video-frame-%04d.png -c:v libx264 -vf "fps=25,format=yuv420p" out.mp4 +``` + +## Notes + +This guide is far from complete. Functionality may change over time, and new dependencies are added and removed as time goes on. + +If you are experiencing issues, please raise them in the [faceswap-playground](https://github.com/deepfakes/faceswap-playground) repository instead of the main repo. diff --git a/faceswap.py b/faceswap.py new file mode 100644 index 0000000..306d85e --- /dev/null +++ b/faceswap.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import sys +if sys.version_info[0] < 3: + raise Exception("This program requires at least python3.2") +if sys.version_info[0] == 3 and sys.version_info[1] < 2: + raise Exception("This program requires at least python3.2") + +from lib.utils import FullHelpArgumentParser + +from scripts.extract import ExtractTrainingData +from scripts.train import TrainingProcessor +from scripts.convert import ConvertImage + +if __name__ == "__main__": + parser = FullHelpArgumentParser() + subparser = parser.add_subparsers() + extract = ExtractTrainingData( + subparser, "extract", "Extract the faces from a pictures.") + train = TrainingProcessor( + subparser, "train", "This command trains the model for the two faces A and B.") + convert = ConvertImage( + subparser, "convert", "Convert a source image to a new one with the face swapped.") + arguments = parser.parse_args() + arguments.func(arguments) diff --git a/lib/FaceFilter.py b/lib/FaceFilter.py new file mode 100644 index 0000000..113678a --- /dev/null +++ b/lib/FaceFilter.py @@ -0,0 +1,25 @@ +# import dlib +# import numpy as np +import face_recognition +# import face_recognition_models + +class FaceFilter(): + def __init__(self, reference_file_path, threshold = 0.6): + image = face_recognition.load_image_file(reference_file_path) + self.encoding = face_recognition.face_encodings(image)[0] # Note: we take only first face, so the reference file should only contain one face. We could also keep all faces found and filter against multiple faces + self.threshold = threshold + + def check(self, detected_face): + encodings = face_recognition.face_encodings(detected_face.image)[0] # we could use detected landmarks, but I did not manage to do so + score = face_recognition.face_distance([self.encoding], encodings) + print(score) + return score <= self.threshold + + +# # Copy/Paste (mostly) from private method in face_recognition +# face_recognition_model = face_recognition_models.face_recognition_model_location() +# face_encoder = dlib.face_recognition_model_v1(face_recognition_model) + +# def convert(detected_face): +# return np.array(face_encoder.compute_face_descriptor(detected_face.image, detected_face.landmarks, 1)) +# # end of Copy/Paste diff --git a/lib/ModelAE.py b/lib/ModelAE.py new file mode 100644 index 0000000..2dd6f90 --- /dev/null +++ b/lib/ModelAE.py @@ -0,0 +1,77 @@ +# AutoEncoder base classes + +import time +import numpy +from lib.training_data import minibatchAB, stack_images + +encoderH5 = '/encoder.h5' +decoder_AH5 = '/decoder_A.h5' +decoder_BH5 = '/decoder_B.h5' + +class ModelAE: + def __init__(self, model_dir): + + self.model_dir = model_dir + + self.encoder = self.Encoder() + self.decoder_A = self.Decoder() + self.decoder_B = self.Decoder() + + self.initModel() + + def load(self, swapped): + (face_A,face_B) = (decoder_AH5, decoder_BH5) if not swapped else (decoder_BH5, decoder_AH5) + + try: + self.encoder.load_weights(self.model_dir + encoderH5) + self.decoder_A.load_weights(self.model_dir + face_A) + self.decoder_B.load_weights(self.model_dir + face_B) + print('loaded model weights') + return True + except Exception as e: + print('Failed loading existing training data.') + print(e) + return False + + def save_weights(self): + self.encoder.save_weights(self.model_dir + encoderH5) + self.decoder_A.save_weights(self.model_dir + decoder_AH5) + self.decoder_B.save_weights(self.model_dir + decoder_BH5) + print('saved model weights') + +class TrainerAE(): + def __init__(self, model, fn_A, fn_B, batch_size=64): + self.batch_size = batch_size + self.model = model + self.images_A = minibatchAB(fn_A, self.batch_size) + self.images_B = minibatchAB(fn_B, self.batch_size) + + def train_one_step(self, iter, viewer): + epoch, warped_A, target_A = next(self.images_A) + epoch, warped_B, target_B = next(self.images_B) + + loss_A = self.model.autoencoder_A.train_on_batch(warped_A, target_A) + loss_B = self.model.autoencoder_B.train_on_batch(warped_B, target_B) + print("[{0}] [#{1:05d}] loss_A: {2:.5f}, loss_B: {3:.5f}".format(time.strftime("%H:%M:%S"), iter, loss_A, loss_B), + end='\r') + + if viewer is not None: + viewer(self.show_sample(target_A[0:14], target_B[0:14]), "training") + + def show_sample(self, test_A, test_B): + figure_A = numpy.stack([ + test_A, + self.model.autoencoder_A.predict(test_A), + self.model.autoencoder_B.predict(test_A), + ], axis=1) + figure_B = numpy.stack([ + test_B, + self.model.autoencoder_B.predict(test_B), + self.model.autoencoder_A.predict(test_B), + ], axis=1) + + figure = numpy.concatenate([figure_A, figure_B], axis=0) + figure = figure.reshape((4, 7) + figure.shape[1:]) + figure = stack_images(figure) + + return numpy.clip(figure * 255, 0, 255).astype('uint8') diff --git a/lib/PixelShuffler.py b/lib/PixelShuffler.py new file mode 100644 index 0000000..9904191 --- /dev/null +++ b/lib/PixelShuffler.py @@ -0,0 +1,88 @@ +# PixelShuffler layer for Keras +# by t-ae +# https://gist.github.com/t-ae/6e1016cc188104d123676ccef3264981 + +from keras.utils import conv_utils +from keras.engine.topology import Layer +import keras.backend as K + + +class PixelShuffler(Layer): + def __init__(self, size=(2, 2), data_format=None, **kwargs): + super(PixelShuffler, self).__init__(**kwargs) + self.data_format = conv_utils.normalize_data_format(data_format) + self.size = conv_utils.normalize_tuple(size, 2, 'size') + + def call(self, inputs): + + input_shape = K.int_shape(inputs) + if len(input_shape) != 4: + raise ValueError('Inputs should have rank ' + + str(4) + + '; Received input shape:', str(input_shape)) + + if self.data_format == 'channels_first': + batch_size, c, h, w = input_shape + if batch_size is None: + batch_size = -1 + rh, rw = self.size + oh, ow = h * rh, w * rw + oc = c // (rh * rw) + + out = K.reshape(inputs, (batch_size, rh, rw, oc, h, w)) + out = K.permute_dimensions(out, (0, 3, 4, 1, 5, 2)) + out = K.reshape(out, (batch_size, oc, oh, ow)) + return out + + elif self.data_format == 'channels_last': + batch_size, h, w, c = input_shape + if batch_size is None: + batch_size = -1 + rh, rw = self.size + oh, ow = h * rh, w * rw + oc = c // (rh * rw) + + out = K.reshape(inputs, (batch_size, h, w, rh, rw, oc)) + out = K.permute_dimensions(out, (0, 1, 3, 2, 4, 5)) + out = K.reshape(out, (batch_size, oh, ow, oc)) + return out + + def compute_output_shape(self, input_shape): + + if len(input_shape) != 4: + raise ValueError('Inputs should have rank ' + + str(4) + + '; Received input shape:', str(input_shape)) + + if self.data_format == 'channels_first': + height = input_shape[2] * self.size[0] if input_shape[2] is not None else None + width = input_shape[3] * self.size[1] if input_shape[3] is not None else None + channels = input_shape[1] // self.size[0] // self.size[1] + + if channels * self.size[0] * self.size[1] != input_shape[1]: + raise ValueError('channels of input and size are incompatible') + + return (input_shape[0], + channels, + height, + width) + + elif self.data_format == 'channels_last': + height = input_shape[1] * self.size[0] if input_shape[1] is not None else None + width = input_shape[2] * self.size[1] if input_shape[2] is not None else None + channels = input_shape[3] // self.size[0] // self.size[1] + + if channels * self.size[0] * self.size[1] != input_shape[3]: + raise ValueError('channels of input and size are incompatible') + + return (input_shape[0], + height, + width, + channels) + + def get_config(self): + config = {'size': self.size, + 'data_format': self.data_format} + base_config = super(PixelShuffler, self).get_config() + + return dict(list(base_config.items()) + list(config.items())) diff --git a/lib/aligner.py b/lib/aligner.py new file mode 100644 index 0000000..7b85f03 --- /dev/null +++ b/lib/aligner.py @@ -0,0 +1,26 @@ +import numpy + +from lib.umeyama import umeyama + +mean_face_x = numpy.array([ +0.000213256, 0.0752622, 0.18113, 0.29077, 0.393397, 0.586856, 0.689483, 0.799124, +0.904991, 0.98004, 0.490127, 0.490127, 0.490127, 0.490127, 0.36688, 0.426036, +0.490127, 0.554217, 0.613373, 0.121737, 0.187122, 0.265825, 0.334606, 0.260918, +0.182743, 0.645647, 0.714428, 0.793132, 0.858516, 0.79751, 0.719335, 0.254149, +0.340985, 0.428858, 0.490127, 0.551395, 0.639268, 0.726104, 0.642159, 0.556721, +0.490127, 0.423532, 0.338094, 0.290379, 0.428096, 0.490127, 0.552157, 0.689874, +0.553364, 0.490127, 0.42689 ]) + +mean_face_y = numpy.array([ +0.106454, 0.038915, 0.0187482, 0.0344891, 0.0773906, 0.0773906, 0.0344891, +0.0187482, 0.038915, 0.106454, 0.203352, 0.307009, 0.409805, 0.515625, 0.587326, +0.609345, 0.628106, 0.609345, 0.587326, 0.216423, 0.178758, 0.179852, 0.231733, +0.245099, 0.244077, 0.231733, 0.179852, 0.178758, 0.216423, 0.244077, 0.245099, +0.780233, 0.745405, 0.727388, 0.742578, 0.727388, 0.745405, 0.780233, 0.864805, +0.902192, 0.909281, 0.902192, 0.864805, 0.784792, 0.778746, 0.785343, 0.778746, +0.784792, 0.824182, 0.831803, 0.824182 ]) + +landmarks_2D = numpy.stack( [ mean_face_x, mean_face_y ], axis=1 ) + +def get_align_mat(face): + return umeyama( numpy.array(face.landmarksAsXY()[17:]), landmarks_2D, True )[0:2] diff --git a/lib/cli.py b/lib/cli.py new file mode 100644 index 0000000..2166322 --- /dev/null +++ b/lib/cli.py @@ -0,0 +1,137 @@ +import argparse +import os +import time +from tqdm import tqdm + +from pathlib import Path +from lib.FaceFilter import FaceFilter +from lib.faces_detect import detect_faces +from lib.utils import get_image_paths, get_folder + +class FullPaths(argparse.Action): + """Expand user- and relative-paths""" + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, os.path.abspath( + os.path.expanduser(values))) + +class DirectoryProcessor(object): + ''' + Abstract class that processes a directory of images + and writes output to the specified folder + ''' + arguments = None + parser = None + + input_dir = None + output_dir = None + + verify_output = False + images_found = 0 + images_processed = 0 + faces_detected = 0 + + def __init__(self, subparser, command, description='default'): + self.create_parser(subparser, command, description) + self.parse_arguments(description, subparser, command) + + def process_arguments(self, arguments): + self.arguments = arguments + print("Input Directory: {}".format(self.arguments.input_dir)) + print("Output Directory: {}".format(self.arguments.output_dir)) + print('Starting, this may take a while...') + + self.output_dir = get_folder(self.arguments.output_dir) + try: + self.input_dir = get_image_paths(self.arguments.input_dir) + except: + print('Input directory not found. Please ensure it exists.') + exit(1) + + self.images_found = len(self.input_dir) + self.filter = self.load_filter() + self.process() + self.finalize() + + def read_directory(self): + for filename in tqdm(self.input_dir): + if self.arguments.verbose: + print('Processing: {}'.format(os.path.basename(filename))) + + yield filename + self.images_processed = self.images_processed + 1 + + def get_faces(self, image): + faces_count = 0 + for face in detect_faces(image): + if self.filter is not None and not self.filter.check(face): + print('Skipping not recognized face!') + continue + + yield faces_count, face + + self.faces_detected = self.faces_detected + 1 + faces_count +=1 + + if faces_count > 0 and self.arguments.verbose: + print('Note: Found more than one face in an image!') + self.verify_output = True + + def load_filter(self): + filter_file = "filter.jpg" # TODO Pass as argument + if Path(filter_file).exists(): + print('Loading reference image for filtering') + return FaceFilter(filter_file) + + # for now, we limit this class responsability to the read of files. images and faces are processed outside this class + def process(self): + # implement your image processing! + raise NotImplementedError() + + def parse_arguments(self, description, subparser, command): + self.parser.add_argument('-i', '--input-dir', + action=FullPaths, + dest="input_dir", + default="input", + help="Input directory. A directory containing the files \ + you wish to process. Defaults to 'input'") + self.parser.add_argument('-o', '--output-dir', + action=FullPaths, + dest="output_dir", + default="output", + help="Output directory. This is where the converted files will \ + be stored. Defaults to 'output'") + self.parser.add_argument('-v', '--verbose', + action="store_true", + dest="verbose", + default=False, + help="Show verbose output") + self.parser = self.add_optional_arguments(self.parser) + self.parser.set_defaults(func=self.process_arguments) + + def create_parser(self, subparser, command, description): + parser = subparser.add_parser( + command, + description=description, + epilog="Questions and feedback: \ + https://github.com/deepfakes/faceswap-playground" + ) + return parser + + def add_optional_arguments(self, parser): + # Override this for custom arguments + return parser + + def finalize(self): + print('-------------------------') + print('Images found: {}'.format(self.images_found)) + print('Images processed: {}'.format(self.images_processed)) + print('Faces detected: {}'.format(self.faces_detected)) + print('-------------------------') + + if self.verify_output: + print('Note:') + print('Multiple faces were detected in one or more pictures.') + print('Double check your results.') + print('-------------------------') + print('Done!') diff --git a/lib/faces_detect.py b/lib/faces_detect.py new file mode 100644 index 0000000..0d650bd --- /dev/null +++ b/lib/faces_detect.py @@ -0,0 +1,34 @@ +import dlib +import face_recognition +import face_recognition_models + +def detect_faces(frame): + face_locations = face_recognition.face_locations(frame) + landmarks = _raw_face_landmarks(frame, face_locations) + + for ((y, right, bottom, x), landmarks) in zip(face_locations, landmarks): + yield DetectedFace(frame[y: bottom, x: right], x, right - x, y, bottom - y, landmarks) + +# Copy/Paste (mostly) from private method in face_recognition +predictor_68_point_model = face_recognition_models.pose_predictor_model_location() +pose_predictor = dlib.shape_predictor(predictor_68_point_model) + +def _raw_face_landmarks(face_image, face_locations): + face_locations = [_css_to_rect(face_location) for face_location in face_locations] + return [pose_predictor(face_image, face_location) for face_location in face_locations] + +def _css_to_rect(css): + return dlib.rectangle(css[3], css[0], css[1], css[2]) +# end of Copy/Paste + +class DetectedFace(object): + def __init__(self, image, x, w, y, h, landmarks): + self.image = image + self.x = x + self.w = w + self.y = y + self.h = h + self.landmarks = landmarks + + def landmarksAsXY(self): + return [(p.x, p.y) for p in self.landmarks.parts()] diff --git a/lib/training_data.py b/lib/training_data.py new file mode 100644 index 0000000..f836e47 --- /dev/null +++ b/lib/training_data.py @@ -0,0 +1,105 @@ +import cv2 +import numpy +from random import shuffle + +from .utils import BackgroundGenerator +from .umeyama import umeyama + +coverage = 220 # Coverage of the face for training. Larger value will cover more features. @shaoanlu recommends 220. Original is 160 + +random_transform_args = { + 'rotation_range': 10, + 'zoom_range': 0.05, + 'shift_range': 0.05, + 'random_flip': 0.4, +} + +# GAN +# random_transform_args = { +# 'rotation_range': 20, +# 'zoom_range': 0.1, +# 'shift_range': 0.05, +# 'random_flip': 0.5, +# } +def read_image(fn, random_transform_args=random_transform_args): + image = cv2.imread(fn) / 255.0 + image = cv2.resize(image, (256,256)) + image = random_transform( image, **random_transform_args ) + warped_img, target_img = random_warp( image ) + + return warped_img, target_img + +# A generator function that yields epoch, batchsize of warped_img and batchsize of target_img +def minibatch(data, batchsize): + length = len(data) + epoch = i = 0 + shuffle(data) + while True: + size = batchsize + if i+size > length: + shuffle(data) + i = 0 + epoch+=1 + rtn = numpy.float32([read_image(data[j]) for j in range(i,i+size)]) + i+=size + yield epoch, rtn[:,0,:,:,:], rtn[:,1,:,:,:] + +def minibatchAB(images, batchsize): + batch = BackgroundGenerator(minibatch(images, batchsize), 1) + for ep1, warped_img, target_img in batch.iterator(): + yield ep1, warped_img, target_img + +def random_transform(image, rotation_range, zoom_range, shift_range, random_flip): + h, w = image.shape[0:2] + rotation = numpy.random.uniform(-rotation_range, rotation_range) + scale = numpy.random.uniform(1 - zoom_range, 1 + zoom_range) + tx = numpy.random.uniform(-shift_range, shift_range) * w + ty = numpy.random.uniform(-shift_range, shift_range) * h + mat = cv2.getRotationMatrix2D((w // 2, h // 2), rotation, scale) + mat[:, 2] += (tx, ty) + result = cv2.warpAffine( + image, mat, (w, h), borderMode=cv2.BORDER_REPLICATE) + if numpy.random.random() < random_flip: + result = result[:, ::-1] + return result + +# get pair of random warped images from aligned face image +def random_warp(image): + assert image.shape == (256, 256, 3) + range_ = numpy.linspace(128 - coverage//2, 128 + coverage//2, 5) + mapx = numpy.broadcast_to(range_, (5, 5)) + mapy = mapx.T + + mapx = mapx + numpy.random.normal(size=(5, 5), scale=5) + mapy = mapy + numpy.random.normal(size=(5, 5), scale=5) + + interp_mapx = cv2.resize(mapx, (80, 80))[8:72, 8:72].astype('float32') + interp_mapy = cv2.resize(mapy, (80, 80))[8:72, 8:72].astype('float32') + + warped_image = cv2.remap(image, interp_mapx, interp_mapy, cv2.INTER_LINEAR) + + src_points = numpy.stack([mapx.ravel(), mapy.ravel()], axis=-1) + dst_points = numpy.mgrid[0:65:16, 0:65:16].T.reshape(-1, 2) + mat = umeyama(src_points, dst_points, True)[0:2] + + target_image = cv2.warpAffine(image, mat, (64, 64)) + + return warped_image, target_image + +def get_transpose_axes(n): + if n % 2 == 0: + y_axes = list(range(1, n - 1, 2)) + x_axes = list(range(0, n - 1, 2)) + else: + y_axes = list(range(0, n - 1, 2)) + x_axes = list(range(1, n - 1, 2)) + return y_axes, x_axes, [n - 1] + +def stack_images(images): + images_shape = numpy.array(images.shape) + new_axes = get_transpose_axes(len(images_shape)) + new_shape = [numpy.prod(images_shape[x]) for x in new_axes] + return numpy.transpose( + images, + axes=numpy.concatenate(new_axes) + ).reshape(new_shape) diff --git a/lib/umeyama.py b/lib/umeyama.py new file mode 100644 index 0000000..a835484 --- /dev/null +++ b/lib/umeyama.py @@ -0,0 +1,84 @@ +## License (Modified BSD) +## Copyright (C) 2011, the scikit-image team All rights reserved. +## +## Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +## +## Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +## Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +## Neither the name of skimage nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +## THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# umeyama function from scikit-image/skimage/transform/_geometric.py + +import numpy as np + + +def umeyama(src, dst, estimate_scale): + """Estimate N-D similarity transformation with or without scaling. + Parameters + ---------- + src : (M, N) array + Source coordinates. + dst : (M, N) array + Destination coordinates. + estimate_scale : bool + Whether to estimate scaling factor. + Returns + ------- + T : (N + 1, N + 1) + The homogeneous similarity transformation matrix. The matrix contains + NaN values only if the problem is not well-conditioned. + References + ---------- + .. [1] "Least-squares estimation of transformation parameters between two + point patterns", Shinji Umeyama, PAMI 1991, DOI: 10.1109/34.88573 + """ + + num = src.shape[0] + dim = src.shape[1] + + # Compute mean of src and dst. + src_mean = src.mean(axis=0) + dst_mean = dst.mean(axis=0) + + # Subtract mean from src and dst. + src_demean = src - src_mean + dst_demean = dst - dst_mean + + # Eq. (38). + A = np.dot(dst_demean.T, src_demean) / num + + # Eq. (39). + d = np.ones((dim,), dtype=np.double) + if np.linalg.det(A) < 0: + d[dim - 1] = -1 + + T = np.eye(dim + 1, dtype=np.double) + + U, S, V = np.linalg.svd(A) + + # Eq. (40) and (43). + rank = np.linalg.matrix_rank(A) + if rank == 0: + return np.nan * T + elif rank == dim - 1: + if np.linalg.det(U) * np.linalg.det(V) > 0: + T[:dim, :dim] = np.dot(U, V) + else: + s = d[dim - 1] + d[dim - 1] = -1 + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V)) + d[dim - 1] = s + else: + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V.T)) + + if estimate_scale: + # Eq. (41) and (42). + scale = 1.0 / src_demean.var(axis=0).sum() * np.dot(S, d) + else: + scale = 1.0 + + T[:dim, dim] = dst_mean - scale * np.dot(T[:dim, :dim], src_mean.T) + T[:dim, :dim] *= scale + + return T diff --git a/lib/utils.py b/lib/utils.py new file mode 100644 index 0000000..beb5473 --- /dev/null +++ b/lib/utils.py @@ -0,0 +1,51 @@ +import argparse +import sys + +from pathlib import Path +from scandir import scandir + +image_extensions = [".jpg", ".jpeg", ".png"] + +def get_folder(path): + output_dir = Path(path) + output_dir.mkdir(parents=True, exist_ok=True) + return output_dir + +def get_image_paths(directory): + return [x.path for x in scandir(directory) if + any(map(lambda ext: x.name.lower().endswith(ext), image_extensions))] + +class FullHelpArgumentParser(argparse.ArgumentParser): + """ + Identical to the built-in argument parser, but on error + it prints full help message instead of just usage information + """ + def error(self, message): + self.print_help(sys.stderr) + args = {'prog': self.prog, 'message': message} + self.exit(2, '%(prog)s: error: %(message)s\n' % args) + +# From: https://stackoverflow.com/questions/7323664/python-generator-pre-fetch +import threading +import queue as Queue +class BackgroundGenerator(threading.Thread): + def __init__(self, generator, prefetch=1): #See below why prefetch count is flawed + threading.Thread.__init__(self) + self.queue = Queue.Queue(prefetch) + self.generator = generator + self.daemon = True + self.start() + + def run(self): + # Put until queue size is reached. Note: put blocks only if put is called while queue has already reached max size + # => this makes 2 prefetched items! One in the queue, one waiting for insertion! + for item in self.generator: + self.queue.put(item) + self.queue.put(None) + + def iterator(self): + while True: + next_item = self.queue.get() + if next_item is None: + break + yield next_item diff --git a/plugins/Convert_Adjust.py b/plugins/Convert_Adjust.py new file mode 100644 index 0000000..0760715 --- /dev/null +++ b/plugins/Convert_Adjust.py @@ -0,0 +1,64 @@ +# Based on the original https://www.reddit.com/r/deepfakes/ code sample +# Adjust code made by https://github.com/yangchen8710 + +import cv2 +import numpy +import os + +class Convert(object): + def __init__(self, encoder, smooth_mask=True, avg_color_adjust=True, **kwargs): + self.encoder = encoder + + self.use_smooth_mask = smooth_mask + self.use_avg_color_adjust = avg_color_adjust + + def patch_image( self, original, face_detected ): + #assert image.shape == (256, 256, 3) + image = cv2.resize(face_detected.image, (256, 256)) + crop = slice(48, 208) + face = image[crop, crop] + old_face = face.copy() + + face = cv2.resize(face, (64, 64)) + face = numpy.expand_dims(face, 0) + new_face = self.encoder(face / 255.0)[0] + new_face = numpy.clip(new_face * 255, 0, 255).astype(image.dtype) + new_face = cv2.resize(new_face, (160, 160)) + + if self.use_avg_color_adjust: + self.adjust_avg_color(old_face,new_face) + if self.use_smooth_mask: + self.smooth_mask(old_face,new_face) + + new_face = self.superpose(image, new_face, crop) + original[slice(face_detected.y, face_detected.y + face_detected.h), slice(face_detected.x, face_detected.x + face_detected.w)] = cv2.resize(new_face, (face_detected.w, face_detected.h)) + return original + + def adjust_avg_color(self,img_old,img_new): + w,h,c = img_new.shape + for i in range(img_new.shape[-1]): + old_avg = img_old[:, :, i].mean() + new_avg = img_new[:, :, i].mean() + diff_int = (int)(old_avg - new_avg) + for m in range(img_new.shape[0]): + for n in range(img_new.shape[1]): + temp = (img_new[m,n,i] + diff_int) + if temp < 0: + img_new[m,n,i] = 0 + elif temp > 255: + img_new[m,n,i] = 255 + else: + img_new[m,n,i] = temp + + def smooth_mask(self,img_old,img_new): + w,h,c = img_new.shape + crop = slice(0,w) + mask = numpy.zeros_like(img_new) + mask[h//15:-h//15,w//15:-w//15,:] = 255 + mask = cv2.GaussianBlur(mask,(15,15),10) + img_new[crop,crop] = mask/255*img_new + (1-mask/255)*img_old + + def superpose(self,image, new_face, crop): + new_image = image.copy() + new_image[crop, crop] = new_face + return new_image diff --git a/plugins/Convert_Masked.py b/plugins/Convert_Masked.py new file mode 100644 index 0000000..8e0c1d9 --- /dev/null +++ b/plugins/Convert_Masked.py @@ -0,0 +1,82 @@ +# Based on: https://gist.github.com/anonymous/d3815aba83a8f79779451262599b0955 found on https://www.reddit.com/r/deepfakes/ + +import cv2 +import numpy + +from lib.aligner import get_align_mat + +class Convert(): + def __init__(self, encoder, blur_size=2, seamless_clone=False, mask_type="facehullandrect", erosion_kernel_size=None, **kwargs): + self.encoder = encoder + + self.erosion_kernel = None + if erosion_kernel_size is not None: + self.erosion_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(erosion_kernel_size,erosion_kernel_size)) + + self.blur_size = blur_size + self.seamless_clone = seamless_clone + self.mask_type = mask_type.lower() # Choose in 'FaceHullAndRect','FaceHull','Rect' + + def patch_image( self, image, face_detected ): + size = 64 + image_size = image.shape[1], image.shape[0] + + mat = numpy.array(get_align_mat(face_detected)).reshape(2,3) * size + + new_face = self.get_new_face(image,mat,size) + + image_mask = self.get_image_mask( image, new_face, face_detected, mat, image_size ) + + return self.apply_new_face(image, new_face, image_mask, mat, image_size, size) + + def apply_new_face(self, image, new_face, image_mask, mat, image_size, size): + base_image = numpy.copy( image ) + new_image = numpy.copy( image ) + + cv2.warpAffine( new_face, mat, image_size, new_image, cv2.WARP_INVERSE_MAP, cv2.BORDER_TRANSPARENT ) + + outImage = None + if self.seamless_clone: + masky,maskx = cv2.transform( numpy.array([ size/2,size/2 ]).reshape(1,1,2) ,cv2.invertAffineTransform(mat) ).reshape(2).astype(int) + outimage = cv2.seamlessClone(new_image.astype(numpy.uint8),base_image.astype(numpy.uint8),(image_mask*255).astype(numpy.uint8),(masky,maskx) , cv2.NORMAL_CLONE ) + else: + foreground = cv2.multiply(image_mask, new_image.astype(float)) + background = cv2.multiply(1.0 - image_mask, base_image.astype(float)) + outimage = cv2.add(foreground, background) + + return outimage + + def get_new_face(self, image, mat, size): + face = cv2.warpAffine( image, mat, (size,size) ) + face = numpy.expand_dims( face, 0 ) + new_face = self.encoder( face / 255.0 )[0] + + return numpy.clip( new_face * 255, 0, 255 ).astype( image.dtype ) + + def get_image_mask(self, image, new_face, face_detected, mat, image_size): + + face_mask = numpy.zeros(image.shape,dtype=float) + if 'rect' in self.mask_type: + face_src = numpy.ones(new_face.shape,dtype=float) + cv2.warpAffine( face_src, mat, image_size, face_mask, cv2.WARP_INVERSE_MAP, cv2.BORDER_TRANSPARENT ) + + hull_mask = numpy.zeros(image.shape,dtype=float) + if 'hull' in self.mask_type: + hull = cv2.convexHull( numpy.array( face_detected.landmarksAsXY() ).reshape((-1,2)).astype(int) ).flatten().reshape( (-1,2) ) + cv2.fillConvexPoly( hull_mask,hull,(1,1,1) ) + + if self.mask_type == 'rect': + image_mask = face_mask + elif self.mask_type == 'faceHull': + image_mask = hull_mask + else: + image_mask = ((face_mask*hull_mask)) + + + if self.erosion_kernel is not None: + image_mask = cv2.erode(image_mask,self.erosion_kernel,iterations = 1) + + if self.blur_size!=0: + image_mask = cv2.blur(image_mask,(self.blur_size,self.blur_size)) + + return image_mask diff --git a/plugins/Extract_Align.py b/plugins/Extract_Align.py new file mode 100644 index 0000000..a28b755 --- /dev/null +++ b/plugins/Extract_Align.py @@ -0,0 +1,19 @@ +# Based on the original https://www.reddit.com/r/deepfakes/ code sample + contribs + +import cv2 + +from lib.aligner import get_align_mat + +class Extract(object): + def extract(self, image, face, size): + if face.landmarks == None: + print("Warning! landmarks not found. Switching to crop!") + return cv2.resize(face.image, (size, size)) + + alignment = get_align_mat( face ) + return self.transform( image, alignment, size, 48 ) + + def transform( self, image, mat, size, padding=0 ): + mat = mat * (size - 2 * padding) + mat[:,2] += padding + return cv2.warpAffine( image, mat, ( size, size ) ) diff --git a/plugins/Extract_Crop.py b/plugins/Extract_Crop.py new file mode 100644 index 0000000..aa1bcf7 --- /dev/null +++ b/plugins/Extract_Crop.py @@ -0,0 +1,7 @@ +# Based on the original https://www.reddit.com/r/deepfakes/ code sample + +import cv2 + +class Extract(object): + def extract(self, image, face, size): + return cv2.resize(face.image, (size, size)) \ No newline at end of file diff --git a/plugins/Model_LowMem.py b/plugins/Model_LowMem.py new file mode 100644 index 0000000..eda931e --- /dev/null +++ b/plugins/Model_LowMem.py @@ -0,0 +1,68 @@ +# Based on the original https://www.reddit.com/r/deepfakes/ code sample + contribs + +from keras.models import Model as KerasModel +from keras.layers import Input, Dense, Flatten, Reshape +from keras.layers.advanced_activations import LeakyReLU +from keras.layers.convolutional import Conv2D +from keras.optimizers import Adam + +from lib.ModelAE import ModelAE, TrainerAE +from lib.PixelShuffler import PixelShuffler + +IMAGE_SHAPE = (64, 64, 3) +ENCODER_DIM = 512 + +class Model(ModelAE): + def initModel(self): + optimizer = Adam(lr=5e-5, beta_1=0.5, beta_2=0.999) + x = Input(shape=IMAGE_SHAPE) + + self.autoencoder_A = KerasModel(x, self.decoder_A(self.encoder(x))) + self.autoencoder_B = KerasModel(x, self.decoder_B(self.encoder(x))) + + self.autoencoder_A.compile(optimizer=optimizer, loss='mean_absolute_error') + self.autoencoder_B.compile(optimizer=optimizer, loss='mean_absolute_error') + + def converter(self, swap): + autoencoder = self.autoencoder_B if not swap else self.autoencoder_A + return lambda img: autoencoder.predict(img) + + def conv(self, filters): + def block(x): + x = Conv2D(filters, kernel_size=5, strides=2, padding='same')(x) + x = LeakyReLU(0.1)(x) + return x + return block + + def upscale(self, filters): + def block(x): + x = Conv2D(filters * 4, kernel_size=3, padding='same')(x) + x = LeakyReLU(0.1)(x) + x = PixelShuffler()(x) + return x + return block + + def Encoder(self): + input_ = Input(shape=IMAGE_SHAPE) + x = input_ + x = self.conv(128)(x) + x = self.conv(256)(x) + x = self.conv(512)(x) + #x = self.conv(1024)(x) + x = Dense(ENCODER_DIM)(Flatten()(x)) + x = Dense(4 * 4 * 1024)(x) + x = Reshape((4, 4, 1024))(x) + x = self.upscale(512)(x) + return KerasModel(input_, x) + + def Decoder(self): + input_ = Input(shape=(8, 8, 512)) + x = input_ + x = self.upscale(256)(x) + x = self.upscale(128)(x) + x = self.upscale(64)(x) + x = Conv2D(3, kernel_size=5, padding='same', activation='sigmoid')(x) + return KerasModel(input_, x) + +class Trainer(TrainerAE): + """Empty inheritance""" \ No newline at end of file diff --git a/plugins/Model_Original.py b/plugins/Model_Original.py new file mode 100644 index 0000000..c9e74fe --- /dev/null +++ b/plugins/Model_Original.py @@ -0,0 +1,68 @@ +# Based on the original https://www.reddit.com/r/deepfakes/ code sample + contribs + +from keras.models import Model as KerasModel +from keras.layers import Input, Dense, Flatten, Reshape +from keras.layers.advanced_activations import LeakyReLU +from keras.layers.convolutional import Conv2D +from keras.optimizers import Adam + +from lib.ModelAE import ModelAE, TrainerAE +from lib.PixelShuffler import PixelShuffler + +IMAGE_SHAPE = (64, 64, 3) +ENCODER_DIM = 1024 + +class Model(ModelAE): + def initModel(self): + optimizer = Adam(lr=5e-5, beta_1=0.5, beta_2=0.999) + x = Input(shape=IMAGE_SHAPE) + + self.autoencoder_A = KerasModel(x, self.decoder_A(self.encoder(x))) + self.autoencoder_B = KerasModel(x, self.decoder_B(self.encoder(x))) + + self.autoencoder_A.compile(optimizer=optimizer, loss='mean_absolute_error') + self.autoencoder_B.compile(optimizer=optimizer, loss='mean_absolute_error') + + def converter(self, swap): + autoencoder = self.autoencoder_B if not swap else self.autoencoder_A + return lambda img: autoencoder.predict(img) + + def conv(self, filters): + def block(x): + x = Conv2D(filters, kernel_size=5, strides=2, padding='same')(x) + x = LeakyReLU(0.1)(x) + return x + return block + + def upscale(self, filters): + def block(x): + x = Conv2D(filters * 4, kernel_size=3, padding='same')(x) + x = LeakyReLU(0.1)(x) + x = PixelShuffler()(x) + return x + return block + + def Encoder(self): + input_ = Input(shape=IMAGE_SHAPE) + x = input_ + x = self.conv(128)(x) + x = self.conv(256)(x) + x = self.conv(512)(x) + x = self.conv(1024)(x) + x = Dense(ENCODER_DIM)(Flatten()(x)) + x = Dense(4 * 4 * 1024)(x) + x = Reshape((4, 4, 1024))(x) + x = self.upscale(512)(x) + return KerasModel(input_, x) + + def Decoder(self): + input_ = Input(shape=(8, 8, 512)) + x = input_ + x = self.upscale(256)(x) + x = self.upscale(128)(x) + x = self.upscale(64)(x) + x = Conv2D(3, kernel_size=5, padding='same', activation='sigmoid')(x) + return KerasModel(input_, x) + +class Trainer(TrainerAE): + """Empty inheritance""" \ No newline at end of file diff --git a/plugins/PluginLoader.py b/plugins/PluginLoader.py new file mode 100644 index 0000000..a2837ac --- /dev/null +++ b/plugins/PluginLoader.py @@ -0,0 +1,23 @@ + +class PluginLoader(): + @staticmethod + def get_extractor(name): + return PluginLoader._import("Extract", "Extract_{0}".format(name)) + + @staticmethod + def get_converter(name): + return PluginLoader._import("Convert", "Convert_{0}".format(name)) + + @staticmethod + def get_model(name): + return PluginLoader._import("Model", "Model_{0}".format(name)) + + @staticmethod + def get_trainer(name): + return PluginLoader._import("Trainer", "Model_{0}".format(name)) + + @staticmethod + def _import(attr, name): + print("Loading {} from {} plugin...".format(attr, name)) + module = __import__(name, globals(), locals(), [], 1) + return getattr(module, attr) diff --git a/requirements-gpu.txt b/requirements-gpu.txt new file mode 100644 index 0000000..cc2d687 --- /dev/null +++ b/requirements-gpu.txt @@ -0,0 +1,10 @@ +pathlib==1.0.1 +scandir==1.6 +h5py==2.7.1 +Keras==2.1.2 +opencv-python==3.3.0.10 +tensorflow-gpu==1.4.0 +scikit-image +dlib +face_recognition +tqdm diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9d559c5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +pathlib==1.0.1 +scandir==1.6 +h5py==2.7.1 +Keras==2.1.2 +opencv-python==3.3.0.10 +tensorflow==1.4.1 +scikit-image +dlib +face_recognition +tqdm diff --git a/scripts/convert.py b/scripts/convert.py new file mode 100644 index 0000000..58aedf7 --- /dev/null +++ b/scripts/convert.py @@ -0,0 +1,160 @@ +import cv2 +import re +from pathlib import Path +from lib.cli import DirectoryProcessor, FullPaths +from lib.utils import BackgroundGenerator +from lib.faces_detect import detect_faces + +from plugins.PluginLoader import PluginLoader + +class ConvertImage(DirectoryProcessor): + filename = '' + def create_parser(self, subparser, command, description): + self.parser = subparser.add_parser( + command, + help="Convert a source image to a new one with the face swapped.", + description=description, + epilog="Questions and feedback: \ + https://github.com/deepfakes/faceswap-playground" + ) + + def add_optional_arguments(self, parser): + parser.add_argument('-m', '--model-dir', + action=FullPaths, + dest="model_dir", + default="models", + help="Model directory. A directory containing the trained model \ + you wish to process. Defaults to 'models'") + + parser.add_argument('-s', '--swap-model', + action="store_true", + dest="swap_model", + default=False, + help="Swap the model. Instead of A -> B, swap B -> A.") + + parser.add_argument('-c', '--converter', + type=str, + choices=("Masked", "Adjust"), # case sensitive because this is used to load a plugin. + default="Masked", + help="Converter to use.") + + parser.add_argument('-fr', '--frame-ranges', + nargs="+", + type=str, + help="""frame ranges to apply transfer to. eg for frames 10 to 50 and 90 to 100 use --frame-ranges 10-50 90-100. + Files must have the framenumber as the last number in the name!""" + ) + + parser.add_argument('-d', '--discard-frames', + action="store_true", + dest="discard_frames", + default=False, + help="when use with --frame-ranges discards frames that are not processed instead of writing them out unchanged." + ) + + parser.add_argument('-b', '--blur-size', + type=int, + default=2, + help="Blur size. (Masked converter only)") + + parser.add_argument('-S', '--seamless', + action="store_true", + dest="seamless_clone", + default=False, + help="Seamless mode. (Masked converter only)") + + parser.add_argument('-M', '--mask-type', + type=str.lower, #lowercase this, because its just a string later on. + dest="mask_type", + choices=["rect", "facehull", "facehullandrect"], + default="facehullandrect", + help="Mask to use to replace faces. (Masked converter only)") + + parser.add_argument('-e', '--erosion-kernel-size', + dest="erosion_kernel_size", + type=int, + default=None, + help="Erosion kernel size. (Masked converter only)") + + parser.add_argument('-sm', '--smooth-mask', + action="store_true", + dest="smooth_mask", + default=True, + help="Smooth mask (Adjust converter only)") + + parser.add_argument('-aca', '--avg-color-adjust', + action="store_true", + dest="avg_color_adjust", + default=True, + help="Average color adjust. (Adjust converter only)") + + return parser + + def process(self): + # Original model goes with Adjust or Masked converter + # does the LowMem one work with only one? + model_name = "Original" # TODO Pass as argument + conv_name = self.arguments.converter + + model = PluginLoader.get_model(model_name)(self.arguments.model_dir) + if not model.load(self.arguments.swap_model): + print('Model Not Found! A valid model must be provided to continue!') + exit(1) + + converter = PluginLoader.get_converter(conv_name)(model.converter(False), + blur_size=self.arguments.blur_size, + seamless_clone=self.arguments.seamless_clone, + mask_type=self.arguments.mask_type, + erosion_kernel_size=self.arguments.erosion_kernel_size, + smooth_mask=self.arguments.smooth_mask, + avg_color_adjust=self.arguments.avg_color_adjust + ) + + batch = BackgroundGenerator(self.prepare_images(), 1) + + # frame ranges stuff... + self.frame_ranges = None + # split out the frame ranges and parse out "min" and "max" values + minmax = { + "min": 0, # never any frames less than 0 + "max": float("inf") + } + if self.arguments.frame_ranges: + self.frame_ranges = [tuple(map(lambda q: minmax[q] if q in minmax.keys() else int(q), v.split("-"))) for v in self.arguments.frame_ranges] + + # last number regex. I know regex is hacky, but its reliablyhacky(tm). + self.imageidxre = re.compile(r'(\d+)(?!.*\d)') + + for item in batch.iterator(): + self.convert(converter, item) + + def check_skip(self, filename): + try: + idx = int(self.imageidxre.findall(filename)[0]) + return not any(map(lambda b: b[0]<=idx<=b[1], self.frame_ranges)) + except: + return False + + + def convert(self, converter, item): + try: + (filename, image, faces) = item + + skip = self.check_skip(filename) + + if not skip: # process as normal + for idx, face in faces: + image = converter.patch_image(image, face) + + output_file = self.output_dir / Path(filename).name + + if self.arguments.discard_frames and skip: + return + cv2.imwrite(str(output_file), image) + except Exception as e: + print('Failed to convert image: {}. Reason: {}'.format(filename, e)) + + def prepare_images(self): + for filename in self.read_directory(): + image = cv2.imread(filename) + yield filename, image, self.get_faces(image) diff --git a/scripts/extract.py b/scripts/extract.py new file mode 100644 index 0000000..58fe5fa --- /dev/null +++ b/scripts/extract.py @@ -0,0 +1,30 @@ +import cv2 + +from pathlib import Path +from lib.cli import DirectoryProcessor +from plugins.PluginLoader import PluginLoader + +class ExtractTrainingData(DirectoryProcessor): + def create_parser(self, subparser, command, description): + self.parser = subparser.add_parser( + command, + help="Extract the faces from a pictures.", + description=description, + epilog="Questions and feedback: \ + https://github.com/deepfakes/faceswap-playground" + ) + + def process(self): + extractor_name = "Align" # TODO Pass as argument + extractor = PluginLoader.get_extractor(extractor_name)() + + try: + for filename in self.read_directory(): + image = cv2.imread(filename) + for idx, face in self.get_faces(image): + resized_image = extractor.extract(image, face, 256) + output_file = self.output_dir / Path(filename).stem + cv2.imwrite(str(output_file) + str(idx) + Path(filename).suffix, resized_image) + + except Exception as e: + print('Failed to extract from image: {}. Reason: {}'.format(filename, e)) diff --git a/scripts/train.py b/scripts/train.py new file mode 100644 index 0000000..113eca7 --- /dev/null +++ b/scripts/train.py @@ -0,0 +1,169 @@ +import cv2 +import numpy +import time + +from lib.utils import get_image_paths +from lib.cli import FullPaths +from plugins.PluginLoader import PluginLoader + +class TrainingProcessor(object): + arguments = None + + def __init__(self, subparser, command, description='default'): + self.parse_arguments(description, subparser, command) + + def process_arguments(self, arguments): + self.arguments = arguments + print("Model A Directory: {}".format(self.arguments.input_A)) + print("Model B Directory: {}".format(self.arguments.input_B)) + print("Training data directory: {}".format(self.arguments.model_dir)) + + self.process() + + def parse_arguments(self, description, subparser, command): + parser = subparser.add_parser( + command, + help="This command trains the model for the two faces A and B.", + description=description, + epilog="Questions and feedback: \ + https://github.com/deepfakes/faceswap-playground" + ) + + parser.add_argument('-A', '--input-A', + action=FullPaths, + dest="input_A", + default="input_A", + help="Input directory. A directory containing training images for face A.\ + Defaults to 'input'") + parser.add_argument('-B', '--input-B', + action=FullPaths, + dest="input_B", + default="input_B", + help="Input directory. A directory containing training images for face B.\ + Defaults to 'input'") + parser.add_argument('-m', '--model-dir', + action=FullPaths, + dest="model_dir", + default="models", + help="Model directory. This is where the training data will \ + be stored. Defaults to 'model'") + parser.add_argument('-p', '--preview', + action="store_true", + dest="preview", + default=False, + help="Show preview output. If not specified, write progress \ + to file.") + parser.add_argument('-v', '--verbose', + action="store_true", + dest="verbose", + default=False, + help="Show verbose output") + parser.add_argument('-s', '--save-interval', + type=int, + dest="save_interval", + default=100, + help="Sets the number of iterations before saving the model.") + parser.add_argument('-w', '--write-image', + action="store_true", + dest="write_image", + default=False, + help="Writes the training result to a file even on preview mode.") + parser.add_argument('-t', '--trainer', + type=str, + choices=("Original", "LowMem"), + default="Original", + help="Select which trainer to use, LowMem for cards < 2gb.") + parser.add_argument('-bs', '--batch-size', + type=int, + default=64, + help="Batch size, as a power of 2 (64, 128, 256, etc)") + parser = self.add_optional_arguments(parser) + parser.set_defaults(func=self.process_arguments) + + def add_optional_arguments(self, parser): + # Override this for custom arguments + return parser + + def process(self): + import threading + self.stop = False + self.save_now = False + + thr = threading.Thread(target=self.processThread, args=(), kwargs={}) + thr.start() + + if self.arguments.preview: + print('Using live preview') + while True: + try: + for name, image in self.preview_buffer.items(): + cv2.imshow(name, image) + + key = cv2.waitKey(1000) + if key == ord('\n') or key == ord('\r'): + break + if key == ord('s'): + self.save_now = True + except KeyboardInterrupt: + break + else: + input() # TODO how to catch a specific key instead of Enter? + # there isnt a good multiplatform solution: https://stackoverflow.com/questions/3523174/raw-input-in-python-without-pressing-enter + + print("Exit requested! The trainer will complete its current cycle, save the models and quit (it can take up a couple of seconds depending on your training speed). If you want to kill it now, press Ctrl + c") + self.stop = True + thr.join() # waits until thread finishes + + def processThread(self): + print('Loading data, this may take a while...') + # this is so that you can enter case insensitive values for trainer + trainer = self.arguments.trainer + trainer = trainer if trainer != "Lowmem" else "LowMem" + model = PluginLoader.get_model(trainer)(self.arguments.model_dir) + model.load(swapped=False) + + images_A = get_image_paths(self.arguments.input_A) + images_B = get_image_paths(self.arguments.input_B) + trainer = PluginLoader.get_trainer(trainer)(model, + images_A, + images_B, + batch_size=self.arguments.batch_size) + + try: + print('Starting. Press "Enter" to stop training and save model') + + for epoch in range(0, 1000000): + + save_iteration = epoch % self.arguments.save_interval == 0 + + trainer.train_one_step(epoch, self.show if (save_iteration or self.save_now) else None) + + if save_iteration: + model.save_weights() + + if self.stop: + model.save_weights() + exit() + + if self.save_now: + model.save_weights() + self.save_now = False + + except KeyboardInterrupt: + try: + model.save_weights() + except KeyboardInterrupt: + print('Saving model weights has been cancelled!') + exit(0) + + preview_buffer = {} + + def show(self, image, name=''): + try: + if self.arguments.preview: + self.preview_buffer[name] = image + elif self.arguments.write_image: + cv2.imwrite('_sample_{}.jpg'.format(name), image) + except Exception as e: + print("could not preview sample") + print(e)