First commit

influence/all_CNN_c.py

@@ -0,0 +1,142 @@
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals  
+
+import abc
+import sys
+
+import numpy as np
+import pandas as pd
+from sklearn import linear_model, preprocessing, cluster
+import matplotlib.pyplot as plt
+import seaborn as sns
+import scipy.linalg as slin
+import scipy.sparse.linalg as sparselin
+import scipy.sparse as sparse 
+
+import os.path
+import time
+import IPython
+import tensorflow as tf
+import math
+
+from influence.genericNeuralNet import GenericNeuralNet, variable, variable_with_weight_decay
+from influence.dataset import DataSet
+
+def conv2d(x, W, r):
+    return tf.nn.conv2d(x, W, strides=[1, r, r, 1], padding='VALID')
+
+def softplus(x):
+    return tf.log(tf.exp(x) + 1)
+
+
+class All_CNN_C(GenericNeuralNet):
+
+    def __init__(self, input_side, input_channels, conv_patch_size, hidden1_units, hidden2_units, hidden3_units, weight_decay, **kwargs):
+        self.weight_decay = weight_decay
+        self.input_side = input_side
+        self.input_channels = input_channels
+        self.input_dim = self.input_side * self.input_side * self.input_channels
+        self.conv_patch_size = conv_patch_size
+        self.hidden1_units = hidden1_units
+        self.hidden2_units = hidden2_units
+        self.hidden3_units = hidden3_units
+
+        super(All_CNN_C, self).__init__(**kwargs)
+
+
+    def conv2d_softplus(self, input_x, conv_patch_size, input_channels, output_channels, stride):
+        weights = variable_with_weight_decay(
+            'weights', 
+            [conv_patch_size * conv_patch_size * input_channels * output_channels],
+            stddev=2.0 / math.sqrt(float(conv_patch_size * conv_patch_size * input_channels)),
+            wd=self.weight_decay)
+        biases = variable(
+            'biases',
+            [output_channels],
+            tf.constant_initializer(0.0))
+        weights_reshaped = tf.reshape(weights, [conv_patch_size, conv_patch_size, input_channels, output_channels])
+        hidden = tf.nn.tanh(conv2d(input_x, weights_reshaped, stride) + biases)
+
+        return hidden
+
+
+
+    def get_all_params(self):
+        all_params = []
+        for layer in ['h1_a', 'h1_c', 'h2_a', 'h2_c', 'h3_a', 'h3_c', 'softmax_linear']:        
+            for var_name in ['weights', 'biases']:
+                temp_tensor = tf.get_default_graph().get_tensor_by_name("%s/%s:0" % (layer, var_name))            
+                all_params.append(temp_tensor)      
+        return all_params        
+
+
+    def retrain(self, num_steps, feed_dict):        
+
+        retrain_dataset = DataSet(feed_dict[self.input_placeholder], feed_dict[self.labels_placeholder])
+
+        for step in xrange(num_steps):   
+            iter_feed_dict = self.fill_feed_dict_with_batch(retrain_dataset)
+            self.sess.run(self.train_op, feed_dict=iter_feed_dict)
+
+
+    def placeholder_inputs(self):
+        input_placeholder = tf.placeholder(
+            tf.float32, 
+            shape=(None, self.input_dim),
+            name='input_placeholder')
+        labels_placeholder = tf.placeholder(
+            tf.int32,             
+            shape=(None),
+            name='labels_placeholder')
+        return input_placeholder, labels_placeholder
+
+
+    def inference(self, input_x):        
+
+        input_reshaped = tf.reshape(input_x, [-1, self.input_side, self.input_side, self.input_channels])
+
+        # Hidden 1
+        with tf.variable_scope('h1_a'):
+            h1_a = self.conv2d_softplus(input_reshaped, self.conv_patch_size, self.input_channels, self.hidden1_units, stride=1)
+
+        with tf.variable_scope('h1_c'):
+            h1_c = self.conv2d_softplus(h1_a, self.conv_patch_size, self.hidden1_units, self.hidden1_units, stride=2)
+
+        # Hidden 2
+        with tf.variable_scope('h2_a'):
+            h2_a = self.conv2d_softplus(h1_c, self.conv_patch_size, self.hidden1_units, self.hidden2_units, stride=1)
+
+        with tf.variable_scope('h2_c'):
+            h2_c = self.conv2d_softplus(h2_a, self.conv_patch_size, self.hidden2_units, self.hidden2_units, stride=2)
+
+        # Shared layers / hidden 3
+        with tf.variable_scope('h3_a'):
+            h3_a = self.conv2d_softplus(h2_c, self.conv_patch_size, self.hidden2_units, self.hidden3_units, stride=1)        
+
+        last_layer_units = 10
+        with tf.variable_scope('h3_c'):
+            h3_c = self.conv2d_softplus(h3_a, 1, self.hidden3_units, last_layer_units, stride=1)
+
+        h3_d = tf.reduce_mean(h3_c, axis=[1, 2])
+
+        with tf.variable_scope('softmax_linear'):
+
+            weights = variable_with_weight_decay(
+                'weights', 
+                [last_layer_units * self.num_classes],
+                stddev=1.0 / math.sqrt(float(last_layer_units)),
+                wd=self.weight_decay)            
+            biases = variable(
+                'biases',
+                [self.num_classes],
+                tf.constant_initializer(0.0))
+
+            logits = tf.matmul(h3_d, tf.reshape(weights, [last_layer_units, self.num_classes])) + biases
+
+        return logits
+
+    def predictions(self, logits):
+        preds = tf.nn.softmax(logits, name='preds')
+        return preds

influence/binaryLogisticRegressionWithLBFGS.py

@@ -0,0 +1,163 @@
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals  
+
+import abc
+import sys
+
+import numpy as np
+import pandas as pd
+from sklearn import linear_model, preprocessing, cluster
+import scipy.linalg as slin
+import scipy.sparse.linalg as sparselin
+import scipy.sparse as sparse 
+from scipy.special import expit 
+
+import os.path
+import time
+import tensorflow as tf
+import math
+
+from influence.hessians import hessians
+from influence.genericNeuralNet import GenericNeuralNet, variable, variable_with_weight_decay
+from influence.logisticRegressionWithLBFGS import LogisticRegressionWithLBFGS
+
+
+class BinaryLogisticRegressionWithLBFGS(LogisticRegressionWithLBFGS):
+
+    def __init__(self, **kwargs):
+
+        super(BinaryLogisticRegressionWithLBFGS, self).__init__(**kwargs)
+
+        C = 1.0 / (self.num_train_examples * self.weight_decay)        
+        self.sklearn_model = linear_model.LogisticRegression(
+            C=C,
+            tol=1e-8,
+            fit_intercept=False, 
+            solver='lbfgs',
+            warm_start=True,
+            max_iter=1000)
+
+        C_minus_one = 1.0 / ((self.num_train_examples - 1) * self.weight_decay)
+        self.sklearn_model_minus_one = linear_model.LogisticRegression(
+            C=C_minus_one,
+            tol=1e-8,
+            fit_intercept=False, 
+            solver='lbfgs',
+            warm_start=True,
+            max_iter=1000)        
+
+
+    def inference(self, input):                
+        with tf.variable_scope('softmax_linear'):
+            weights = variable_with_weight_decay(
+                'weights', 
+                [self.input_dim],
+                stddev=1.0 / math.sqrt(float(self.input_dim)),
+                wd=self.weight_decay)            
+
+            logits = tf.matmul(input, tf.reshape(weights, [self.input_dim, 1])) # + biases
+            zeros = tf.zeros_like(logits)            
+            logits_with_zeros = tf.concat([zeros, logits], 1)
+
+        self.weights = weights
+
+        return logits_with_zeros
+
+
+    def set_params(self):
+        self.W_placeholder = tf.placeholder(
+            tf.float32,
+            shape=[self.input_dim],
+            name='W_placeholder')
+        set_weights = tf.assign(self.weights, self.W_placeholder, validate_shape=True)
+        return [set_weights]
+
+
+    # Special-purpose function for paper experiments
+    # that has flags for ignoring training error or Hessian
+    def get_influence_on_test_loss(self, test_indices, train_idx, 
+        approx_type='cg', approx_params=None, force_refresh=True, test_description=None,
+        loss_type='normal_loss',
+        ignore_training_error=False,
+        ignore_hessian=False
+        ):
+
+        test_grad_loss_no_reg_val = self.get_test_grad_loss_no_reg_val(test_indices, loss_type=loss_type)
+
+        print('Norm of test gradient: %s' % np.linalg.norm(np.concatenate(test_grad_loss_no_reg_val)))
+
+        start_time = time.time()
+
+        if test_description is None:
+            test_description = test_indices
+
+        approx_filename = os.path.join(self.train_dir, '%s-%s-%s-test-%s.npz' % (self.model_name, approx_type, loss_type, test_description))
+        if ignore_hessian == False:
+            if os.path.exists(approx_filename) and force_refresh == False:
+                inverse_hvp = list(np.load(approx_filename)['inverse_hvp'])
+                print('Loaded inverse HVP from %s' % approx_filename)
+            else:
+                inverse_hvp = self.get_inverse_hvp(
+                    test_grad_loss_no_reg_val,
+                    approx_type,
+                    approx_params)
+                np.savez(approx_filename, inverse_hvp=inverse_hvp)
+                print('Saved inverse HVP to %s' % approx_filename)
+        else:
+            inverse_hvp = test_grad_loss_no_reg_val
+
+        duration = time.time() - start_time
+        print('Inverse HVP took %s sec' % duration)
+
+
+        start_time = time.time()
+
+        num_to_remove = len(train_idx)
+        predicted_loss_diffs = np.zeros([num_to_remove])
+        for counter, idx_to_remove in enumerate(train_idx):            
+
+            if ignore_training_error == False:
+                single_train_feed_dict = self.fill_feed_dict_with_one_ex(self.data_sets.train, idx_to_remove)      
+                train_grad_loss_val = self.sess.run(self.grad_total_loss_op, feed_dict=single_train_feed_dict)
+            else:
+                train_grad_loss_val = [-(self.data_sets.train.labels[idx_to_remove] * 2 - 1) * self.data_sets.train.x[idx_to_remove, :]]
+            predicted_loss_diffs[counter] = np.dot(np.concatenate(inverse_hvp), np.concatenate(train_grad_loss_val)) / self.num_train_examples
+
+        duration = time.time() - start_time
+        print('Multiplying by %s train examples took %s sec' % (num_to_remove, duration))
+
+        return predicted_loss_diffs
+
+
+    def get_loo_influences(self):
+
+        X_train = self.data_sets.train.x
+        Y_train = self.data_sets.train.labels * 2 - 1
+        theta = self.sess.run(self.params)[0]
+
+        # Pre-calculate inverse covariance matrix
+        n = X_train.shape[0]        
+        dim = X_train.shape[1]
+        cov = np.zeros([dim, dim])
+
+        probs = expit(np.dot(X_train, theta.T))
+        weighted_X_train = np.reshape(probs * (1 - probs), (-1, 1)) * X_train
+
+        cov = np.dot(X_train.T, weighted_X_train) / n 
+        cov += self.weight_decay * np.eye(dim)                    
+
+        cov_lu_factor = slin.lu_factor(cov)
+
+        assert(len(Y_train.shape) == 1)
+        x_train_theta = np.reshape(X_train.dot(theta.T), [-1])
+        sigma = expit(-Y_train * x_train_theta)
+
+        d_theta = slin.lu_solve(cov_lu_factor, X_train.T).T 
+
+        quad_x = np.sum(X_train * d_theta, axis=1)
+
+        return sigma * quad_x
+
+