From 42730a30aaa5ecd5a5c93d89bd01d35bd9417a39 Mon Sep 17 00:00:00 2001 From: GZWQ <1520761588@qq.com> Date: Wed, 14 Nov 2018 17:42:59 -0600 Subject: [PATCH] Add files via upload --- README.md | 43 ++++++++++++- load_data.py | 126 ++++++++++++++++++++++++++++++++++++++ train_.py | 132 ++++++++++++++++++++++++++++++++++++++++ train_class.py | 135 +++++++++++++++++++++++++++++++++++++++++ utility_privacy.py | 146 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 581 insertions(+), 1 deletion(-) create mode 100644 load_data.py create mode 100644 train_.py create mode 100644 train_class.py create mode 100644 utility_privacy.py diff --git a/README.md b/README.md index 09656c2..e585e2c 100644 --- a/README.md +++ b/README.md @@ -1 +1,42 @@ -# adversarial_obfuscator \ No newline at end of file +# adversarial-obfuscator + +Unofficial implementation of the paper [Protecting Visual Secrets using Adversarial Nets](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8014908) + +Here is my experiment results. + +## Utility Evaluation + +Scene 1 : Trained on **original** images, Test on **original** images + +Scene 2 : Trained on **original** images, Test on **generated** images + +Scene 3 : Trained on **generated** images, Test on **generated** images + + + +| Scene | Accuracy | +| :---: | :------: | +| 1 | 94.61% | +| 2 | 87.785% | +| 3 | 90.643% | + + + +## Privacy Evaluation + +Scene 1 : Trained on **original** images, Test on **generated** images + +Scene 2 : Trained on **generated** images, Test on **generated** images + + + +| Scene | Accuracy | +| :---: | :------: | +| 1 | 50% | +| 2 | 83.537% | + + + +# Acknowledgments + +Inspired by [Erik Linder-Norén](https://github.com/eriklindernoren/Keras-GAN/tree/master/dcgan) and [JWC](https://github.com/yushuinanrong/PPRL-VGAN) \ No newline at end of file diff --git a/load_data.py b/load_data.py new file mode 100644 index 0000000..6a86e7e --- /dev/null +++ b/load_data.py @@ -0,0 +1,126 @@ +from keras.datasets import cifar10 +import numpy as np +from keras.utils import np_utils +import qrcode,cv2 +nb_classes = 2 +from PIL import Image +import matplotlib.pyplot as plt + +def cifar(): + print("Loading cifar10 data ...") + + # the data, shuffled and split between train and test sets + (X_train, y_train), (X_test, y_test) = cifar10.load_data() + + ## choose "airplane" and "horse" training pics + airplane_idx = [] + horse_idx = [] + for index in range(len(y_train)): + label = y_train[index] + if label == 0: + airplane_idx.append(index) + elif label == 7: + horse_idx.append(index) + + airplane_data = X_train[airplane_idx,:,:,:] + horse_data = X_train[horse_idx,:,:,:] + X_train = np.concatenate([airplane_data,horse_data],axis=0) + airplane_data = np.zeros(shape=(5000,1)) + horse_data = np.ones(shape=(5000,1)) + y_train = np.concatenate([airplane_data,horse_data],axis=0) + + ## choose "airplane" and "horse" test pics + airplane_idx = [] + horse_idx = [] + for index in range(len(y_test)): + label = y_test[index] + if label == 0: + airplane_idx.append(index) + elif label == 7: + horse_idx.append(index) + + airplane_data = X_test[airplane_idx, :, :, :] + horse_data = X_test[horse_idx, :, :, :] + X_test = np.concatenate([airplane_data, horse_data], axis=0) + airplane_data = np.zeros(shape=(1000,1)) + horse_data = np.ones(shape=(1000,1)) + y_test = np.concatenate([airplane_data, horse_data], axis=0) + + print('X_train shape:', X_train.shape) + print(X_train.shape[0], 'train samples') + print(X_test.shape[0], 'test samples') + + # convert class vectors to binary class matrices + # Y_train = np_utils.to_categorical(y_train, nb_classes) + # Y_test = np_utils.to_categorical(y_test, nb_classes) + + Y_train = y_train.reshape(-1,1) + Y_test = y_test.reshape(-1,1) + + + X_train = X_train.astype('float32') + X_test = X_test.astype('float32') + + print("Finished data loading!!!") + return X_train,Y_train,X_test,Y_test + +def load_cifar10(): + X_train_public, Y_train_public, X_test_public, Y_test_public = cifar() + X_train_secret,X_test_secret = [],[] + X_train_public_temp,X_test_public_temp = [],[] + + + print("Generating secret training dataset ...") + for index,pic in enumerate(X_train_public): + pic_temp = (pic-127.5) / 127.5 + X_train_public_temp.append(pic_temp) + qr = gene_qr(index) + qr_pos = np.random.uniform(0,21,size=2) + pic = Image.fromarray(np.uint8(pic)) + pic.paste(qr,(int(qr_pos[0]),int(qr_pos[1]))) + # plt.imshow(pic) + # plt.show() + pic = np.asarray(pic) + pic = (pic.copy()-127.5) / 127.5 + X_train_secret.append(pic) + if index % 100 == 0: + print(index,"/",len(X_train_public)) + + + print("Generating secret test dataset ...") + for index,pic in enumerate(X_test_public): + pic_temp = (pic - 127.5) / 127.5 + X_test_public_temp.append(pic_temp) + qr = gene_qr(index) + qr_pos = np.random.uniform(0,21,size=2) + pic = Image.fromarray(np.uint8(pic)) + pic.paste(qr, (int(qr_pos[0]), int(qr_pos[1]))) + # plt.imshow(pic) + # plt.show() + pic = np.asarray(pic) + pic = (pic.copy() - 127.5) / 127.5 + X_test_secret.append(pic) + if index % 100 == 0: + print(index,"/",len(X_test_public)) + + X_train_public = np.array(X_train_public_temp) + X_test_public = np.array(X_test_public_temp) + X_train_secret = np.array(X_train_secret) + X_test_secret = np.array(X_test_secret) + + return X_train_public, Y_train_public, X_test_public, Y_test_public,\ + X_train_secret, Y_train_public, X_test_secret, Y_test_public + + +def gene_qr(index): + qr = qrcode.QRCode( + version=1, + error_correction=qrcode.constants.ERROR_CORRECT_L, + box_size=10, + border=2, + ) + qr.add_data(str(index)) + qr.make(fit=True) + img = qr.make_image() + img = img.resize((10,10)) + return img diff --git a/train_.py b/train_.py new file mode 100644 index 0000000..3f56a62 --- /dev/null +++ b/train_.py @@ -0,0 +1,132 @@ +from __future__ import print_function, division + +import load_data +from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply +from keras.layers import Conv2D, Deconv2D +from keras.models import Sequential, Model +from keras.optimizers import Adam +import random +import matplotlib.pyplot as plt +import os,time +from PIL import Image +# from tqdm import tqdm +import numpy as np + +class ae_gan(): + def __init__(self): + self.img_rows, self.img_cols, self.channels = 32, 32, 3 + self.img_shape = (self.img_rows, self.img_cols, self.channels) + + optimizer = Adam(lr=0.0002,beta_1=0.5) + self.attack = self.discriminator() + self.attack.compile(loss=['binary_crossentropy'], + optimizer=optimizer, + metrics=['accuracy']) + + self.ae = self.autoencoder() + input_image = Input(shape=self.img_shape) + generated_img = self.ae(input_image) + + self.attack.trainable = False + valid = self.attack(generated_img) + + self.combined_model = Model(input_image,[valid,generated_img]) + self.combined_model.compile(loss=['binary_crossentropy','mae'], + loss_weights=[0.5, 0.5], + optimizer=optimizer) + + def autoencoder(self): + input = Input(shape=self.img_shape) + h = Conv2D(64, (5, 5), strides=2, padding='same', activation='relu')(input) + h = Conv2D(128, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Conv2D(256, (5, 5), strides=2, padding='same', activation='relu')(h) + encoded = Conv2D(512, (5, 5), strides=2, padding='same', activation='relu')(h) + + h = Deconv2D(512, (5, 5), strides=2, padding='same', activation='relu')(encoded) + h = Deconv2D(256, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Deconv2D(128, (5, 5), strides=2, padding='same', activation='relu')(h) + decoded = Deconv2D(3, (5, 5), strides=2, padding='same', activation='tanh')(h) + + auto_encoder = Model(input, decoded) + auto_encoder.summary() + + return auto_encoder + + def discriminator(self): + input = Input(shape=self.img_shape) + h = Conv2D(64, (5, 5), strides=2, padding='same', activation='relu')(input) + h = Conv2D(128, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Conv2D(256, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Conv2D(512, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Flatten()(h) + output_secret = Dense(1, activation='relu')(h) + discriminator = Model(input, output_secret) + discriminator.summary() + + return discriminator + + def train(self,epochs, batch_size=128, sample_interval=50): + x_train_public, y_train_public, _, _, \ + x_train_secret, y_train_secret, _, _ = load_data.load_cifar10() + + label_secret = np.ones(shape=(batch_size, 1)) + label_public = np.zeros(shape=(batch_size, 1)) + + for epoch in range(1,epochs+1): + start = time.time() + print("In the epoch ",epoch,"/",epochs) + + ####### generate pics for public pics ####### + idx_public = random.sample(range(0, x_train_public.shape[0]), batch_size) + image_batch_public = x_train_public[idx_public, :, :, :] + generated_images_public = self.ae.predict(image_batch_public) + + ####### generate pics for secret pics ####### + idx_secret = random.sample(range(0, x_train_secret.shape[0]), batch_size) + image_batch_secret = x_train_secret[idx_secret, :, :, :] + generated_images_secret = self.ae.predict(image_batch_secret) + + l1 = self.attack.train_on_batch(image_batch_public,label_public) + l2 = self.attack.train_on_batch(generated_images_public,label_public) + l3 = self.attack.train_on_batch(image_batch_secret,label_secret) + l4 = self.attack.train_on_batch(generated_images_secret,label_secret) + + g_loss1 = self.combined_model.train_on_batch(image_batch_public,[label_public,image_batch_public]) + g_loss2 = self.combined_model.train_on_batch(image_batch_secret,[label_public,image_batch_secret]) + + print("Epoch ",epoch,"took time",time.time()-start) + if epoch % 20 == 0: + self.save_model(epoch) + self.sample_images(image_batch_secret[0],epoch,'secret') + self.sample_images(image_batch_public[0],epoch,'public') + def sample_images(self, image, epoch, label): + image = np.expand_dims(image,axis=0) + gen_imgs = self.ae.predict(image) # output pixel size is between (-1,1) + + gen_imgs = 127.5 * gen_imgs + 127.5 + + data = gen_imgs[0].astype(np.uint8) + output_path = './images_vaegan_'+label+'/' + if not os.path.exists(output_path): + os.mkdir(output_path) + img = Image.fromarray(data,'RGB') + img.save(output_path + "%d.png" % epoch) + plt.close() + + def save_model(self, epoch): + + def save(model, epoch, model_name): + output_path = './models_vaegan/' + if not os.path.exists(output_path): + os.mkdir(output_path) + model_path = output_path + str(epoch) + "_" + model_name + ".h5" + model.save(model_path) + + save(self.ae, epoch, "autoencoder") + # save(self.attack, epoch, "discriminator") + +if __name__ == '__main__': + model = ae_gan() + model.train(epochs=1000,batch_size=32,sample_interval=200) + + diff --git a/train_class.py b/train_class.py new file mode 100644 index 0000000..0c4cfd6 --- /dev/null +++ b/train_class.py @@ -0,0 +1,135 @@ +from __future__ import print_function, division + +import load_data +from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply +from keras.layers import Conv2D, Deconv2D +from keras.models import Sequential, Model +from keras.optimizers import Adam +import random +import matplotlib.pyplot as plt +import os,time +from PIL import Image +# from tqdm import tqdm +import numpy as np + +class ae_gan(): + def __init__(self): + self.img_rows, self.img_cols, self.channels = 32, 32, 3 + self.img_shape = (self.img_rows, self.img_cols, self.channels) + + optimizer = Adam(lr=0.0002,beta_1=0.5) + self.attack = self.discriminator() + self.attack.compile(loss=['binary_crossentropy','binary_crossentropy'], + optimizer=optimizer, + metrics=['accuracy']) + + self.ae = self.autoencoder() + input_image = Input(shape=self.img_shape) + generated_img = self.ae(input_image) + + self.attack.trainable = False + valid,class_ = self.attack(generated_img) + + self.combined_model = Model(input_image,[valid,generated_img,class_]) + self.combined_model.compile(loss=['binary_crossentropy','mae','binary_crossentropy'], + loss_weights=[0.2, 0.4, 0.4], + optimizer=optimizer) + + def autoencoder(self): + input = Input(shape=self.img_shape) + h = Conv2D(64, (5, 5), strides=2, padding='same', activation='relu')(input) + h = Conv2D(128, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Conv2D(256, (5, 5), strides=2, padding='same', activation='relu')(h) + encoded = Conv2D(512, (5, 5), strides=2, padding='same', activation='relu')(h) + + h = Deconv2D(512, (5, 5), strides=2, padding='same', activation='relu')(encoded) + h = Deconv2D(256, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Deconv2D(128, (5, 5), strides=2, padding='same', activation='relu')(h) + decoded = Deconv2D(3, (5, 5), strides=2, padding='same', activation='tanh')(h) + + auto_encoder = Model(input, decoded) + auto_encoder.summary() + + return auto_encoder + + def discriminator(self): + input = Input(shape=self.img_shape) + h = Conv2D(64, (5, 5), strides=2, padding='same', activation='relu')(input) + h = Conv2D(128, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Conv2D(256, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Conv2D(512, (5, 5), strides=2, padding='same', activation='relu')(h) + h = Flatten()(h) + output_secret = Dense(1, activation='sigmoid')(h) + output_class = Dense(1,activation='sigmoid')(h) + discriminator = Model(input, [output_secret,output_class]) + discriminator.summary() + + return discriminator + + def train(self,epochs, batch_size=128, sample_interval=50): + x_train_public, y_train_public, _, _, \ + x_train_secret, y_train_secret, _, _ = load_data.load_cifar10() + + label_secret = np.ones(shape=(batch_size, 1)) + label_public = np.zeros(shape=(batch_size, 1)) + + for epoch in range(epochs): + start = time.time() + print("In the epoch ",epoch,"/",epochs) + + ####### generate pics for public pics ####### + idx_public = random.sample(range(0, x_train_public.shape[0]), batch_size) + image_batch_public = x_train_public[idx_public, :, :, :] + label_batch_public = y_train_public[idx_public, :] + generated_images_public = self.ae.predict(image_batch_public) + + ####### generate pics for secret pics ####### + idx_secret = random.sample(range(0, x_train_secret.shape[0]), batch_size) + image_batch_secret = x_train_secret[idx_secret, :, :, :] + label_batch_secret = y_train_secret[idx_secret, :] + generated_images_secret = self.ae.predict(image_batch_secret) + + l1 = self.attack.train_on_batch(image_batch_public,[label_public,label_batch_public]) + l2 = self.attack.train_on_batch(generated_images_public,[label_public,label_batch_public]) + l3 = self.attack.train_on_batch(image_batch_secret,[label_secret,label_batch_secret]) + l4 = self.attack.train_on_batch(generated_images_secret,[label_secret,label_batch_secret]) + + g_loss1 = self.combined_model.train_on_batch(image_batch_public,[label_public,image_batch_public,label_batch_public]) + g_loss2 = self.combined_model.train_on_batch(image_batch_secret,[label_public,image_batch_secret,label_batch_secret]) + + print("Epoch ",epoch,"took time",time.time()-start) + if epoch % 20 == 0: + self.save_model(epoch) + self.sample_images(image_batch_secret[0],epoch,'secret') + self.sample_images(image_batch_public[0],epoch,'public') + def sample_images(self, image, epoch, label): + image = np.expand_dims(image,axis=0) + gen_imgs = self.ae.predict(image) # output pixel size is between (-1,1) + + gen_imgs = 127.5 * gen_imgs + 127.5 + + data = gen_imgs[0].astype(np.uint8) + output_path = './images_vaegan_'+label+'/' + if not os.path.exists(output_path): + os.mkdir(output_path) + img = Image.fromarray(data,'RGB') + img.save(output_path + "%d.png" % epoch) + plt.close() + + def save_model(self, epoch): + + def save(model, epoch, model_name): + output_path = './models_vaegan/' + if not os.path.exists(output_path): + os.mkdir(output_path) + model_path = output_path + str(epoch) + "_" + model_name + ".h5" + model.save(model_path) + + save(self.ae, epoch, "autoencoder") + # save(self.attack, epoch, "discriminator") + +if __name__ == '__main__': + model = ae_gan() + model.train(epochs=1000,batch_size=32,sample_interval=200) + + diff --git a/utility_privacy.py b/utility_privacy.py new file mode 100644 index 0000000..d4fd9e7 --- /dev/null +++ b/utility_privacy.py @@ -0,0 +1,146 @@ +from __future__ import print_function +from keras.layers import Dense, Dropout, Flatten,Input +from keras.optimizers import Adam,SGD +from keras.models import Model +from keras.layers.normalization import BatchNormalization +from keras.layers import Conv2D, MaxPooling2D +from keras.layers.advanced_activations import LeakyReLU +import load_data +from keras.models import load_model +import numpy as np +import os + +def generated_images(data): + model = load_model('./models_vaegan/1980_autoencoder.h5') + generated_images = model.predict(data) + return generated_images + +x_train_public, y_train_public, x_test_public, y_test_public,\ + x_train_secret, y_train_secret, x_test_secret, y_test_secret = load_data.load_cifar10() +x_train_public_generated = generated_images(x_train_public) +x_test_public_generated = generated_images(x_test_public) +x_train_secret_generated = generated_images(x_train_secret) +x_test_secret_generated = generated_images(x_test_secret) + + +def cnn_model(): + d0 = Input((x_train_public.shape[1:])) + # x0 = Dense(img_rows*img_cols*1, activation = 'relu')(d0) + # x0 = Reshape((img_rows,img_cols,1))(x0) + x = Conv2D(32, (5, 5), padding='same', name='id_conv1')(d0) + x = LeakyReLU(0.2)(x) + x = BatchNormalization()(x) + x = MaxPooling2D((3, 3), padding='same', strides=(2, 2))(x) + + x = Conv2D(32, (5, 5), padding='same', name='id_conv2')(x) + x = LeakyReLU(0.2)(x) + x = BatchNormalization()(x) + x = MaxPooling2D((3, 3), padding='same', strides=(2, 2))(x) + + x = Conv2D(32, (3, 3), padding='same', name='id_conv3')(x) + x = LeakyReLU(0.2)(x) + x = BatchNormalization()(x) + x = MaxPooling2D((3, 3), padding='same', strides=(2, 2))(x) + + x = Conv2D(32, (3, 3), padding='same', name='id_conv4')(x) + x = LeakyReLU(0.2)(x) + x = BatchNormalization()(x) + x = MaxPooling2D((3, 3), padding='same', strides=(2, 2))(x) + + x = Flatten()(x) + x = Dense(32, name='id_dense1')(x) + x = LeakyReLU(0.2)(x) + x = Dropout(0.5)(x) + output_main = Dense(1, activation='sigmoid', name='id_dense2')(x) + + return Model(d0, output_main) + + +sgd = SGD(lr=0.005, momentum=0.9, decay=1e-7, nesterov=True) +model = cnn_model() +model.compile(loss=['binary_crossentropy'], + optimizer=sgd, + metrics=['accuracy']) +model.summary() + +def eval_fun(x_train,y_train,x_test,y_test,util_privacy): + output = '../'+util_privacy[0]+'/' + if not os.path.isdir(output): + os.mkdir(output) + result = model.fit(x_train,y_train, + batch_size=50, + epochs=10, + validation_data=(x_test,y_test), + shuffle=True) + filename = output + util_privacy[1]+ '.txt' + print(result.history['val_acc']) + with open(filename, mode='w') as f1: + f1.write(str(result.history['val_acc'])) + f1.write('\n') + f1.write('Ave_ACC is : ' + str + (sum(result.history['val_acc']) / len(result.history['val_acc']))) + + +if __name__ == '__main__': + + + ####################################################### + + original_data_train = np.concatenate((x_train_public, x_train_secret), axis=0) + original_data_train_class_label = np.concatenate((y_train_public,y_train_secret),axis=0) + original_data_test = np.concatenate((x_test_public,x_test_secret),axis=0) + original_data_test_class_label = np.concatenate((y_test_public,y_test_secret),axis=0) + + generated_data_train = np.concatenate((x_train_public_generated, x_train_secret_generated), axis=0) + generated_data_train_class_label = np.concatenate((y_train_public,y_train_secret),axis=0) + generated_data_test = np.concatenate((x_test_public_generated, x_test_secret_generated), axis=0) + generated_data_test_class_label = np.concatenate((y_test_public,y_test_secret),axis=0) + + label_original_data = np.zeros(shape=(len(generated_data_train), 1)) + label_original_data[len(generated_data_train) // 2:, :] = 1 + + with_secret_label = np.zeros(shape=(len(generated_data_test), 1)) + with_secret_label[len(generated_data_test) // 2:, :] = 1 + ####################################################### + Eval = 'up' # 'privacy' / 'up' + if Eval == 'util': + print("In the UTILITY process") + + #eval_fun(original_data_train, original_data_train_class_label, + # original_data_test, original_data_test_class_label,['utility','baseline']) + + + eval_fun(original_data_train, original_data_train_class_label, + generated_data_test, generated_data_test_class_label,['utility','a']) + + + eval_fun(generated_data_train, generated_data_train_class_label, + generated_data_test, generated_data_test_class_label,['utility','b']) + + elif Eval == 'privacy': + + print("In the PRIVACY process") + + eval_fun(original_data_train, label_original_data, + generated_data_test, with_secret_label,['privacy','weak']) + + eval_fun(generated_data_train, label_original_data, + generated_data_test, with_secret_label,['privacy','strong']) + elif Eval =='up': + print("In the BOTH process") + #eval_fun(original_data_train, original_data_train_class_label, + # original_data_test, original_data_test_class_label, ['utility', 'baseline']) + + eval_fun(original_data_train, original_data_train_class_label, + generated_data_test, generated_data_test_class_label, ['utility', 'a']) + + eval_fun(generated_data_train, generated_data_train_class_label, + generated_data_test, generated_data_test_class_label, ['utility', 'b']) + + eval_fun(original_data_train, label_original_data, + generated_data_test, with_secret_label, ['privacy', 'weak']) + + eval_fun(generated_data_train, label_original_data, + generated_data_test, with_secret_label, ['privacy', 'strong']) + +